diff --git a/backend/app/routes/migration_routes.py b/backend/app/routes/migration_routes.py index db28fee..62e8fbc 100644 --- a/backend/app/routes/migration_routes.py +++ b/backend/app/routes/migration_routes.py @@ -1,9 +1,13 @@ +import json +from collections import defaultdict from uuid import UUID from fastapi import APIRouter, Depends, HTTPException +from supabase._async.client import AsyncClient from app.core.dependencies import get_current_admin -from app.schemas.classification_schemas import Classification +from app.core.supabase import get_async_supabase +from app.schemas.classification_schemas import Classification, ExtractedFile from app.schemas.migration_schemas import Migration, MigrationCreate from app.schemas.relationship_schemas import Relationship from app.services.classification_service import ( @@ -18,7 +22,8 @@ RelationshipService, get_relationship_service, ) -from app.utils.migrations import create_migrations +from app.utils.migrations import _table_name_for_classification, create_migrations +from app.utils.tenant_connection import get_schema_name router = APIRouter(prefix="/migrations", tags=["Migrations"]) @@ -56,7 +61,6 @@ async def generate_migrations( Then insert the new migrations into the `migrations` table and return them. """ try: - # 1) Load current state from DB classifications: list[ Classification ] = await classification_service.get_classifications(tenant_id) @@ -72,8 +76,6 @@ async def generate_migrations( status_code=404, detail="No classifications found for tenant" ) - # 2) Compute *new* migrations (pure function) - # IMPORTANT: this should return list[MigrationCreate] new_migration_creates: list[MigrationCreate] = create_migrations( classifications=classifications, relationships=relationships, @@ -81,10 +83,8 @@ async def generate_migrations( ) if not new_migration_creates: - # Nothing new to add return [] - # 3) Insert into DB and return the created migrations created: list[Migration] = [] for m in new_migration_creates: new_id = await migration_service.create_migration(m) @@ -122,6 +122,88 @@ async def execute_migrations( raise HTTPException(status_code=500, detail=str(e)) from e +@router.post("/load_data/{tenant_id}") +async def load_data_for_tenant( + tenant_id: UUID, + classification_service: ClassificationService = Depends(get_classification_service), + supabase: AsyncClient = Depends(get_async_supabase), + admin=Depends(get_current_admin), +) -> dict: + """ + Full data sync for a tenant: + + - Fetch all extracted files + their classifications + - Group by classification + - For each classification: + * derive table name (same as migrations) using helper function + * DELETE existing rows for that tenant in tenant-specific schema + * INSERT rows for each file in that classification in tenant-specific schema + """ + try: + extracted_files: list[ + ExtractedFile + ] = await classification_service.get_extracted_files(tenant_id) + + if not extracted_files: + return { + "status": "ok", + "tables_updated": [], + "message": "No extracted files found", + } + + files_by_class_id: dict[UUID, list[ExtractedFile]] = defaultdict(list) + + for ef in extracted_files: + if ef.classification is None: + continue + files_by_class_id[ef.classification.classification_id].append(ef) + + # Get tenant-specific schema name + schema_name = get_schema_name(tenant_id) + updated_tables: list[str] = [] + + for class_files in files_by_class_id.values(): + classification = class_files[0].classification + table_name = _table_name_for_classification(classification) + qualified_table_name = f'"{schema_name}"."{table_name}"' + + # Delete existing rows for this tenant in the tenant-specific schema + # Use parameterized approach via dollar-quoting for safety + tenant_id_str = str(tenant_id) + delete_sql = f"DELETE FROM {qualified_table_name} WHERE tenant_id = '{tenant_id_str}';" + await supabase.rpc("execute_sql", {"query": delete_sql}).execute() + + # Insert new rows into the tenant-specific schema + if class_files: + # Build INSERT statement with proper JSONB escaping using dollar-quoting + values = [] + for idx, f in enumerate(class_files): + # Use dollar-quoting for JSONB to avoid SQL injection + # Convert to JSON string - dollar-quoting doesn't require escaping + data_json = json.dumps(f.extracted_data) + # Use dollar-quoting with unique tag per value to safely handle any characters + tag = f"json{idx}" + values.append( + f"('{str(f.extracted_file_id)}', '{tenant_id_str}', ${tag}${data_json}${tag}$::jsonb)" + ) + + insert_sql = f""" +INSERT INTO {qualified_table_name} (id, tenant_id, data) +VALUES {", ".join(values)}; +""".strip() + await supabase.rpc("execute_sql", {"query": insert_sql}).execute() + + updated_tables.append(table_name) + + return { + "status": "ok", + "tables_updated": updated_tables, + "message": "Data synced from extracted_files into generated tables", + } + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) from e + + @router.get("/connection-url/{tenant_id}") async def get_tenant_connection_url( tenant_id: UUID, @@ -130,15 +212,6 @@ async def get_tenant_connection_url( ) -> dict: """ Get a PostgreSQL connection URL for a specific tenant. - - This URL is scoped to only show the tenant's generated tables. - - Query params: - include_public: If true, also include public schema (for shared tables) - - Example: - GET /migrations/connection-url/{tenant_id} - GET /migrations/connection-url/{tenant_id}?include_public=true """ from app.utils.tenant_connection import get_schema_name, get_tenant_connection_url diff --git a/backend/app/services/pattern_recognition_service.py b/backend/app/services/pattern_recognition_service.py index fd8cd92..0f3fd6e 100644 --- a/backend/app/services/pattern_recognition_service.py +++ b/backend/app/services/pattern_recognition_service.py @@ -80,11 +80,20 @@ async def analyze_and_store_relationships( ) -> list[RelationshipCreate]: """ Main workflow: - 1. Fetch classifications and extracted files - 2. Run pattern recognition analysis - 3. Store relationships in database - 4. Return created relationships + 1. Delete existing relationships for this tenant + 2. Fetch classifications and extracted files + 3. Run pattern recognition analysis + 4. Store relationships in database + 5. Return created relationships """ + # DELETE existing relationships first to avoid duplicates + await ( + self.supabase.table("relationships") + .delete() + .eq("tenant_id", str(tenant_id)) + .execute() + ) + # Fetch data classifications = await self.get_classifications(tenant_id) extracted_files = await self.get_extracted_files(tenant_id) diff --git a/backend/app/utils/migrations.py b/backend/app/utils/migrations.py index bfe8a39..531fff0 100644 --- a/backend/app/utils/migrations.py +++ b/backend/app/utils/migrations.py @@ -1,4 +1,4 @@ -# app/utils/migrations.py +import hashlib from app.schemas.classification_schemas import Classification from app.schemas.migration_schemas import Migration, MigrationCreate @@ -8,9 +8,24 @@ def _table_name_for_classification(c: Classification) -> str: """ Deterministic mapping from classification name to SQL table name. - Example: "Robot Specifications" -> "robotspecifications" + Converts spaces and special characters to underscores, keeps only alphanumeric and underscores. + Example: "Product Brochure/Leaflet" -> "product_brochure_leaflet" + Example: "Robot Specifications" -> "robot_specifications" """ - return c.name.replace(" ", "").lower() + # Convert to lowercase + name = c.name.lower() + # Replace non-alphanumeric characters with underscores + name = "".join(char if char.isalnum() else "_" for char in name) + # Remove consecutive underscores and strip leading/trailing underscores + while "__" in name: + name = name.replace("__", "_") + name = name.strip("_") + + # Ensure it starts with a letter (not a number) + if name and name[0].isdigit(): + name = "tbl_" + name + + return name def _get_schema_name(tenant_id) -> str: @@ -21,6 +36,91 @@ def _get_schema_name(tenant_id) -> str: return f"tenant_{str(tenant_id).replace('-', '_')}" +def _get_created_tables(migrations: list[Migration], schema_name: str) -> set[str]: + """ + Get all table names that have been created by migrations for this schema. + Returns: set of table names (without schema prefix) + """ + created_tables = set() + prefix = f"create_table_{schema_name}_" + for m in migrations: + if m.name.startswith(prefix): + table_name = m.name.replace(prefix, "") + created_tables.add(table_name) + return created_tables + + +def _get_dropped_tables(migrations: list[Migration], schema_name: str) -> set[str]: + """ + Get table names that have already been dropped for this schema. + Returns: set of table names (without schema prefix) + """ + dropped = set() + prefix = f"drop_table_{schema_name}_" + for m in migrations: + if m.name.startswith(prefix): + table_name = m.name.replace(prefix, "") + dropped.add(table_name) + return dropped + + +def _truncate_constraint_name(name: str, max_length: int = 63) -> str: + """ + Truncate constraint name to max_length bytes while preserving uniqueness. + PostgreSQL identifier limit is 63 bytes. + """ + # Convert to bytes to check actual length + name_bytes = name.encode("utf-8") + if len(name_bytes) <= max_length: + return name + + # Truncate to max_length bytes, ensuring we don't cut in the middle of a multi-byte character + truncated_bytes = name_bytes[:max_length] + # Remove any incomplete trailing bytes + while truncated_bytes and (truncated_bytes[-1] & 0xC0) == 0x80: + truncated_bytes = truncated_bytes[:-1] + + return truncated_bytes.decode("utf-8", errors="ignore") + + +def _make_unique_constraint_name( + base_name: str, existing_names: set[str], max_length: int = 63 +) -> str: + """ + Generate a unique constraint name, truncating if necessary. + If truncated name conflicts, appends a hash suffix. + """ + # First try the base name + if base_name not in existing_names: + truncated = _truncate_constraint_name(base_name, max_length) + if truncated not in existing_names: + existing_names.add(truncated) + return truncated + + # If base name is taken, truncate and add hash suffix + truncated = _truncate_constraint_name( + base_name, max_length - 9 + ) # Reserve space for _XXXXXXXX + # Generate a short hash from the original name + hash_suffix = hashlib.md5(base_name.encode("utf-8")).hexdigest()[:8] + unique_name = f"{truncated}_{hash_suffix}" + + # Ensure it's still within limit + unique_name = _truncate_constraint_name(unique_name, max_length) + + # If still conflicts (unlikely), keep trying with different suffixes + counter = 0 + while unique_name in existing_names and counter < 100: + counter += 1 + hash_suffix = hashlib.md5(f"{base_name}_{counter}".encode()).hexdigest()[:8] + unique_name = _truncate_constraint_name( + f"{truncated}_{hash_suffix}", max_length + ) + + existing_names.add(unique_name) + return unique_name + + def create_migrations( classifications: list[Classification], relationships: list[Relationship], @@ -30,16 +130,20 @@ def create_migrations( PURE FUNCTION. Given: - - classifications: what tables we conceptually want + - classifications: what tables we conceptually want NOW - relationships: how those tables relate (1-1, 1-many, many-many) - initial_migrations: migrations that already exist in DB Returns: - list[MigrationCreate] = new migrations to append on top - NOW WITH SCHEMA-PER-TENANT: - - First migration creates the tenant schema - - All tables are created within that schema + This function handles: + 1. CREATE SCHEMA for the tenant + 2. CREATE TABLE for new classifications + 3. DROP TABLE for removed classifications + 4. Relationship migrations + + All SQL is schema-qualified for tenant isolation. """ if not classifications: return [] @@ -52,11 +156,16 @@ def create_migrations( new_migrations: list[MigrationCreate] = [] - # All classifications belong to the same tenant - tenant_id = classifications[0].tenant_id - schema_name = _get_schema_name(tenant_id) + # Get tenant info and schema name + tenant_id = classifications[0].tenant_id if classifications else None + if not tenant_id: + # If no classifications exist, try to get tenant_id from migrations + if initial_migrations: + tenant_id = initial_migrations[0].tenant_id + + schema_name = _get_schema_name(tenant_id) if tenant_id else "public" - # ===== STEP 1: CREATE SCHEMA ===== + # ===== STEP 0: CREATE SCHEMA ===== schema_migration_name = f"create_schema_{schema_name}" if schema_migration_name not in existing_names: @@ -64,30 +173,70 @@ def create_migrations( MigrationCreate( tenant_id=tenant_id, name=schema_migration_name, - sql=f"CREATE SCHEMA IF NOT EXISTS {schema_name};", + sql=f'CREATE SCHEMA IF NOT EXISTS "{schema_name}";', sequence=next_seq, ) ) existing_names.add(schema_migration_name) next_seq += 1 + # ===== STEP 1: Handle DROP migrations for removed classifications ===== + # Get current state of tables from migrations (passing schema_name) + created_tables = _get_created_tables(initial_migrations, schema_name) + dropped_tables = _get_dropped_tables(initial_migrations, schema_name) + active_tables = created_tables - dropped_tables + + # Build current classification table names + current_classification_tables = { + _table_name_for_classification(c) for c in classifications + } + + # Tables that were created but no longer in classifications = should be dropped + tables_to_drop = active_tables - current_classification_tables + + for table_name in sorted(tables_to_drop): + # Remove schema prefix if present (helper functions might include it) + clean_table_name = ( + table_name.split(".")[-1] if "." in table_name else table_name + ) + mig_name = f"drop_table_{schema_name}_{clean_table_name}" + + if mig_name in existing_names: + continue + + # Schema-qualified DROP with CASCADE + sql = f'DROP TABLE IF EXISTS "{schema_name}"."{clean_table_name}" CASCADE;' + + if tenant_id: + new_migrations.append( + MigrationCreate( + tenant_id=tenant_id, + name=mig_name, + sql=sql, + sequence=next_seq, + ) + ) + existing_names.add(mig_name) + next_seq += 1 + # ===== STEP 2: CREATE TABLES (in tenant schema) ===== + for c in classifications: table_name = _table_name_for_classification(c) - qualified_table_name = f"{schema_name}.{table_name}" mig_name = f"create_table_{schema_name}_{table_name}" if mig_name in existing_names: continue + # Schema-qualified CREATE sql = f""" - CREATE TABLE IF NOT EXISTS {qualified_table_name} ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - tenant_id UUID NOT NULL, - data JSONB NOT NULL, - created_at TIMESTAMPTZ DEFAULT NOW() +CREATE TABLE IF NOT EXISTS "{schema_name}"."{table_name}" ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + tenant_id UUID NOT NULL, + data JSONB NOT NULL, + created_at TIMESTAMPTZ DEFAULT NOW() ); - """.strip() +""".strip() new_migrations.append( MigrationCreate( @@ -101,58 +250,133 @@ def create_migrations( next_seq += 1 # ===== STEP 3: CREATE RELATIONSHIPS (in tenant schema) ===== + # Track constraint names within this migration batch to ensure uniqueness + constraint_names_used = set() + for rel in relationships: from_table = _table_name_for_classification(rel.from_classification) to_table = _table_name_for_classification(rel.to_classification) - qualified_from = f"{schema_name}.{from_table}" - qualified_to = f"{schema_name}.{to_table}" + # Skip relationships where either table doesn't exist anymore + if ( + from_table not in current_classification_tables + or to_table not in current_classification_tables + ): + continue # Support both Enum and plain string for rel.type - rel_type = getattr(rel.type, "value", rel.type) + raw_type = getattr(rel.type, "value", rel.type) + rel_type_norm = str(raw_type).upper().replace("-", "_") - mig_name = f"rel_{rel_type.lower()}_{schema_name}_{from_table}_{to_table}" + mig_name = f"rel_{rel_type_norm.lower()}_{schema_name}_{from_table}_{to_table}" if mig_name in existing_names: continue - if rel_type == "ONE_TO_MANY": + if rel_type_norm == "ONE_TO_MANY": + # Schema-qualified ALTER TABLE for one-to-many + # Constraint names don't need schema prefix since they're schema-qualified + base_constraint_name = f"fk_{from_table}_{to_table}" + constraint_name = _make_unique_constraint_name( + base_constraint_name, constraint_names_used + ) sql = f""" - ALTER TABLE {qualified_from} - ADD COLUMN IF NOT EXISTS {to_table}_id UUID, - ADD CONSTRAINT fk_{schema_name}_{from_table}_{to_table} - FOREIGN KEY ({to_table}_id) - REFERENCES {qualified_to}(id); - """.strip() - - elif rel_type == "ONE_TO_ONE": +DO $$ +BEGIN + ALTER TABLE "{schema_name}"."{from_table}" + ADD COLUMN IF NOT EXISTS "{to_table}_id" UUID; + + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + JOIN pg_namespace n ON c.connamespace = n.oid + WHERE c.conname = '{constraint_name}' + AND n.nspname = '{schema_name}' + ) THEN + ALTER TABLE "{schema_name}"."{from_table}" + ADD CONSTRAINT "{constraint_name}" + FOREIGN KEY ("{to_table}_id") + REFERENCES "{schema_name}"."{to_table}"(id); + END IF; +END $$; +""".strip() + + elif rel_type_norm == "ONE_TO_ONE": + # Schema-qualified ALTER TABLE for one-to-one + # Constraint names don't need schema prefix since they're schema-qualified + base_constraint_name = f"fk_{from_table}_{to_table}" + constraint_name = _make_unique_constraint_name( + base_constraint_name, constraint_names_used + ) + base_unique_constraint_name = f"{base_constraint_name}_unique" + unique_constraint_name = _make_unique_constraint_name( + base_unique_constraint_name, constraint_names_used + ) sql = f""" - ALTER TABLE {qualified_from} - ADD COLUMN IF NOT EXISTS {to_table}_id UUID UNIQUE, - ADD CONSTRAINT fk_{schema_name}_{from_table}_{to_table} - FOREIGN KEY ({to_table}_id) - REFERENCES {qualified_to}(id); - """.strip() - - elif rel_type == "MANY_TO_MANY": +DO $$ +BEGIN + ALTER TABLE "{schema_name}"."{from_table}" + ADD COLUMN IF NOT EXISTS "{to_table}_id" UUID; + + -- Add FK constraint if not exists + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + JOIN pg_namespace n ON c.connamespace = n.oid + WHERE c.conname = '{constraint_name}' + AND n.nspname = '{schema_name}' + ) THEN + ALTER TABLE "{schema_name}"."{from_table}" + ADD CONSTRAINT "{constraint_name}" + FOREIGN KEY ("{to_table}_id") + REFERENCES "{schema_name}"."{to_table}"(id); + END IF; + + -- Add UNIQUE constraint if not exists + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint c + JOIN pg_namespace n ON c.connamespace = n.oid + WHERE c.conname = '{unique_constraint_name}' + AND n.nspname = '{schema_name}' + ) THEN + ALTER TABLE "{schema_name}"."{from_table}" + ADD CONSTRAINT "{unique_constraint_name}" UNIQUE ("{to_table}_id"); + END IF; +END $$; +""".strip() + + elif rel_type_norm == "MANY_TO_MANY": + # Schema-qualified CREATE TABLE for join table join_table = f"{from_table}_{to_table}_join" - qualified_join = f"{schema_name}.{join_table}" + + # Constraint names don't need schema prefix since they're schema-qualified + base_fk_from_name = f"fk_{join_table}_{from_table}" + base_fk_to_name = f"fk_{join_table}_{to_table}" + base_unique_name = f"uniq_{join_table}" + + fk_from_constraint = _make_unique_constraint_name( + base_fk_from_name, constraint_names_used + ) + fk_to_constraint = _make_unique_constraint_name( + base_fk_to_name, constraint_names_used + ) + unique_constraint = _make_unique_constraint_name( + base_unique_name, constraint_names_used + ) sql = f""" - CREATE TABLE IF NOT EXISTS {qualified_join} ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - {from_table}_id UUID NOT NULL, - {to_table}_id UUID NOT NULL, - CONSTRAINT fk_{schema_name}_{join_table}_{from_table} - FOREIGN KEY ({from_table}_id) - REFERENCES {qualified_from}(id), - CONSTRAINT fk_{schema_name}_{join_table}_{to_table} - FOREIGN KEY ({to_table}_id) - REFERENCES {qualified_to}(id), - CONSTRAINT uniq_{schema_name}_{join_table} - UNIQUE ({from_table}_id, {to_table}_id) - ); - """.strip() +CREATE TABLE IF NOT EXISTS "{schema_name}"."{join_table}" ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + "{from_table}_id" UUID NOT NULL, + "{to_table}_id" UUID NOT NULL, + CONSTRAINT "{fk_from_constraint}" + FOREIGN KEY ("{from_table}_id") + REFERENCES "{schema_name}"."{from_table}"(id), + CONSTRAINT "{fk_to_constraint}" + FOREIGN KEY ("{to_table}_id") + REFERENCES "{schema_name}"."{to_table}"(id), + CONSTRAINT "{unique_constraint}" + UNIQUE ("{from_table}_id", "{to_table}_id") +); +""".strip() else: sql = f"-- TODO: implement SQL for relationship {mig_name}" diff --git a/backend/app/utils/preprocess/pdf_extractor.py b/backend/app/utils/preprocess/pdf_extractor.py index 50d5ba4..857f300 100644 --- a/backend/app/utils/preprocess/pdf_extractor.py +++ b/backend/app/utils/preprocess/pdf_extractor.py @@ -1,4 +1,5 @@ import json +import os from app.core.litellm import LLMClient, ModelType @@ -16,7 +17,11 @@ async def extract_pdf_data( pdf_bytes: bytes, file_name: str, - llm_model: ModelType = ModelType.GEMINI_PRO, + llm_model: ModelType = ( + ModelType.GEMINI_FLASH + if os.getenv("ENVIRONMENT") == "development" + else ModelType.GEMINI_PRO + ), ) -> dict: model.set_model(llm_model) response = await model.chat( diff --git a/docker-compose.yml b/docker-compose.yml index c85e803..e7f6ae7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ services: WEBHOOK_BASE_URL: ${WEBHOOK_BASE_URL} WEBHOOK_SECRET: ${WEBHOOK_SECRET} GEMINI_API_KEY: ${GEMINI_API_KEY} - DATABASE_URL: ${DATABASE_URL} + DATABASE_URL: ${DB_URL} volumes: - ./backend:/app extra_hosts: diff --git a/frontend/src/components/admin/steps/LoadDataStep.tsx b/frontend/src/components/admin/steps/LoadDataStep.tsx new file mode 100644 index 0000000..1d01e33 --- /dev/null +++ b/frontend/src/components/admin/steps/LoadDataStep.tsx @@ -0,0 +1,78 @@ +import type { FC } from 'react' +import { Button } from '../../ui/Button' +import { ErrorAlert } from '../../ui/ErrorAlert' +import { useLoadData } from '../../../hooks/migrations.hooks' + +interface LoadDataStepProps { + onCompleted?: () => void +} + +export const LoadDataStep: FC = ({ onCompleted }) => { + const { loadData, isLoadingData, loadDataError, loadDataResponse } = + useLoadData() + + const handleLoadData = async () => { + await loadData() + onCompleted?.() + } + + return ( +
+
+
+

Load Data

+

+ Sync extracted file data into generated database tables. This will + delete existing rows and insert new rows for each file in its + classification table. +

+
+
+ +
+
+ + {loadDataError && ( + + )} + + {isLoadingData ? ( +
+ Loading data into tables... +
+ ) : loadDataResponse ? ( +
+
+
+ {loadDataResponse.message} +
+ {loadDataResponse.tables_updated && + loadDataResponse.tables_updated.length > 0 && ( +
+
+ Tables Updated: +
+
+ {loadDataResponse.tables_updated.map((table, index) => ( + + {table} + + ))} +
+
+ )} +
+
+ ) : ( +
+ Click "Load Data" to sync extracted files into generated tables. +
+ )} +
+ ) +} diff --git a/frontend/src/hooks/migrations.hooks.tsx b/frontend/src/hooks/migrations.hooks.tsx index f62b3ca..bbfbbd7 100644 --- a/frontend/src/hooks/migrations.hooks.tsx +++ b/frontend/src/hooks/migrations.hooks.tsx @@ -100,6 +100,42 @@ export const useExecuteMigrations = () => { } } +export interface LoadDataResponse { + status: string + tables_updated: string[] + message: string +} + +export const useLoadData = () => { + const { currentTenant } = useAuth() + const queryClient = useQueryClient() + + const mutation = useMutation({ + mutationFn: async (): Promise => { + if (!currentTenant) { + throw new Error('No tenant selected') + } + + const { data } = await api.post( + `/migrations/load_data/${currentTenant.id}` + ) + return data + }, + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: QUERY_KEYS.migrations.list(currentTenant?.id), + }) + }, + }) + + return { + loadData: mutation.mutateAsync, + isLoadingData: mutation.isPending, + loadDataError: mutation.error, + loadDataResponse: mutation.data, + } +} + export interface ConnectionUrlResponse { tenant_id: string schema_name: string @@ -135,4 +171,4 @@ export const useGetConnectionUrl = () => { connectionUrlIsLoading: isLoading, connectionUrlError: error, } -} \ No newline at end of file +} diff --git a/frontend/src/pages/AdminPage.tsx b/frontend/src/pages/AdminPage.tsx index b7ef4d7..bef348f 100644 --- a/frontend/src/pages/AdminPage.tsx +++ b/frontend/src/pages/AdminPage.tsx @@ -5,6 +5,7 @@ import { ClassificationStep } from '../components/admin/steps/ClassificationStep import { AssignClassificationsStep } from '../components/admin/steps/AssignClassificationsStep' import { PatternRecognitionStep } from '../components/admin/steps/PatternRecognitionStep' import { MigrationsStep } from '../components/admin/steps/MigrationsStep' +import { LoadDataStep } from '../components/admin/steps/LoadDataStep' import { ConnectionUrlStep } from '../components/admin/steps/ConnectionUrlStep' import { useGetClassifications } from '../hooks/classification.hooks' import { useGetAllFiles } from '../hooks/files.hooks' @@ -12,6 +13,7 @@ import { useGetAllExtractedFiles } from '../hooks/extractedFile.hooks' import { useGetRelationships } from '../hooks/patternRecognition.hooks' import { useListMigrations, + useLoadData, useGetConnectionUrl, } from '../hooks/migrations.hooks' @@ -23,6 +25,8 @@ export function AdminPage() { const { extractedFiles } = useGetAllExtractedFiles() const { relationships } = useGetRelationships() const { migrations } = useListMigrations() + const { loadDataResponse } = useLoadData() + const { connectionUrl } = useGetConnectionUrl() const hasClassifications = (classifications?.length ?? 0) > 0 const totalFiles = files?.length ?? 0 @@ -31,16 +35,17 @@ export function AdminPage() { .length ?? 0 const hasExtractedFiles = extractedFiles?.some(ef => ef.status === 'completed') ?? false - const { connectionUrl } = useGetConnectionUrl() const hasRelationships = (relationships?.length ?? 0) > 0 const hasMigrations = (migrations?.length ?? 0) > 0 + const hasDataLoaded = !!loadDataResponse const hasConnectionUrl = !!connectionUrl const step1Complete = hasClassifications const step2Complete = step1Complete && totalFiles > 0 && classifiedFiles > 0 const step3Complete = hasRelationships const step4Complete = hasMigrations - const step5Complete = hasConnectionUrl + const step5Complete = hasDataLoaded + const step6Complete = hasConnectionUrl const steps: AdminStep[] = [ { @@ -76,7 +81,7 @@ export function AdminPage() { : 'disabled', }, { - label: 'Get Connection URL', + label: 'Load Data', status: step5Complete ? 'completed' : activeStep === 4 @@ -85,6 +90,16 @@ export function AdminPage() { ? 'pending' : 'disabled', }, + { + label: 'Get Connection URL', + status: step6Complete + ? 'completed' + : activeStep === 5 + ? 'current' + : step5Complete + ? 'pending' + : 'disabled', + }, ] const canGoNext = @@ -92,7 +107,8 @@ export function AdminPage() { (activeStep === 1 && step2Complete) || (activeStep === 2 && step3Complete) || activeStep === 3 || - activeStep === 4 + (activeStep === 4 && step5Complete) || + activeStep === 5 const handleNext = () => { if (canGoNext && activeStep < steps.length - 1) { @@ -133,7 +149,10 @@ export function AdminPage() { setActiveStep(3)} /> )} {activeStep === 3 && } - {activeStep === 4 && } + {activeStep === 4 && ( + setActiveStep(5)} /> + )} + {activeStep === 5 && }
diff --git a/frontend/src/utils/constants.ts b/frontend/src/utils/constants.ts index b4b200a..0149b80 100644 --- a/frontend/src/utils/constants.ts +++ b/frontend/src/utils/constants.ts @@ -56,7 +56,8 @@ export const QUERY_KEYS = { lists: () => [...QUERY_KEYS.migrations.all(), 'list'] as const, list: (tenantId: string | undefined) => [...QUERY_KEYS.migrations.lists(), tenantId] as const, - connectionUrl: () => [...QUERY_KEYS.migrations.all(), 'connection-url'] as const, + connectionUrl: () => + [...QUERY_KEYS.migrations.all(), 'connection-url'] as const, connectionUrlDetail: (tenantId: string | undefined) => [...QUERY_KEYS.migrations.connectionUrl(), tenantId] as const, }, diff --git a/frontend/vercel.json b/frontend/vercel.json index b73cda2..e2a4bd7 100644 --- a/frontend/vercel.json +++ b/frontend/vercel.json @@ -1,5 +1,5 @@ { "rewrites": [ - { "source": "/(.*)", "destination": "/" } + { "source": "/(.*)", "destination": "/" } ] - } \ No newline at end of file +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index f61a23f..64c3dd0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "cortex-etl", + "name": "cortex-etl-source", "version": "1.0.0", "lockfileVersion": 3, "requires": true,