diff --git a/apps/backend/src/rhesis/backend/app/crud/__init__.py b/apps/backend/src/rhesis/backend/app/crud/__init__.py index d27996730c..54f40fd8c7 100644 --- a/apps/backend/src/rhesis/backend/app/crud/__init__.py +++ b/apps/backend/src/rhesis/backend/app/crud/__init__.py @@ -327,13 +327,11 @@ def get_test_set( ) -> Optional[models.TestSet]: """ Get a test set by its UUID, applying proper visibility filtering and organization scoping. + + Raises ``ItemDeletedException`` for a soft-deleted test set. """ - return ( - QueryBuilder(db, models.TestSet) - .with_organization_filter(organization_id) # Add organization filtering - .with_visibility_filter(user_id) - .with_custom_filter(lambda q: q.filter(models.TestSet.id == test_set_id)) - .first() + return get_item_detail( + db, models.TestSet, test_set_id, organization_id=organization_id, user_id=user_id ) @@ -446,9 +444,14 @@ def get_test_set_by_nano_id_or_slug( ) -> Optional[models.TestSet]: """ Get a test set by its nano_id or slug, applying proper visibility filtering. + + Raises ``ItemDeletedException`` for a soft-deleted test set. """ - return ( + from rhesis.backend.app.utils.crud_utils import _check_and_raise_if_deleted + + item = ( QueryBuilder(db, models.TestSet) + .with_deleted() .with_related(*_TEST_SET_RELATED_FIELDS) .with_organization_filter(organization_id) .with_visibility_filter(user_id) @@ -459,6 +462,7 @@ def get_test_set_by_nano_id_or_slug( ) .first() ) + return _check_and_raise_if_deleted(item, models.TestSet, identifier, False) def resolve_test_set( @@ -467,6 +471,11 @@ def resolve_test_set( """ Resolve a test set from any valid identifier (UUID, nano_id, or slug). Returns None if not found or if there's an error parsing the identifier. + + Raises: + ItemDeletedException: If the identifier resolves to a soft-deleted + test set. Not caught here so callers get the same 410 behavior as + a direct ID lookup. """ try: # First try UUID @@ -605,16 +614,13 @@ def get_test_set_tests( def get_test_configuration( db: Session, test_configuration_id: uuid.UUID, organization_id: str = None, user_id: str = None ) -> Optional[models.TestConfiguration]: - from rhesis.backend.app.utils.crud_utils import _check_and_raise_if_deleted - - item = ( - QueryBuilder(db, models.TestConfiguration) - .with_deleted() - .with_organization_filter(organization_id) - .with_visibility_filter(user_id) - .filter_by_id(test_configuration_id) + return get_item_detail( + db, + models.TestConfiguration, + test_configuration_id, + organization_id=organization_id, + user_id=user_id, ) - return _check_and_raise_if_deleted(item, models.TestConfiguration, test_configuration_id, False) def get_test_configurations( @@ -1091,18 +1097,14 @@ def get_test_result( db: Session, test_result_id: uuid.UUID, organization_id: str = None, user_id: str = None ) -> Optional[models.TestResult]: """Get test_result with relationships (tags, test, test_run) eagerly loaded.""" - from rhesis.backend.app.utils.crud_utils import _check_and_raise_if_deleted - - item = ( - QueryBuilder(db, models.TestResult) - .with_deleted() - .with_related(*_TEST_RESULT_RELATED_FIELDS) - .with_default_derived_field_loads() - .with_organization_filter(organization_id) - .with_visibility_filter(user_id) - .filter_by_id(test_result_id) + return get_item_detail( + db, + models.TestResult, + test_result_id, + organization_id=organization_id, + user_id=user_id, + related_fields=_TEST_RESULT_RELATED_FIELDS, ) - return _check_and_raise_if_deleted(item, models.TestResult, test_result_id, False) def get_test_results( @@ -1241,17 +1243,14 @@ def get_tool( db: Session, tool_id: uuid.UUID, organization_id: str, user_id: str = None ) -> Optional[models.Tool]: """Get a specific tool by ID with relationships loaded""" - from rhesis.backend.app.utils.crud_utils import _check_and_raise_if_deleted - - item = ( - QueryBuilder(db, models.Tool) - .with_deleted() - .with_related(*_TOOL_RELATED_FIELDS) - .with_organization_filter(organization_id) - .with_visibility_filter(user_id) - .filter_by_id(tool_id) + return get_item_detail( + db, + models.Tool, + tool_id, + organization_id=organization_id, + user_id=user_id, + related_fields=_TOOL_RELATED_FIELDS, ) - return _check_and_raise_if_deleted(item, models.Tool, tool_id, False) def get_tools( @@ -1277,22 +1276,6 @@ def get_tools( ) -def get_tool_by_provider( - db: Session, organization_id: str, provider_value: str -) -> Optional[models.Tool]: - """Get organization's tool by provider type_value (e.g., 'notion', 'github').""" - return ( - db.query(models.Tool) - .join(models.TypeLookup, models.Tool.tool_provider_type_id == models.TypeLookup.id) - .filter( - models.Tool.organization_id == uuid.UUID(organization_id), - models.TypeLookup.type_value == provider_value, - models.Tool.deleted_at.is_(None), # Exclude soft-deleted tools - ) - .first() - ) - - def create_tool( db: Session, tool: schemas.ToolCreate, organization_id: str, user_id: str = None ) -> models.Tool: @@ -1337,17 +1320,14 @@ def get_architect_session_detail( user_id: str = None, ) -> Optional[models.ArchitectSession]: """Get an architect session with its messages eagerly loaded.""" - from rhesis.backend.app.utils.crud_utils import _check_and_raise_if_deleted - - item = ( - QueryBuilder(db, models.ArchitectSession) - .with_deleted() - .with_related(include(models.ArchitectSession.messages)) - .with_organization_filter(organization_id) - .with_visibility_filter(user_id) - .filter_by_id(session_id) + return get_item_detail( + db, + models.ArchitectSession, + session_id, + organization_id=organization_id, + user_id=user_id, + related_fields=(include(models.ArchitectSession.messages),), ) - return _check_and_raise_if_deleted(item, models.ArchitectSession, session_id, False) def get_architect_sessions( diff --git a/apps/backend/src/rhesis/backend/app/crud/metric.py b/apps/backend/src/rhesis/backend/app/crud/metric.py index ed6a021ed2..40ff15bc07 100644 --- a/apps/backend/src/rhesis/backend/app/crud/metric.py +++ b/apps/backend/src/rhesis/backend/app/crud/metric.py @@ -16,6 +16,8 @@ from rhesis.backend.app.utils.crud_utils import ( create_item, delete_item, + get_item, + get_item_detail, get_items_detail, update_item, ) @@ -50,19 +52,16 @@ def get_metric( ) -> Optional[models.Metric]: """Get a specific metric by ID with its related objects, including many-to-many relationships. - Deliberately not routed through ``get_item_detail``: a soft-deleted metric must come - back as ``None`` here (tested behavior), not raise ``ItemDeletedException`` -- unlike - every other entity's single-item fetch, callers of this one don't need to tell "not - found" apart from "deleted". + Raises ``ItemDeletedException`` for a soft-deleted metric, same as every other + entity's single-item fetch. """ - return ( - QueryBuilder(db, models.Metric) - .with_related(*_METRIC_RELATED_FIELDS) - .with_default_derived_field_loads() - .with_organization_filter(organization_id) - .with_visibility_filter(user_id) - .with_custom_filter(lambda q: q.filter(models.Metric.id == metric_id)) - .first() + return get_item_detail( + db, + models.Metric, + metric_id, + organization_id=organization_id, + user_id=user_id, + related_fields=_METRIC_RELATED_FIELDS, ) @@ -269,23 +268,12 @@ def add_requirement_to_metric( bool: True if the requirement was added, False if it was already associated """ # Verify the metric exists AND belongs to the organization (SECURITY CRITICAL) - metric = ( - db.query(models.Metric) - .filter(models.Metric.id == metric_id, models.Metric.organization_id == organization_id) - .first() - ) + metric = get_item(db, models.Metric, metric_id, organization_id) if not metric: raise ValueError(f"Metric with id {metric_id} not found or not accessible") # Verify the requirement exists AND belongs to the organization (SECURITY CRITICAL) - requirement = ( - db.query(models.Requirement) - .filter( - models.Requirement.id == requirement_id, - models.Requirement.organization_id == organization_id, - ) - .first() - ) + requirement = get_item(db, models.Requirement, requirement_id, organization_id) if not requirement: raise ValueError(f"Requirement with id {requirement_id} not found or not accessible") @@ -332,23 +320,12 @@ def remove_requirement_from_metric( bool: True if the requirement was removed, False if it wasn't associated """ # Verify the metric exists AND belongs to the organization (SECURITY CRITICAL) - metric = ( - db.query(models.Metric) - .filter(models.Metric.id == metric_id, models.Metric.organization_id == organization_id) - .first() - ) + metric = get_item(db, models.Metric, metric_id, organization_id) if not metric: raise ValueError(f"Metric with id {metric_id} not found or not accessible") # Verify the requirement exists AND belongs to the organization (SECURITY CRITICAL) - requirement = ( - db.query(models.Requirement) - .filter( - models.Requirement.id == requirement_id, - models.Requirement.organization_id == organization_id, - ) - .first() - ) + requirement = get_item(db, models.Requirement, requirement_id, organization_id) if not requirement: raise ValueError(f"Requirement with id {requirement_id} not found or not accessible") @@ -392,13 +369,7 @@ def get_metric_requirements( List of requirements associated with the metric """ # Verify the metric exists AND belongs to the organization (SECURITY CRITICAL) - metric = ( - db.query(models.Metric) - .filter( - models.Metric.id == metric_id, models.Metric.organization_id == UUID(organization_id) - ) - .first() - ) + metric = get_item(db, models.Metric, metric_id, organization_id) if not metric: raise ValueError(f"Metric with id {metric_id} not found or not accessible") @@ -444,14 +415,7 @@ def get_requirement_metrics( List of metrics associated with the requirement """ # Verify the requirement exists AND belongs to the organization (SECURITY CRITICAL) - requirement = ( - db.query(models.Requirement) - .filter( - models.Requirement.id == requirement_id, - models.Requirement.organization_id == UUID(organization_id), - ) - .first() - ) + requirement = get_item(db, models.Requirement, requirement_id, organization_id) if not requirement: raise ValueError(f"Requirement with id {requirement_id} not found or not accessible") @@ -518,11 +482,7 @@ def add_metric_to_test_set( raise ValueError(f"Test set with id {test_set_id} not found or not accessible") # Verify the metric exists AND belongs to the organization (SECURITY CRITICAL) - metric = ( - db.query(models.Metric) - .filter(models.Metric.id == metric_id, models.Metric.organization_id == organization_id) - .first() - ) + metric = get_item(db, models.Metric, metric_id, organization_id) if not metric: raise ValueError(f"Metric with id {metric_id} not found or not accessible") @@ -578,11 +538,7 @@ def remove_metric_from_test_set( raise ValueError(f"Test set with id {test_set_id} not found or not accessible") # Verify the metric exists AND belongs to the organization (SECURITY CRITICAL) - metric = ( - db.query(models.Metric) - .filter(models.Metric.id == metric_id, models.Metric.organization_id == organization_id) - .first() - ) + metric = get_item(db, models.Metric, metric_id, organization_id) if not metric: raise ValueError(f"Metric with id {metric_id} not found or not accessible") diff --git a/apps/backend/src/rhesis/backend/app/crud/model.py b/apps/backend/src/rhesis/backend/app/crud/model.py index e137587ce5..8f5a84fe8f 100644 --- a/apps/backend/src/rhesis/backend/app/crud/model.py +++ b/apps/backend/src/rhesis/backend/app/crud/model.py @@ -11,9 +11,8 @@ stay editable so users can still organize it. Both raise ``ValueError`` with "protected" in the message; the router turns that into a 403. -``get_model`` re-applies the organization filter by hand on top of the ambient scope filter, -because a leaked model row hands out a stored provider API key. The other functions go -through ``crud_utils``, which applies it for them. +``get_model`` intentionally doesn't follow this repo's usual soft-delete contract -- see +its own docstring. The block that moved here also held ``test_model_connection``, a stub that never did anything but return ``True`` -- the real check was never written. It had no callers (the @@ -33,7 +32,7 @@ get_items_detail, update_item, ) -from rhesis.backend.app.utils.query_utils import include +from rhesis.backend.app.utils.query_utils import QueryBuilder, include # Relationships serialized by schemas.ModelDetail -- provider_type, status. # owner/assignee: unused, excluded. Public (no leading underscore) since @@ -47,20 +46,17 @@ def get_model( db: Session, model_id: uuid.UUID, organization_id: str = None, user_id: str = None ) -> Optional[models.Model]: - """Get a specific model by ID with its related objects and organization filtering""" - query = ( - db.query(models.Model) - .options(include(models.Model.provider_type)) - .filter(models.Model.id == model_id) - ) - - # Apply organization filtering (SECURITY CRITICAL) - if organization_id: - from uuid import UUID as UUIDType - - query = query.filter(models.Model.organization_id == UUIDType(organization_id)) + """Get a specific model by ID with its related objects and organization filtering. - return query.first() + Doesn't call ``get_item_detail`` directly: callers here need a soft-deleted model + to return ``None``, not raise ``ItemDeletedException``, so they can fall back. + """ + return ( + QueryBuilder(db, models.Model) + .with_related(include(models.Model.provider_type)) + .with_organization_filter(organization_id) + .filter_by_id(model_id) + ) def get_models( diff --git a/apps/backend/src/rhesis/backend/app/crud/source.py b/apps/backend/src/rhesis/backend/app/crud/source.py index 957118f496..d64dcb620b 100644 --- a/apps/backend/src/rhesis/backend/app/crud/source.py +++ b/apps/backend/src/rhesis/backend/app/crud/source.py @@ -27,6 +27,7 @@ from rhesis.backend.app.utils.crud_utils import ( create_item, delete_item, + get_item_detail, update_item, ) from rhesis.backend.app.utils.query_utils import QueryBuilder, include @@ -49,18 +50,14 @@ def get_source( Use get_source_with_content() to load the content field. Relationships (source_type, user) are loaded for display. """ - from rhesis.backend.app.utils.crud_utils import _check_and_raise_if_deleted - - item = ( - QueryBuilder(db, models.Source) - .with_deleted() - .with_related(*_SOURCE_RELATED_FIELDS) - .with_default_derived_field_loads() - .with_organization_filter(organization_id) - .with_visibility_filter(user_id) - .filter_by_id(source_id) + return get_item_detail( + db, + models.Source, + source_id, + organization_id=organization_id, + user_id=user_id, + related_fields=_SOURCE_RELATED_FIELDS, ) - return _check_and_raise_if_deleted(item, models.Source, source_id, False) def get_source_with_content( @@ -73,19 +70,16 @@ def get_source_with_content( """Get source with content field explicitly loaded (a deferred column).""" from sqlalchemy.orm import undefer - from rhesis.backend.app.utils.crud_utils import _check_and_raise_if_deleted - - item = ( - QueryBuilder(db, models.Source) - .with_deleted() - .with_related(*_SOURCE_RELATED_FIELDS) - .with_default_derived_field_loads() - .with_organization_filter(organization_id) - .with_visibility_filter(user_id) - .with_custom_filter(lambda q: q.options(undefer(models.Source.content))) - .filter_by_id(source_id) + return get_item_detail( + db, + models.Source, + source_id, + organization_id=organization_id, + user_id=user_id, + include_deleted=include_deleted, + related_fields=_SOURCE_RELATED_FIELDS, + extra_filter=lambda q: q.options(undefer(models.Source.content)), ) - return _check_and_raise_if_deleted(item, models.Source, source_id, include_deleted) def get_sources( diff --git a/apps/backend/src/rhesis/backend/app/crud/telemetry.py b/apps/backend/src/rhesis/backend/app/crud/telemetry.py index 2a69e3b9ea..515f0b7e13 100644 --- a/apps/backend/src/rhesis/backend/app/crud/telemetry.py +++ b/apps/backend/src/rhesis/backend/app/crud/telemetry.py @@ -130,20 +130,13 @@ def get_trace_by_db_id( trace_db_id: str, organization_id: str, ) -> Optional[models.Trace]: - """Get a single trace span row by its database UUID.""" - from uuid import UUID + """Get a single trace span row by its database UUID. - return ( - db.query(models.Trace) - .filter( - and_( - models.Trace.id == UUID(trace_db_id), - models.Trace.organization_id == UUID(organization_id), - models.Trace.deleted_at.is_(None), - ) - ) - .first() - ) + Raises ``ItemDeletedException`` for a soft-deleted trace. + """ + from rhesis.backend.app.utils.crud_utils import get_item_detail + + return get_item_detail(db, models.Trace, UUID(trace_db_id), organization_id=organization_id) def get_trace_by_id( @@ -237,34 +230,6 @@ def get_trace_id_for_conversation( return result[0] if result else None -def get_span_by_id( - db: Session, - span_id: str, - project_id: str, -) -> Optional[models.Trace]: - """ - Get a single span by span ID. - - Args: - db: Database session - span_id: OpenTelemetry span ID - project_id: Project ID for access control - - Returns: - Trace model or None if not found - """ - return ( - db.query(models.Trace) - .filter( - and_( - models.Trace.span_id == span_id, - models.Trace.project_id == project_id, - ) - ) - .first() - ) - - def _escape_like_pattern(term: str) -> str: """Escape SQL LIKE wildcards in user-provided search text.""" return term.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") diff --git a/apps/backend/src/rhesis/backend/app/routers/metric.py b/apps/backend/src/rhesis/backend/app/routers/metric.py index 3d78549d6b..5b4c3594a7 100644 --- a/apps/backend/src/rhesis/backend/app/routers/metric.py +++ b/apps/backend/src/rhesis/backend/app/routers/metric.py @@ -272,19 +272,6 @@ def read_metric( organization_id, user_id = tenant_context db_metric = metric_crud.get_metric(db, metric_id, organization_id, user_id) if db_metric is None: - # get_metric's query silently excludes soft-deleted rows (like most - # plain getters); check separately here so a deleted metric still gets - # its own 410, matching the other standard entity routes' contract. - from rhesis.backend.app.utils.crud_utils import _check_and_raise_if_deleted - from rhesis.backend.app.utils.query_utils import QueryBuilder - - deleted_check = ( - QueryBuilder(db, models.Metric) - .with_deleted() - .with_organization_filter(organization_id) - .filter_by_id(metric_id) - ) - _check_and_raise_if_deleted(deleted_check, models.Metric, metric_id, False) raise HTTPException(status_code=404, detail="Metric not found") return db_metric diff --git a/apps/backend/src/rhesis/backend/app/routers/test_set.py b/apps/backend/src/rhesis/backend/app/routers/test_set.py index 8074b84731..74f01d0b30 100644 --- a/apps/backend/src/rhesis/backend/app/routers/test_set.py +++ b/apps/backend/src/rhesis/backend/app/routers/test_set.py @@ -445,42 +445,34 @@ def update_test_set( response_class=StreamingResponse, **capability(Permission.TestSet.EXPORT), ) +@handle_database_exceptions(entity_name="test set") def download_test_set_prompts( test_set_identifier: str, db: Session = Depends(get_tenant_db_session), tenant_context=Depends(get_tenant_context), # SECURITY: Extract tenant context current_user: User = Depends(require_current_user_or_token), ): - try: - # Resolve test set - organization_id, user_id = tenant_context # SECURITY: Get tenant context - db_test_set = resolve_test_set_or_raise(test_set_identifier, db, organization_id) - - # Get prompts with organization filtering (SECURITY CRITICAL) - prompts = get_prompts_for_test_set(db, db_test_set.id, organization_id) + # Resolve test set + organization_id, user_id = tenant_context # SECURITY: Get tenant context + db_test_set = resolve_test_set_or_raise(test_set_identifier, db, organization_id) - # Check if prompts list is empty before trying to create CSV - if not prompts: - raise HTTPException( - status_code=404, detail=f"No prompts found in test set: {test_set_identifier}" - ) - - csv_data = prompts_to_csv(prompts) - - response = StreamingResponse(iter([csv_data]), media_type="text/csv") - response.headers["Content-Disposition"] = ( - f"attachment; filename=test_set_{test_set_identifier}.csv" - ) - return response + # Get prompts with organization filtering (SECURITY CRITICAL) + prompts = get_prompts_for_test_set(db, db_test_set.id, organization_id) - except HTTPException: - raise - except Exception as e: + # Check if prompts list is empty before trying to create CSV + if not prompts: raise HTTPException( - status_code=500, - detail=f"Failed to download test set prompts for {test_set_identifier}: {str(e)}", + status_code=404, detail=f"No prompts found in test set: {test_set_identifier}" ) + csv_data = prompts_to_csv(prompts) + + response = StreamingResponse(iter([csv_data]), media_type="text/csv") + response.headers["Content-Disposition"] = ( + f"attachment; filename=test_set_{test_set_identifier}.csv" + ) + return response + @router.get("/{test_set_identifier}/tests", response_model=list[schemas.TestDetail]) def get_test_set_tests( diff --git a/apps/backend/src/rhesis/backend/app/services/endpoint/service.py b/apps/backend/src/rhesis/backend/app/services/endpoint/service.py index 364cd6678b..168a434cd9 100644 --- a/apps/backend/src/rhesis/backend/app/services/endpoint/service.py +++ b/apps/backend/src/rhesis/backend/app/services/endpoint/service.py @@ -393,21 +393,13 @@ def _get_endpoint( project_id: str = None, ) -> Endpoint: """Fetch an endpoint by ID, applying organization and project security filtering.""" - from uuid import UUID - - from sqlalchemy import or_ - - query = db.query(Endpoint).filter(Endpoint.id == endpoint_id) - if organization_id: - query = query.filter(Endpoint.organization_id == UUID(organization_id)) - if project_id: - query = query.filter( - or_( - Endpoint.project_id == UUID(project_id), - Endpoint.project_id.is_(None), - ) - ) - endpoint = query.first() + from rhesis.backend.app.crud import get_endpoint + from rhesis.backend.app.utils.database_exceptions import ItemDeletedException + + try: + endpoint = get_endpoint(db, endpoint_id, organization_id, None, project_id=project_id) + except ItemDeletedException: + raise HTTPException(status_code=410, detail="Endpoint has been deleted") if not endpoint: raise HTTPException(status_code=404, detail="Endpoint not found or not accessible") return endpoint diff --git a/apps/backend/src/rhesis/backend/app/services/explorer/tests.py b/apps/backend/src/rhesis/backend/app/services/explorer/tests.py index b5b64ab655..6b497df780 100644 --- a/apps/backend/src/rhesis/backend/app/services/explorer/tests.py +++ b/apps/backend/src/rhesis/backend/app/services/explorer/tests.py @@ -422,6 +422,8 @@ def import_explorer_test_set_from_source( ------ ValueError If the source set is missing, or already configured for Explorer. + ItemDeletedException + If source_test_set_identifier resolves to a soft-deleted test set. """ db_source = crud.resolve_test_set(source_test_set_identifier, db, organization_id) if db_source is None: @@ -514,6 +516,8 @@ def export_regular_test_set_from_explorer( ------ ValueError If the source set is missing, or is not configured for Explorer. + ItemDeletedException + If source_test_set_identifier resolves to a soft-deleted test set. """ db_source = crud.resolve_test_set(source_test_set_identifier, db, organization_id) if db_source is None: diff --git a/apps/backend/src/rhesis/backend/app/services/garak/sync.py b/apps/backend/src/rhesis/backend/app/services/garak/sync.py index f6080611a3..792457009f 100644 --- a/apps/backend/src/rhesis/backend/app/services/garak/sync.py +++ b/apps/backend/src/rhesis/backend/app/services/garak/sync.py @@ -24,6 +24,8 @@ from rhesis.backend.app.models.test_set import TestSet from rhesis.backend.app.schemas import test_set as test_set_schemas from rhesis.backend.app.services.test import bulk_create_tests +from rhesis.backend.app.utils.crud_utils import get_item_detail +from rhesis.backend.app.utils.database_exceptions import ItemDeletedException from .probes import GarakProbeInfo, GarakProbeService from .taxonomy import GarakTaxonomy, resolve_requirement @@ -60,6 +62,28 @@ def preload_probes(self, probes_by_module: Dict[str, List[GarakProbeInfo]]) -> N """ self._probes_by_module = probes_by_module + def _get_test_set_or_raise( + self, test_set_id: str, organization_id: str, user_id: str = None + ) -> TestSet: + """Fetch the TestSet for a sync operation. + + Raises: + ValueError: If the test set has been soft-deleted or doesn't exist. + """ + try: + test_set = get_item_detail( + self.db, + TestSet, + UUID(test_set_id), + organization_id=organization_id, + user_id=user_id, + ) + except ItemDeletedException: + raise ValueError(f"Test set {test_set_id} has been deleted") + if not test_set: + raise ValueError(f"Test set not found: {test_set_id}") + return test_set + def sync_test_set( self, test_set_id: str, @@ -84,21 +108,8 @@ def sync_test_set( Raises: ValueError: If test set not found or not a Garak-imported test set """ - test_set_uuid = UUID(test_set_id) - org_uuid = UUID(organization_id) - # Get the test set - test_set = ( - self.db.query(TestSet) - .filter( - TestSet.id == test_set_uuid, - TestSet.organization_id == org_uuid, - ) - .first() - ) - - if not test_set: - raise ValueError(f"Test set not found: {test_set_id}") + test_set = self._get_test_set_or_raise(test_set_id, organization_id, user_id) # Verify it's a Garak-imported test set if not test_set.attributes or test_set.attributes.get("source") != "garak": @@ -215,17 +226,7 @@ def resolve_sync_target(self, test_set_id: str, organization_id: str) -> Dict[st or has no Garak probe information (same conditions/messages as ``sync_test_set``). """ - test_set = ( - self.db.query(TestSet) - .filter( - TestSet.id == UUID(test_set_id), - TestSet.organization_id == UUID(organization_id), - ) - .first() - ) - - if not test_set: - raise ValueError(f"Test set not found: {test_set_id}") + test_set = self._get_test_set_or_raise(test_set_id, organization_id) if not test_set.attributes or test_set.attributes.get("source") != "garak": raise ValueError(f"Test set {test_set_id} is not a Garak-imported test set") @@ -454,16 +455,9 @@ def can_sync(self, test_set_id: str, organization_id: str) -> bool: Returns: True if the test set is a Garak-imported test set """ - test_set = ( - self.db.query(TestSet) - .filter( - TestSet.id == UUID(test_set_id), - TestSet.organization_id == UUID(organization_id), - ) - .first() - ) - - if not test_set: + try: + test_set = self._get_test_set_or_raise(test_set_id, organization_id) + except ValueError: return False return test_set.attributes is not None and test_set.attributes.get("source") == "garak" @@ -483,19 +477,12 @@ def get_sync_preview( Returns: Dictionary with preview information or None if not syncable """ - test_set_uuid = UUID(test_set_id) - org_uuid = UUID(organization_id) - - test_set = ( - self.db.query(TestSet) - .filter( - TestSet.id == test_set_uuid, - TestSet.organization_id == org_uuid, - ) - .first() - ) + try: + test_set = self._get_test_set_or_raise(test_set_id, organization_id) + except ValueError: + return None - if not test_set or not test_set.attributes: + if not test_set.attributes: return None if test_set.attributes.get("source") != "garak": diff --git a/apps/backend/src/rhesis/backend/app/services/preflight/checks.py b/apps/backend/src/rhesis/backend/app/services/preflight/checks.py index e91634402a..16e83fa171 100644 --- a/apps/backend/src/rhesis/backend/app/services/preflight/checks.py +++ b/apps/backend/src/rhesis/backend/app/services/preflight/checks.py @@ -16,6 +16,7 @@ from rhesis.backend.app.models.user import User from rhesis.backend.app.schemas.metric import MetricScope from rhesis.backend.app.schemas.preflight import PreflightCheckResult, PreflightCheckStatus +from rhesis.backend.app.utils.crud_utils import get_item_detail from .constants import ( CHECK_REQUIREMENT_METRIC_COVERAGE, @@ -488,7 +489,9 @@ async def check_metric_compatibility( metric_ids = [m.id for m in selected_metrics] metrics = db.query(Metric).filter(Metric.id.in_(metric_ids)).all() elif metric_mode == "use_test_set": - test_set = db.query(TestSet).filter(TestSet.id == test_set_id).first() + test_set = get_item_detail( + db, TestSet, test_set_id, organization_id=str(endpoint.organization_id) + ) if test_set: metrics = list(test_set.metrics) else: @@ -635,7 +638,9 @@ async def check_metric_functionality( metric_ids = [m.id for m in selected_metrics] metrics = db.query(Metric).filter(Metric.id.in_(metric_ids)).all() elif metric_mode == "use_test_set": - test_set = db.query(TestSet).filter(TestSet.id == test_set_id).first() + test_set = get_item_detail( + db, TestSet, test_set_id, organization_id=str(user.organization_id) + ) if test_set: metrics = list(test_set.metrics) else: @@ -711,6 +716,7 @@ async def check_requirement_metric_coverage( db: Session, test_set_id: UUID, metric_mode: str, + organization_id: str, selected_metrics: Optional[list] = None, correlation_id: Optional[str] = None, publish: bool = True, @@ -751,7 +757,7 @@ async def check_requirement_metric_coverage( ", ".join(names) if names else None, ) elif metric_mode == "use_test_set": - test_set = db.query(TestSet).filter(TestSet.id == test_set_id).first() + test_set = get_item_detail(db, TestSet, test_set_id, organization_id=organization_id) if not test_set: result = _make_result( check_id, diff --git a/apps/backend/src/rhesis/backend/app/services/preflight/orchestrator.py b/apps/backend/src/rhesis/backend/app/services/preflight/orchestrator.py index d06b83af95..31838b4f6c 100644 --- a/apps/backend/src/rhesis/backend/app/services/preflight/orchestrator.py +++ b/apps/backend/src/rhesis/backend/app/services/preflight/orchestrator.py @@ -12,6 +12,8 @@ from rhesis.backend.app.models.user import User from rhesis.backend.app.schemas.preflight import PreflightCheckResult, PreflightCheckStatus from rhesis.backend.app.schemas.websocket import ChannelTarget, EventType, WebSocketMessage +from rhesis.backend.app.utils.crud_utils import get_item_detail +from rhesis.backend.app.utils.database_exceptions import ItemDeletedException from .checks import ( check_requirement_metric_coverage, @@ -93,7 +95,14 @@ async def run_preflight_checks_multi( {str(ts_id): ts_name for ts_id, ts_name, _ in test_sets} if multi else {} ) - endpoint = db.query(Endpoint).filter(Endpoint.id == endpoint_id).first() + try: + endpoint = get_item_detail( + db, Endpoint, endpoint_id, organization_id=str(user.organization_id) + ) + endpoint_status = "not found" + except ItemDeletedException: + endpoint = None + endpoint_status = "has been deleted" any_multi_turn = any(mt for _, _, mt in test_sets) # --- Shared checks --- @@ -111,7 +120,7 @@ async def run_preflight_checks_multi( r = _make_result( CHECK_ENDPOINT_CONNECTIVITY, PreflightCheckStatus.FAILED, - "Endpoint not found", + f"Endpoint {endpoint_status}", ) _apply_test_set_fields(r) results.append(r) @@ -121,7 +130,7 @@ async def run_preflight_checks_multi( r = _make_result( CHECK_ENDPOINT_CONNECTIVITY, PreflightCheckStatus.FAILED, - "Endpoint not found", + f"Endpoint {endpoint_status}", "The endpoint is required even when reusing outputs.", ) else: @@ -176,6 +185,7 @@ async def run_preflight_checks_multi( db, ts_id, metric_mode, + str(user.organization_id), selected_metrics, correlation_id, publish, diff --git a/apps/backend/src/rhesis/backend/app/services/prompt.py b/apps/backend/src/rhesis/backend/app/services/prompt.py index 0c4bf493a2..6d9cf70875 100644 --- a/apps/backend/src/rhesis/backend/app/services/prompt.py +++ b/apps/backend/src/rhesis/backend/app/services/prompt.py @@ -7,19 +7,16 @@ from rhesis.backend.app.models import Prompt, Test, TestSet from rhesis.backend.app.models.test import test_test_set_association +from rhesis.backend.app.utils.crud_utils import get_item_detail def get_prompts_for_test_set( db: Session, test_set_id: uuid.UUID, organization_id: str = None ) -> List[dict]: - # First check if test set exists AND belongs to organization (SECURITY CRITICAL) - query = db.query(TestSet).filter(TestSet.id == test_set_id) - if organization_id: - from uuid import UUID - - query = query.filter(TestSet.organization_id == UUID(organization_id)) - - test_set_exists = query.first() + # First check if test set exists AND belongs to organization (SECURITY CRITICAL). + # Raises ItemDeletedException for a soft-deleted test set; the sole caller + # (download_test_set_prompts) already lets that through to its 410 response. + test_set_exists = get_item_detail(db, TestSet, test_set_id, organization_id=organization_id) if not test_set_exists: raise ValueError("Test Set not found or not accessible") diff --git a/apps/backend/src/rhesis/backend/app/services/test.py b/apps/backend/src/rhesis/backend/app/services/test.py index dc879d51b0..8468c6a44f 100644 --- a/apps/backend/src/rhesis/backend/app/services/test.py +++ b/apps/backend/src/rhesis/backend/app/services/test.py @@ -16,10 +16,12 @@ from rhesis.backend.app.models.user import User from rhesis.backend.app.utils.crud_utils import ( create_item, + get_item_detail, get_or_create_entity, get_or_create_status, get_or_create_type_lookup, ) +from rhesis.backend.app.utils.database_exceptions import ItemDeletedException from rhesis.backend.app.utils.user_model_utils import ( ensure_language_model, get_user_generation_model, @@ -183,18 +185,19 @@ def load_defaults(): def _validate_test_set( - db: Session, test_set_id: str, organization_id: str = None + db: Session, test_set_id: str, organization_id: str = None, user_id: str = None ) -> tuple[models.TestSet | None, Dict[str, Any] | None]: """Validate test set exists and return it or error response.""" - query = db.query(models.TestSet).filter(models.TestSet.id == test_set_id) - - # Apply organization filter if provided (SECURITY CRITICAL) - if organization_id: - from uuid import UUID - - query = query.filter(models.TestSet.organization_id == UUID(organization_id)) - - test_set = query.first() + try: + test_set = get_item_detail( + db, models.TestSet, test_set_id, organization_id=organization_id, user_id=user_id + ) + except ItemDeletedException: + return None, { + "success": False, + "total_tests": 0, + "message": f"Test set with ID {test_set_id} has been deleted", + } if not test_set: return None, { "success": False, @@ -309,7 +312,7 @@ def bulk_create_test_set_associations( Handles validation of test IDs and existing associations. """ # First validate the test set exists AND belongs to organization (SECURITY CRITICAL) - test_set, error_response = _validate_test_set(db, test_set_id, organization_id) + test_set, error_response = _validate_test_set(db, test_set_id, organization_id, user_id) if error_response: return error_response @@ -921,24 +924,14 @@ def create_test_set_associations( - message: Detailed message about the operation result - metadata: Dictionary containing detailed information about the operation """ - from rhesis.backend.app.models import TestSet - # Transaction management is handled by the session context manager try: # Verify test set exists AND belongs to organization (SECURITY CRITICAL) - from uuid import UUID - - test_set = ( - db.query(TestSet) - .filter(TestSet.id == test_set_id, TestSet.organization_id == UUID(organization_id)) - .first() - ) - if not test_set: + _, error_response = _validate_test_set(db, test_set_id, organization_id, user_id) + if error_response: return { - "success": False, - "total_tests": 0, - "message": f"Test set with ID {test_set_id} not found or not accessible", + **error_response, "metadata": { "new_associations": 0, "existing_associations": 0, @@ -1020,26 +1013,13 @@ def remove_test_set_associations( - removed_associations: Number of associations removed - message: Detailed message about the operation result """ - from rhesis.backend.app.models import TestSet - # Transaction management is handled by the session context manager try: # Verify test set exists AND belongs to organization (SECURITY CRITICAL) - from uuid import UUID - - test_set = ( - db.query(TestSet) - .filter(TestSet.id == test_set_id, TestSet.organization_id == UUID(organization_id)) - .first() - ) - if not test_set: - return { - "success": False, - "total_tests": 0, - "removed_associations": 0, - "message": f"Test set with ID {test_set_id} not found or not accessible", - } + _, error_response = _validate_test_set(db, test_set_id, organization_id, user_id) + if error_response: + return {**error_response, "removed_associations": 0} # Check if any of the provided test IDs are actually associated with the test set existing_associations = db.execute( diff --git a/apps/backend/src/rhesis/backend/app/services/test_set.py b/apps/backend/src/rhesis/backend/app/services/test_set.py index b0bd907591..9ba7943eb6 100644 --- a/apps/backend/src/rhesis/backend/app/services/test_set.py +++ b/apps/backend/src/rhesis/backend/app/services/test_set.py @@ -35,9 +35,15 @@ def get_test_set(db: Session, test_set_id: uuid.UUID, organization_id: str = None): - """Get test set by ID with organization filtering for security""" - return ( + """Get test set by ID with organization filtering for security. + + Raises ItemDeletedException for a soft-deleted test set. + """ + from rhesis.backend.app.utils.crud_utils import _check_and_raise_if_deleted + + item = ( QueryBuilder(db, TestSet) + .with_deleted() .with_custom_filter(lambda q: q.filter(TestSet.id == test_set_id)) .with_related( # TestSet.prompts is one-to-many -- include() picks selectinload for @@ -56,6 +62,7 @@ def get_test_set(db: Session, test_set_id: uuid.UUID, organization_id: str = Non .with_organization_filter(organization_id) .first() ) + return _check_and_raise_if_deleted(item, TestSet, test_set_id, False) def create_pending_test_set( @@ -552,6 +559,7 @@ def update_test_set_attributes( from uuid import UUID from rhesis.backend.app.crud import get_test_set + from rhesis.backend.app.utils.database_exceptions import ItemDeletedException # Validate UUID try: @@ -559,10 +567,13 @@ def update_test_set_attributes( except ValueError: raise ValueError(ERROR_INVALID_UUID.format(entity="test set", id=test_set_id)) - test_set = get_test_set(db, test_set_uuid, organization_id, user_id) + try: + test_set = get_test_set(db, test_set_uuid, organization_id, user_id) + except ItemDeletedException: + test_set = None if not test_set: - # Test set may have been soft-deleted; nothing to update. + # Test set may not exist or may have been soft-deleted; nothing to update. return # Explorer test sets manage their own attributes; skip regeneration. @@ -601,6 +612,10 @@ def get_last_completed_test_run( Returns: Dict with id, status, created_at, test_count, pass_rate; or None if no completed run exists. + + Raises: + ItemDeletedException: If test_set_identifier resolves to a + soft-deleted test set (via crud.resolve_test_set). """ from rhesis.backend.app import crud from rhesis.backend.app.models.status import Status diff --git a/apps/backend/src/rhesis/backend/app/utils/database_exceptions.py b/apps/backend/src/rhesis/backend/app/utils/database_exceptions.py index 34e7e0590c..0b5d4b9b06 100644 --- a/apps/backend/src/rhesis/backend/app/utils/database_exceptions.py +++ b/apps/backend/src/rhesis/backend/app/utils/database_exceptions.py @@ -194,6 +194,9 @@ async def async_wrapper(*args, **kwargs): except HTTPException: # Re-raise HTTP exceptions without modification raise + except ItemDeletedException: + # Let the app-level handler turn this into its 410 response + raise except Exception as e: DatabaseExceptionHandler.handle_database_error( e, @@ -216,6 +219,9 @@ def sync_wrapper(*args, **kwargs): except HTTPException: # Re-raise HTTP exceptions without modification raise + except ItemDeletedException: + # Let the app-level handler turn this into its 410 response + raise except Exception as e: DatabaseExceptionHandler.handle_database_error( e, diff --git a/apps/backend/src/rhesis/backend/app/utils/execution_validation.py b/apps/backend/src/rhesis/backend/app/utils/execution_validation.py index 949a92602a..362c427960 100644 --- a/apps/backend/src/rhesis/backend/app/utils/execution_validation.py +++ b/apps/backend/src/rhesis/backend/app/utils/execution_validation.py @@ -13,6 +13,7 @@ from rhesis.backend.app.auth.user_utils import require_current_user_or_token from rhesis.backend.app.dependencies import get_tenant_db_session from rhesis.backend.app.models.user import User +from rhesis.backend.app.utils.database_exceptions import ItemDeletedException from rhesis.backend.app.utils.model_errors import ModelConfigurationError from rhesis.backend.app.utils.user_model_utils import ( validate_user_evaluation_model, @@ -122,6 +123,10 @@ def handle_execution_error(error: Exception, operation: str = "execute tests") - # Already an HTTPException, re-raise as-is raise error + if isinstance(error, ItemDeletedException): + # Let the app-level handler turn this into its 410 response + raise error + if isinstance(error, ModelConfigurationError): return _convert_model_error_to_http_exception(error, operation) diff --git a/apps/backend/src/rhesis/backend/tasks/base.py b/apps/backend/src/rhesis/backend/tasks/base.py index 9de57e8d02..d3ce89e665 100644 --- a/apps/backend/src/rhesis/backend/tasks/base.py +++ b/apps/backend/src/rhesis/backend/tasks/base.py @@ -9,6 +9,7 @@ from rhesis.backend.app.database import ( get_db_with_tenant_variables, ) +from rhesis.backend.app.utils.database_exceptions import ItemDeletedException from rhesis.backend.app.utils.model_errors import ModelConfigurationError from rhesis.backend.tasks.enums import DEFAULT_MAX_RETRIES, DEFAULT_RETRY_BACKOFF_MAX @@ -96,8 +97,10 @@ def get_display_name(self) -> str: # Checked by Celery before autoretry_for. A bad model configuration (wrong # region, unknown model, missing credentials) returns the same error on - # every attempt, so retrying it just multiplies the log noise. - dont_autoretry_for = (ModelConfigurationError,) + # every attempt, so retrying it just multiplies the log noise. A + # soft-deleted row is the same story: whatever it referenced stays deleted + # on every retry. + dont_autoretry_for = (ModelConfigurationError, ItemDeletedException) # Maximum number of retries - use centralized constant max_retries = DEFAULT_MAX_RETRIES diff --git a/apps/backend/src/rhesis/backend/tasks/embedding/graph.py b/apps/backend/src/rhesis/backend/tasks/embedding/graph.py index acdcad8e9e..0e429ba5b6 100644 --- a/apps/backend/src/rhesis/backend/tasks/embedding/graph.py +++ b/apps/backend/src/rhesis/backend/tasks/embedding/graph.py @@ -169,13 +169,18 @@ def _run_embedding_graph( ) -> None: from rhesis.backend.app.crud import user as user_crud from rhesis.backend.app.services.embedding.graph_builder import build_2d_graph + from rhesis.backend.app.utils.database_exceptions import ItemDeletedException user = user_crud.get_user_by_id(db, user_id) if user is None: logger.warning("Skipping graph computation: user not found", extra={"user_id": user_id}) return - parent = load_parent(db, user) + try: + parent = load_parent(db, user) + except ItemDeletedException: + parent = None + if parent is None: logger.warning(f"Skipping graph computation: {parent_name} not found") return diff --git a/apps/backend/src/rhesis/backend/tasks/execution/batch/context.py b/apps/backend/src/rhesis/backend/tasks/execution/batch/context.py index 36008304c3..897c83ed84 100644 --- a/apps/backend/src/rhesis/backend/tasks/execution/batch/context.py +++ b/apps/backend/src/rhesis/backend/tasks/execution/batch/context.py @@ -138,6 +138,7 @@ def prefetch_execution_context( trace_id: Optional[str] = None, ) -> ExecutionContext: """Pre-fetch all shared data in a single session before async execution.""" + from rhesis.backend.app.crud import get_endpoint from rhesis.backend.app.crud import user as user_crud from rhesis.backend.app.database import bind_scope_to_session from rhesis.backend.app.models.requirement import Requirement @@ -154,9 +155,18 @@ def prefetch_execution_context( bind_scope_to_session(session, organization_id, user_id or "", project_id) + # get_test_set/get_endpoint raise ItemDeletedException for a soft-deleted + # row; it's in BaseTask.dont_autoretry_for, so this fails the task + # immediately instead of retrying against a row that will never come back. test_set = get_test_set(session, str(test_config.test_set_id), organization_id) - endpoint = session.query(Endpoint).filter(Endpoint.id == test_config.endpoint_id).first() + endpoint = get_endpoint( + session, + test_config.endpoint_id, + organization_id, + user_id, + project_id=project_id or None, + ) if not endpoint: raise ValueError(f"Endpoint {test_config.endpoint_id} not found") diff --git a/apps/backend/src/rhesis/backend/tasks/execution/executors/data.py b/apps/backend/src/rhesis/backend/tasks/execution/executors/data.py index 81ed4acc61..cdc8089297 100644 --- a/apps/backend/src/rhesis/backend/tasks/execution/executors/data.py +++ b/apps/backend/src/rhesis/backend/tasks/execution/executors/data.py @@ -7,7 +7,7 @@ from sqlalchemy.orm import Session from rhesis.backend.app.models.test import Test -from rhesis.backend.app.utils.query_utils import QueryBuilder, include +from rhesis.backend.app.utils.query_utils import include from rhesis.backend.tasks.execution.metrics_utils import get_requirement_metrics logger = logging.getLogger(__name__) @@ -35,16 +35,25 @@ def get_test_and_prompt( """ # Import here to avoid circular dependency from rhesis.backend.app.constants import TestType + from rhesis.backend.app.utils.crud_utils import get_item_detail + from rhesis.backend.app.utils.database_exceptions import ItemDeletedException from rhesis.backend.tasks.execution.modes import get_test_type # Get the test. Reads test.prompt below for single-turn tests -- eager-load it explicitly. - test = ( - QueryBuilder(db, Test) - .with_related(include(Test.prompt)) - .with_organization_filter(organization_id) - .with_custom_filter(lambda q: q.filter(Test.id == UUID(test_id))) - .first() - ) + # Raised as ValueError (not ItemDeletedException) so every failure mode of + # this function shares one exception type -- callers only catch/re-raise + # generically, so this is purely about message clarity ("deleted" vs + # "not found"), not Celery retry semantics. + try: + test = get_item_detail( + db, + Test, + UUID(test_id), + organization_id=organization_id, + related_fields=(include(Test.prompt),), + ) + except ItemDeletedException: + raise ValueError(f"Test with ID {test_id} has been deleted") if not test: raise ValueError(f"Test with ID {test_id} not found") diff --git a/apps/backend/src/rhesis/backend/tasks/test_set.py b/apps/backend/src/rhesis/backend/tasks/test_set.py index cc514628f0..448bc656ac 100644 --- a/apps/backend/src/rhesis/backend/tasks/test_set.py +++ b/apps/backend/src/rhesis/backend/tasks/test_set.py @@ -16,6 +16,8 @@ load_defaults, ) from rhesis.backend.app.services.usage import dispatch_accrual +from rhesis.backend.app.utils.crud_utils import _check_and_raise_if_deleted +from rhesis.backend.app.utils.query_utils import QueryBuilder from rhesis.backend.app.utils.user_model_utils import ( get_generation_model_with_override, ) @@ -318,7 +320,11 @@ def _attach_tests_to_existing_test_set( test_set_uuid = _uuid.UUID(test_set_id) with self.get_db_session() as db: with bypass_tenant_filter(): - db_test_set = db.query(TestSet).filter(TestSet.id == test_set_uuid).first() + # ItemDeletedException is in BaseTask.dont_autoretry_for, so a + # soft-deleted row fails this task immediately instead of retrying + # against a row that will never come back. + test_set_query = QueryBuilder(db, TestSet).with_deleted().filter_by_id(test_set_uuid) + db_test_set = _check_and_raise_if_deleted(test_set_query, TestSet, test_set_uuid, False) if db_test_set is None: raise ValueError(f"TestSet with id {test_set_id!r} not found in database") diff --git a/tests/backend/crud/test_model_get_contract.py b/tests/backend/crud/test_model_get_contract.py new file mode 100644 index 0000000000..814d36079c --- /dev/null +++ b/tests/backend/crud/test_model_get_contract.py @@ -0,0 +1,58 @@ +""" +get_model's soft-delete contract is deliberately different from this repo's other +single-item getters: it's used across services/tasks as an internal "resolve the +configured model, fall back if unavailable" helper, so it must keep collapsing +"missing" and "deleted" into a plain None instead of raising ItemDeletedException +-- see the module docstring on rhesis.backend.app.crud.model for why. +""" + +import pytest +from sqlalchemy.orm import Session + +from rhesis.backend.app.crud import model as model_crud +from rhesis.backend.app.schemas.model import ModelCreate + + +@pytest.mark.unit +@pytest.mark.crud +class TestGetModelSoftDeleteContract: + def test_get_model_returns_none_for_deleted( + self, test_db: Session, test_org_id: str, authenticated_user_id: str + ): + model = model_crud.create_model( + db=test_db, + model=ModelCreate( + name="Soft Delete Model", + model_name="test-model", + endpoint="https://test.example.com", + key="test-key", + ), + organization_id=test_org_id, + user_id=authenticated_user_id, + ) + model_id = model.id + + model_crud.delete_model( + test_db, model_id, organization_id=test_org_id, user_id=authenticated_user_id + ) + + result = model_crud.get_model(test_db, model_id, organization_id=test_org_id) + assert result is None + + def test_get_model_requires_organization_id( + self, test_db: Session, test_org_id: str, authenticated_user_id: str + ): + model = model_crud.create_model( + db=test_db, + model=ModelCreate( + name="Org Filter Model", + model_name="test-model", + endpoint="https://test.example.com", + key="test-key", + ), + organization_id=test_org_id, + user_id=authenticated_user_id, + ) + + with pytest.raises(ValueError, match="organization_id is required"): + model_crud.get_model(test_db, model.id) diff --git a/tests/backend/crud/test_test_set_crud.py b/tests/backend/crud/test_test_set_crud.py new file mode 100644 index 0000000000..8b7df685f9 --- /dev/null +++ b/tests/backend/crud/test_test_set_crud.py @@ -0,0 +1,138 @@ +""" +TestSet CRUD Operations Testing + +Regression coverage for get_test_set / get_test_set_by_nano_id_or_slug's soft-delete +contract: a deleted test set must raise ItemDeletedException, like every other +entity's single-item fetch, instead of silently collapsing into "not found". + +Run with: python -m pytest tests/backend/crud/test_test_set_crud.py -v +""" + +import pytest +from sqlalchemy.orm import Session + +from rhesis.backend.app import crud, models +from rhesis.backend.app.utils.database_exceptions import ItemDeletedException + + +@pytest.mark.unit +@pytest.mark.crud +class TestTestSetSoftDeleteContract: + """A soft-deleted test set must raise ItemDeletedException, not return None.""" + + def test_get_test_set_raises_for_deleted( + self, test_db: Session, test_org_id: str, authenticated_user_id: str + ): + test_set = models.TestSet( + name="Soft Delete Test Set", + organization_id=test_org_id, + user_id=authenticated_user_id, + ) + test_db.add(test_set) + test_db.commit() + test_db.refresh(test_set) + test_set_id = test_set.id + + crud.delete_test_set( + test_db, test_set_id, organization_id=test_org_id, user_id=authenticated_user_id + ) + + with pytest.raises(ItemDeletedException): + crud.get_test_set(test_db, test_set_id, organization_id=test_org_id) + + def test_get_test_set_by_nano_id_or_slug_raises_for_deleted( + self, test_db: Session, test_org_id: str, authenticated_user_id: str + ): + test_set = models.TestSet( + name="Soft Delete Test Set By Slug", + slug="soft-delete-test-set-by-slug", + organization_id=test_org_id, + user_id=authenticated_user_id, + ) + test_db.add(test_set) + test_db.commit() + test_db.refresh(test_set) + test_set_id = test_set.id + slug = test_set.slug + + crud.delete_test_set( + test_db, test_set_id, organization_id=test_org_id, user_id=authenticated_user_id + ) + + with pytest.raises(ItemDeletedException): + crud.get_test_set_by_nano_id_or_slug(test_db, slug, organization_id=test_org_id) + + def test_get_test_set_returns_none_for_nonexistent(self, test_db: Session, test_org_id: str): + import uuid + + result = crud.get_test_set(test_db, uuid.uuid4(), organization_id=test_org_id) + assert result is None + + +@pytest.mark.unit +@pytest.mark.crud +class TestUpdateTestSetAttributesSoftDeleteHandling: + """update_test_set_attributes must still no-op when a linked test set is deleted. + + Regression test: crud.get_test_set now raises ItemDeletedException instead of + returning None, so update_test_set_attributes (called by crud.update_test for + every test set a test belongs to) must catch that itself -- otherwise updating + a test would fail with a 410 just because an unrelated linked test set was + soft-deleted. + """ + + def test_update_test_succeeds_when_linked_test_set_is_deleted( + self, test_db: Session, test_org_id: str, authenticated_user_id: str + ): + from rhesis.backend.app.models.test import test_test_set_association + + compliance = models.Requirement( + name="Compliance", + organization_id=test_org_id, + user_id=authenticated_user_id, + ) + robustness = models.Requirement( + name="Robustness", + organization_id=test_org_id, + user_id=authenticated_user_id, + ) + test_db.add_all([compliance, robustness]) + test_db.flush() + + db_test = models.Test( + requirement_id=compliance.id, + organization_id=test_org_id, + user_id=authenticated_user_id, + ) + test_set = models.TestSet( + name="Deleted Linked Test Set", + organization_id=test_org_id, + user_id=authenticated_user_id, + ) + test_db.add_all([db_test, test_set]) + test_db.flush() + + test_db.execute( + test_test_set_association.insert().values( + test_id=db_test.id, + test_set_id=test_set.id, + organization_id=test_org_id, + user_id=authenticated_user_id, + ) + ) + test_db.commit() + + crud.delete_test_set( + test_db, test_set.id, organization_id=test_org_id, user_id=authenticated_user_id + ) + + result = crud.update_test( + db=test_db, + test_id=db_test.id, + test={"requirement_id": robustness.id}, + organization_id=test_org_id, + user_id=authenticated_user_id, + ) + + assert result is not None + assert result.requirement_id == robustness.id diff --git a/tests/backend/metrics/test_database_integration.py b/tests/backend/metrics/test_database_integration.py index eca4c865f0..42640bb45c 100644 --- a/tests/backend/metrics/test_database_integration.py +++ b/tests/backend/metrics/test_database_integration.py @@ -5,6 +5,8 @@ and the storage of evaluation results in TestResult.test_metrics. """ +import pytest + from rhesis.backend.app import crud, models, schemas from rhesis.backend.app.crud.metric import ( create_metric, @@ -13,6 +15,7 @@ get_metrics, update_metric, ) +from rhesis.backend.app.utils.database_exceptions import ItemDeletedException class TestDatabaseIntegration: @@ -142,8 +145,8 @@ def test_delete_metric(self, test_db, test_org_id, authenticated_user_id): delete_metric(test_db, metric_id, test_org_id, authenticated_user_id) # Verify it's deleted (soft delete) - deleted_metric = get_metric(test_db, metric_id, test_org_id) - assert deleted_metric is None + with pytest.raises(ItemDeletedException): + get_metric(test_db, metric_id, test_org_id) def test_metric_with_model_relationship( self, test_db, test_model, test_org_id, authenticated_user_id diff --git a/tests/backend/routes/test_metric.py b/tests/backend/routes/test_metric.py index 129c0f9836..b26d604910 100644 --- a/tests/backend/routes/test_metric.py +++ b/tests/backend/routes/test_metric.py @@ -177,6 +177,38 @@ def test_metric_requirement_relationship_error_handling(self, metric_factory): assert "not found" in response.json()["detail"].lower() +@pytest.mark.integration +class TestMetricSoftDeleteContract(MetricTestMixin, BaseEntityTests): + """A soft-deleted metric must surface 410 GONE everywhere, including through + @handle_database_exceptions-wrapped routes -- not just the plain GET.""" + + def test_update_deleted_metric_returns_410(self, metric_factory): + metric = metric_factory.create(self.get_sample_data()) + metric_id = metric["id"] + + delete_response = metric_factory.client.delete(self.endpoints.remove(metric_id)) + assert delete_response.status_code == status.HTTP_200_OK + + response = metric_factory.client.put( + self.endpoints.put(metric_id), json=self.get_update_data() + ) + + assert response.status_code == status.HTTP_410_GONE + + def test_improve_deleted_metric_returns_410(self, metric_factory): + metric = metric_factory.create(self.get_sample_data()) + metric_id = metric["id"] + + delete_response = metric_factory.client.delete(self.endpoints.remove(metric_id)) + assert delete_response.status_code == status.HTTP_200_OK + + response = metric_factory.client.post( + self.endpoints.improve(metric_id), json={"prompt": "make it stricter"} + ) + + assert response.status_code == status.HTTP_410_GONE + + # === METRIC-SPECIFIC VALIDATION TESTS === diff --git a/tests/backend/routes/test_telemetry_soft_delete.py b/tests/backend/routes/test_telemetry_soft_delete.py new file mode 100644 index 0000000000..02c77cd602 --- /dev/null +++ b/tests/backend/routes/test_telemetry_soft_delete.py @@ -0,0 +1,77 @@ +""" +Regression coverage for the trace soft-delete contract. + +get_trace_by_db_id must raise ItemDeletedException for a soft-deleted trace, like +every other entity's single-item fetch, instead of silently returning None -- and +routes built on it (e.g. add_trace_review) must surface that as 410 GONE, not a +bare 404. +""" + +import uuid +from datetime import datetime, timezone + +import pytest +from fastapi import status +from fastapi.testclient import TestClient + +from rhesis.backend.app import models +from rhesis.backend.app.crud.telemetry import get_trace_by_db_id +from rhesis.backend.app.utils.database_exceptions import ItemDeletedException +from tests.backend.routes.fixtures.data_factories import TraceDataFactory + + +def _ingest_trace(client: TestClient, project_id: str) -> dict: + span_data = TraceDataFactory.sample_data(project_id=project_id) + trace_batch = {"spans": [span_data]} + response = client.post("/telemetry/traces", json=trace_batch) + assert response.status_code == status.HTTP_200_OK + return {"trace_id": span_data["trace_id"], "db_id": response.json().get("trace_db_id")} + + +def _get_trace_db_id(client: TestClient, project_id: str, trace_id: str) -> str: + response = client.get(f"/telemetry/traces/{trace_id}?project_id={project_id}") + assert response.status_code == status.HTTP_200_OK + root_spans = response.json().get("root_spans", []) + assert root_spans + return root_spans[0]["id"] + + +@pytest.mark.integration +class TestTraceSoftDeleteContract: + def test_get_trace_by_db_id_raises_for_deleted( + self, test_db, authenticated_client: TestClient, db_project, test_org_id: str + ): + ingested = _ingest_trace(authenticated_client, str(db_project.id)) + trace_db_id = _get_trace_db_id( + authenticated_client, str(db_project.id), ingested["trace_id"] + ) + + trace = test_db.query(models.Trace).filter(models.Trace.id == trace_db_id).first() + trace.deleted_at = datetime.now(timezone.utc) + test_db.commit() + + with pytest.raises(ItemDeletedException): + get_trace_by_db_id(test_db, trace_db_id, test_org_id) + + def test_add_review_on_deleted_trace_returns_410( + self, test_db, authenticated_client: TestClient, db_project + ): + ingested = _ingest_trace(authenticated_client, str(db_project.id)) + trace_db_id = _get_trace_db_id( + authenticated_client, str(db_project.id), ingested["trace_id"] + ) + + trace = test_db.query(models.Trace).filter(models.Trace.id == trace_db_id).first() + trace.deleted_at = datetime.now(timezone.utc) + test_db.commit() + + response = authenticated_client.post( + f"/telemetry/traces/{trace_db_id}/reviews", + json={ + "status_id": str(uuid.uuid4()), + "comments": "should not get here", + "target": {"type": "trace", "reference": None}, + }, + ) + + assert response.status_code == status.HTTP_410_GONE diff --git a/tests/backend/routes/test_test_set_soft_delete.py b/tests/backend/routes/test_test_set_soft_delete.py new file mode 100644 index 0000000000..7ae1c329cf --- /dev/null +++ b/tests/backend/routes/test_test_set_soft_delete.py @@ -0,0 +1,35 @@ +""" +Regression coverage for the test set soft-delete contract. + +A soft-deleted test set must surface 410 GONE on every route that resolves it by +identifier -- not just GET, but also routes wrapped in their own try/except +(download) or routed through handle_execution_error (execute) -- since either +could otherwise swallow ItemDeletedException into a bare 500. +""" + +import pytest +from fastapi import status +from fastapi.testclient import TestClient + + +@pytest.mark.integration +class TestTestSetSoftDeleteContract: + def test_get_deleted_test_set_returns_410(self, authenticated_client: TestClient, db_test_set): + test_set_id = db_test_set.id + + delete_response = authenticated_client.delete(f"/test_sets/{test_set_id}") + assert delete_response.status_code == status.HTTP_200_OK + + response = authenticated_client.get(f"/test_sets/{test_set_id}") + assert response.status_code == status.HTTP_410_GONE + + def test_download_deleted_test_set_returns_410( + self, authenticated_client: TestClient, db_test_set + ): + test_set_id = db_test_set.id + + delete_response = authenticated_client.delete(f"/test_sets/{test_set_id}") + assert delete_response.status_code == status.HTTP_200_OK + + response = authenticated_client.get(f"/test_sets/{test_set_id}/download") + assert response.status_code == status.HTTP_410_GONE diff --git a/tests/backend/services/garak/test_sync.py b/tests/backend/services/garak/test_sync.py index 3eac72cddd..1b2e8e9e48 100644 --- a/tests/backend/services/garak/test_sync.py +++ b/tests/backend/services/garak/test_sync.py @@ -7,6 +7,7 @@ from faker import Faker from sqlalchemy.orm import Session +from rhesis.backend.app import crud from rhesis.backend.app.models.test import Test, test_test_set_association from rhesis.backend.app.models.test_set import TestSet from rhesis.backend.app.services.garak.probes import GarakProbeInfo @@ -128,6 +129,63 @@ def test_can_sync_test_set_not_found(self, test_db: Session, test_org_id): assert result is False +@pytest.mark.unit +@pytest.mark.service +class TestGarakSyncServiceSoftDeleteContract: + """A soft-deleted test set must not be treated as syncable, and must not + crash any of the four lookups with an unhandled ItemDeletedException.""" + + @pytest.fixture + def deleted_garak_test_set(self, test_db: Session, test_org_id, authenticated_user_id): + test_set = TestSet( + name="Deleted Garak Test Set", + description="Test", + organization_id=test_org_id, + user_id=authenticated_user_id, + attributes={ + "source": "garak", + "garak_module": "dan", + "garak_probe_class": "Dan_11_0", + }, + ) + test_db.add(test_set) + test_db.commit() + test_db.refresh(test_set) + + crud.delete_test_set( + test_db, test_set.id, organization_id=test_org_id, user_id=authenticated_user_id + ) + return test_set + + def test_can_sync_returns_false_for_deleted( + self, test_db: Session, test_org_id, deleted_garak_test_set + ): + service = GarakSyncService(test_db) + assert service.can_sync(str(deleted_garak_test_set.id), str(test_org_id)) is False + + def test_get_sync_preview_returns_none_for_deleted( + self, test_db: Session, test_org_id, deleted_garak_test_set + ): + service = GarakSyncService(test_db) + assert service.get_sync_preview(str(deleted_garak_test_set.id), str(test_org_id)) is None + + def test_resolve_sync_target_raises_for_deleted( + self, test_db: Session, test_org_id, deleted_garak_test_set + ): + service = GarakSyncService(test_db) + with pytest.raises(ValueError, match="has been deleted"): + service.resolve_sync_target(str(deleted_garak_test_set.id), str(test_org_id)) + + def test_sync_test_set_raises_for_deleted( + self, test_db: Session, test_org_id, authenticated_user_id, deleted_garak_test_set + ): + service = GarakSyncService(test_db) + with pytest.raises(ValueError, match="has been deleted"): + service.sync_test_set( + str(deleted_garak_test_set.id), str(test_org_id), str(authenticated_user_id) + ) + + @pytest.mark.unit @pytest.mark.service class TestGarakSyncServiceProbeIds: diff --git a/tests/backend/services/test_endpoint_service.py b/tests/backend/services/test_endpoint_service.py index 0902827b35..e47671bf09 100644 --- a/tests/backend/services/test_endpoint_service.py +++ b/tests/backend/services/test_endpoint_service.py @@ -43,13 +43,15 @@ def test_get_endpoint_success(self, test_db: Session, db_endpoint_minimal: Endpo """Test successful endpoint retrieval""" service = EndpointService() - result = service._get_endpoint(test_db, str(db_endpoint_minimal.id)) + result = service._get_endpoint( + test_db, str(db_endpoint_minimal.id), str(db_endpoint_minimal.organization_id) + ) assert result is not None assert result.id == db_endpoint_minimal.id assert result.name == db_endpoint_minimal.name - def test_get_endpoint_not_found(self, test_db: Session): + def test_get_endpoint_not_found(self, test_db: Session, test_org_id: str): """Test endpoint retrieval when endpoint doesn't exist""" import uuid @@ -59,11 +61,32 @@ def test_get_endpoint_not_found(self, test_db: Session): nonexistent_id = str(uuid.uuid4()) with pytest.raises(HTTPException) as exc_info: - service._get_endpoint(test_db, nonexistent_id) + service._get_endpoint(test_db, nonexistent_id, test_org_id) assert exc_info.value.status_code == 404 assert "Endpoint not found" in str(exc_info.value.detail) + def test_get_endpoint_deleted_returns_410( + self, test_db: Session, db_endpoint_minimal: Endpoint + ): + """A soft-deleted endpoint must surface as 410, not a bare 404.""" + from rhesis.backend.app import crud + + organization_id = str(db_endpoint_minimal.organization_id) + crud.delete_endpoint( + test_db, + db_endpoint_minimal.id, + organization_id=organization_id, + user_id=str(db_endpoint_minimal.user_id), + ) + + service = EndpointService() + + with pytest.raises(HTTPException) as exc_info: + service._get_endpoint(test_db, str(db_endpoint_minimal.id), organization_id) + + assert exc_info.value.status_code == 410 + @pytest.mark.asyncio async def test_invoke_endpoint_success(self, test_db: Session, db_endpoint_minimal: Endpoint): """Test successful endpoint invocation""" @@ -78,17 +101,26 @@ async def test_invoke_endpoint_success(self, test_db: Session, db_endpoint_minim with patch( "rhesis.backend.app.services.endpoint.service.create_invoker", return_value=mock_invoker ) as mock_create: - result = await service.invoke_endpoint(test_db, str(db_endpoint_minimal.id), input_data) + result = await service.invoke_endpoint( + test_db, + str(db_endpoint_minimal.id), + input_data, + organization_id=str(db_endpoint_minimal.organization_id), + ) assert result == {"response": "success"} # Verify invoker was called mock_invoker.invoke.assert_called_once() - + # Verify the context was created with the right parameters mock_create.assert_called_once() context = mock_create.call_args[0][0] assert context.db == test_db - assert context.input_data == input_data + # organization_id is injected server-side into the enriched input + assert context.input_data == { + **input_data, + "organization_id": str(db_endpoint_minimal.organization_id), + } assert context.endpoint.id == db_endpoint_minimal.id @pytest.mark.asyncio @@ -105,7 +137,12 @@ async def test_invoke_endpoint_value_error( "rhesis.backend.app.services.endpoint.service.create_invoker", return_value=mock_invoker ): with pytest.raises(EndpointInvocationError) as exc_info: - await service.invoke_endpoint(test_db, str(db_endpoint_minimal.id), {}) + await service.invoke_endpoint( + test_db, + str(db_endpoint_minimal.id), + {}, + organization_id=str(db_endpoint_minimal.organization_id), + ) assert exc_info.value.status_code == 400 assert exc_info.value.transient is False @@ -125,7 +162,12 @@ async def test_invoke_endpoint_general_exception( "rhesis.backend.app.services.endpoint.service.create_invoker", return_value=mock_invoker ): with pytest.raises(EndpointInvocationError) as exc_info: - await service.invoke_endpoint(test_db, str(db_endpoint_minimal.id), {}) + await service.invoke_endpoint( + test_db, + str(db_endpoint_minimal.id), + {}, + organization_id=str(db_endpoint_minimal.organization_id), + ) assert exc_info.value.status_code == 500 assert exc_info.value.transient is False @@ -271,16 +313,18 @@ async def test_full_invocation_flow(self): mock_invoker = Mock() mock_invoker.invoke = AsyncMock(return_value={"status": "success", "data": "response"}) - # Mock database + # Mock database -- _get_endpoint is mocked directly rather than the DB + # query chain, since it now delegates to crud.get_endpoint internally. mock_db = Mock(spec=Session) - mock_query = Mock() - mock_query.filter.return_value.first.return_value = mock_endpoint - mock_db.query.return_value = mock_query input_data = {"message": "test input"} - with patch( - "rhesis.backend.app.services.endpoint.service.create_invoker", return_value=mock_invoker + with ( + patch.object(service, "_get_endpoint", return_value=mock_endpoint), + patch( + "rhesis.backend.app.services.endpoint.service.create_invoker", + return_value=mock_invoker, + ), ): result = await service.invoke_endpoint(mock_db, "endpoint123", input_data) @@ -294,22 +338,25 @@ async def test_error_handling_chain(self): mock_db = Mock(spec=Session) # Test endpoint not found - mock_query = Mock() - mock_query.filter.return_value.first.return_value = None - mock_db.query.return_value = mock_query - - with pytest.raises(HTTPException) as exc_info: - await service.invoke_endpoint(mock_db, "nonexistent", {}) + with patch.object( + service, + "_get_endpoint", + side_effect=HTTPException(status_code=404, detail="Endpoint not found"), + ): + with pytest.raises(HTTPException) as exc_info: + await service.invoke_endpoint(mock_db, "nonexistent", {}) - assert exc_info.value.status_code == 404 + assert exc_info.value.status_code == 404 # Test invoker creation failure mock_endpoint = Mock(spec=Endpoint) - mock_query.filter.return_value.first.return_value = mock_endpoint - with patch( - "rhesis.backend.app.services.endpoint.service.create_invoker", - side_effect=ValueError("Invalid connection type"), + with ( + patch.object(service, "_get_endpoint", return_value=mock_endpoint), + patch( + "rhesis.backend.app.services.endpoint.service.create_invoker", + side_effect=ValueError("Invalid connection type"), + ), ): with pytest.raises(EndpointInvocationError) as exc_info: await service.invoke_endpoint(mock_db, "endpoint123", {}) @@ -985,7 +1032,9 @@ async def test_case_b_file_text_reaches_messages_for_stateless_endpoint(self): extracted_text = "Extracted text from doc" enriched = [{"filename": "doc.pdf", "data": "x", "extracted_text": extracted_text}] - injected_input = f"Tell me about this\n\n[Attached file(s):]\n\n--- doc.pdf ---\n{extracted_text}" + injected_input = ( + f"Tell me about this\n\n[Attached file(s):]\n\n--- doc.pdf ---\n{extracted_text}" + ) fake_store = MagicMock() fake_store.exists.return_value = False diff --git a/tests/backend/services/test_preflight.py b/tests/backend/services/test_preflight.py index 9a54ed1419..b6fc5b7737 100644 --- a/tests/backend/services/test_preflight.py +++ b/tests/backend/services/test_preflight.py @@ -48,7 +48,9 @@ def test_shared_check_no_test_set(self): def test_shared_check_ignores_test_set_id(self): ts_id = str(uuid4()) - assert _make_composite_key(CHECK_ENDPOINT_CONNECTIVITY, ts_id) == CHECK_ENDPOINT_CONNECTIVITY + assert ( + _make_composite_key(CHECK_ENDPOINT_CONNECTIVITY, ts_id) == CHECK_ENDPOINT_CONNECTIVITY + ) def test_per_test_set_check_with_id(self): ts_id = str(uuid4()) @@ -189,9 +191,7 @@ async def test_db_error(self): class TestCheckEvaluationModel: - MODEL_UTIL = ( - "rhesis.backend.app.utils.user_model_utils.get_evaluation_model_with_override" - ) + MODEL_UTIL = "rhesis.backend.app.utils.user_model_utils.get_evaluation_model_with_override" @pytest.mark.asyncio async def test_model_passes(self): @@ -300,9 +300,7 @@ async def test_timeout(self): class TestValidateMetricsLoadable: - MODEL_UTIL = ( - "rhesis.backend.app.utils.user_model_utils.get_evaluation_model_with_override" - ) + MODEL_UTIL = "rhesis.backend.app.utils.user_model_utils.get_evaluation_model_with_override" VALIDATE_CONFIGS = "rhesis.backend.metrics.metric_config.validate_metric_configs" PREPARE_METRICS = "rhesis.backend.metrics.strategies.local.prepare_metrics" @@ -408,8 +406,9 @@ def test_none_mapping(self): class TestCheckMetricEndpointIssues: - def _make_metric(self, name, class_name=None, context_required=False, - ground_truth_required=False): + def _make_metric( + self, name, class_name=None, context_required=False, ground_truth_required=False + ): m = MagicMock() m.name = name m.class_name = class_name @@ -510,8 +509,7 @@ def _make_db(self, metrics=None, total_tests=5, missing_ground_truth=0): # missing ground truth count missing_count_query = MagicMock() - missing_count_query.join.return_value.join.return_value.filter.return_value\ - .filter.return_value.count.return_value = missing_ground_truth + missing_count_query.join.return_value.join.return_value.filter.return_value.filter.return_value.count.return_value = missing_ground_truth return db @@ -532,6 +530,29 @@ async def test_no_metrics_skipped(self): assert result.status == PreflightCheckStatus.SKIPPED + @pytest.mark.asyncio + async def test_deleted_test_set_reports_failed_not_crash(self): + """A soft-deleted test set must surface as a FAILED check result, not + raise ItemDeletedException uncaught (use_test_set mode fetches the + test set directly).""" + from rhesis.backend.app.services.preflight.checks import check_metric_compatibility + from rhesis.backend.app.utils.database_exceptions import ItemDeletedException + + db = MagicMock() + endpoint = MagicMock() + ts_id = uuid4() + + with patch( + "rhesis.backend.app.services.preflight.checks.get_item_detail", + side_effect=ItemDeletedException("TestSet", str(ts_id)), + ): + result = await check_metric_compatibility( + db, endpoint, ts_id, "use_test_set", publish=False + ) + + assert result.status == PreflightCheckStatus.FAILED + assert "deleted" in result.detail.lower() + @pytest.mark.asyncio async def test_passed_when_all_compatible(self): from rhesis.backend.app.services.preflight.checks import check_metric_compatibility @@ -551,8 +572,7 @@ async def test_passed_when_all_compatible(self): ts_id = uuid4() result = await check_metric_compatibility( - db, endpoint, ts_id, "define_custom", - selected_metrics=[metric], publish=False + db, endpoint, ts_id, "define_custom", selected_metrics=[metric], publish=False ) assert result.status == PreflightCheckStatus.PASSED @@ -577,8 +597,7 @@ async def test_warning_when_context_missing(self): ts_id = uuid4() result = await check_metric_compatibility( - db, endpoint, ts_id, "define_custom", - selected_metrics=[metric], publish=False + db, endpoint, ts_id, "define_custom", selected_metrics=[metric], publish=False ) assert result.status == PreflightCheckStatus.WARNING @@ -601,16 +620,14 @@ async def test_warning_on_partial_ground_truth(self): # total_tests count (filter().count()) db.query.return_value.filter.return_value.count.return_value = 10 # missing ground truth count (join().join().filter().filter().count()) - db.query.return_value.join.return_value.join.return_value\ - .filter.return_value.filter.return_value.count.return_value = 3 + db.query.return_value.join.return_value.join.return_value.filter.return_value.filter.return_value.count.return_value = 3 endpoint = MagicMock() endpoint.response_mapping = {} ts_id = uuid4() result = await check_metric_compatibility( - db, endpoint, ts_id, "define_custom", - selected_metrics=[metric], publish=False + db, endpoint, ts_id, "define_custom", selected_metrics=[metric], publish=False ) assert result.status == PreflightCheckStatus.WARNING @@ -637,8 +654,13 @@ async def test_multi_turn_skips_ground_truth_query(self): ts_id = uuid4() result = await check_metric_compatibility( - db, endpoint, ts_id, "define_custom", - selected_metrics=[metric], is_multi_turn=True, publish=False + db, + endpoint, + ts_id, + "define_custom", + selected_metrics=[metric], + is_multi_turn=True, + publish=False, ) # No ground-truth warning should appear since the query is skipped @@ -657,8 +679,7 @@ async def test_db_error_returns_failed(self): ts_id = uuid4() result = await check_metric_compatibility( - db, endpoint, ts_id, "define_custom", - selected_metrics=[MagicMock()], publish=False + db, endpoint, ts_id, "define_custom", selected_metrics=[MagicMock()], publish=False ) assert result.status == PreflightCheckStatus.FAILED @@ -678,28 +699,26 @@ async def test_reuse_skips_connectivity(self): ts_id = uuid4() endpoint = MagicMock() - db.query.return_value.filter.return_value.first.return_value = endpoint async def mock_check(*args, **kwargs): - return _make_result(args[0] if isinstance(args[0], str) else "check", - PreflightCheckStatus.PASSED) + return _make_result( + args[0] if isinstance(args[0], str) else "check", PreflightCheckStatus.PASSED + ) with ( patch( - "rhesis.backend.app.services.preflight.orchestrator" - ".check_evaluation_model", + "rhesis.backend.app.services.preflight.orchestrator.get_item_detail", + return_value=endpoint, + ), + patch( + "rhesis.backend.app.services.preflight.orchestrator.check_evaluation_model", new_callable=AsyncMock, - return_value=_make_result( - CHECK_EVALUATION_MODEL, PreflightCheckStatus.PASSED - ), + return_value=_make_result(CHECK_EVALUATION_MODEL, PreflightCheckStatus.PASSED), ), patch( - "rhesis.backend.app.services.preflight.orchestrator" - ".check_test_set_not_empty", + "rhesis.backend.app.services.preflight.orchestrator.check_test_set_not_empty", new_callable=AsyncMock, - return_value=_make_result( - CHECK_TEST_SET_NOT_EMPTY, PreflightCheckStatus.PASSED - ), + return_value=_make_result(CHECK_TEST_SET_NOT_EMPTY, PreflightCheckStatus.PASSED), ), patch( "rhesis.backend.app.services.preflight.orchestrator" @@ -710,19 +729,63 @@ async def mock_check(*args, **kwargs): ), ), patch( - "rhesis.backend.app.services.preflight.orchestrator" - ".check_metric_compatibility", + "rhesis.backend.app.services.preflight.orchestrator.check_metric_compatibility", new_callable=AsyncMock, - return_value=_make_result( - CHECK_METRIC_COMPATIBILITY, PreflightCheckStatus.PASSED - ), + return_value=_make_result(CHECK_METRIC_COMPATIBILITY, PreflightCheckStatus.PASSED), + ), + patch( + "rhesis.backend.app.services.preflight.orchestrator.check_metric_functionality", + new_callable=AsyncMock, + return_value=_make_result(CHECK_METRIC_FUNCTIONALITY, PreflightCheckStatus.PASSED), + ), + ): + results = await run_preflight_checks_multi( + db=db, + user=user, + test_sets=[(ts_id, "Test Set", False)], + endpoint_id=uuid4(), + scoring_target="reuse", + publish=False, + ) + + statuses = {r.check_id: r.status for r in results} + assert statuses[CHECK_ENDPOINT_CONNECTIVITY] == PreflightCheckStatus.SKIPPED + + @pytest.mark.asyncio + async def test_deleted_endpoint_reports_failed_not_crash(self): + """A soft-deleted endpoint must surface as a FAILED connectivity check + with a distinct message, not raise ItemDeletedException uncaught.""" + from rhesis.backend.app.services.preflight.orchestrator import ( + run_preflight_checks_multi, + ) + from rhesis.backend.app.utils.database_exceptions import ItemDeletedException + + db = MagicMock() + user = MagicMock() + user.organization_id = uuid4() + ts_id = uuid4() + + with ( + patch( + "rhesis.backend.app.services.preflight.orchestrator.get_item_detail", + side_effect=ItemDeletedException("Endpoint", "some-id"), + ), + patch( + "rhesis.backend.app.services.preflight.orchestrator.check_evaluation_model", + new_callable=AsyncMock, + return_value=_make_result(CHECK_EVALUATION_MODEL, PreflightCheckStatus.PASSED), + ), + patch( + "rhesis.backend.app.services.preflight.orchestrator.check_test_set_not_empty", + new_callable=AsyncMock, + return_value=_make_result(CHECK_TEST_SET_NOT_EMPTY, PreflightCheckStatus.PASSED), ), patch( "rhesis.backend.app.services.preflight.orchestrator" - ".check_metric_functionality", + ".check_requirement_metric_coverage", new_callable=AsyncMock, return_value=_make_result( - CHECK_METRIC_FUNCTIONALITY, PreflightCheckStatus.PASSED + CHECK_REQUIREMENT_METRIC_COVERAGE, PreflightCheckStatus.PASSED ), ), ): @@ -731,9 +794,11 @@ async def mock_check(*args, **kwargs): user=user, test_sets=[(ts_id, "Test Set", False)], endpoint_id=uuid4(), - scoring_target="reuse", + scoring_target="fresh", publish=False, ) statuses = {r.check_id: r.status for r in results} - assert statuses[CHECK_ENDPOINT_CONNECTIVITY] == PreflightCheckStatus.SKIPPED + details = {r.check_id: (r.message or "") for r in results} + assert statuses[CHECK_ENDPOINT_CONNECTIVITY] == PreflightCheckStatus.FAILED + assert "deleted" in details[CHECK_ENDPOINT_CONNECTIVITY].lower() diff --git a/tests/backend/services/test_test.py b/tests/backend/services/test_test.py index 4ea3c66a24..0890767bf5 100644 --- a/tests/backend/services/test_test.py +++ b/tests/backend/services/test_test.py @@ -428,6 +428,34 @@ def test_create_test_set_associations_test_set_not_found( assert "not found" in result["message"] assert result["metadata"]["new_associations"] == 0 + def test_create_test_set_associations_test_set_deleted( + self, test_db: Session, authenticated_user_id, test_org_id + ): + """A soft-deleted test set must not crash with ItemDeletedException.""" + from rhesis.backend.app import crud + + test_set_data = create_test_set_data() + test_set = models.TestSet( + **test_set_data, organization_id=test_org_id, user_id=authenticated_user_id + ) + test_db.add(test_set) + test_db.commit() + + crud.delete_test_set( + test_db, test_set.id, organization_id=test_org_id, user_id=authenticated_user_id + ) + + result = test_service.create_test_set_associations( + db=test_db, + test_set_id=str(test_set.id), + test_ids=[str(uuid.uuid4())], + organization_id=test_org_id, + user_id=authenticated_user_id, + ) + + assert result["success"] is False + assert "deleted" in result["message"] + def test_remove_test_set_associations_success( self, test_db: Session, @@ -538,6 +566,34 @@ def test_remove_test_set_associations_test_set_not_found( assert result["removed_associations"] == 0 assert "not found" in result["message"] + def test_remove_test_set_associations_test_set_deleted( + self, test_db: Session, authenticated_user_id, test_org_id + ): + """A soft-deleted test set must not crash with ItemDeletedException.""" + from rhesis.backend.app import crud + + test_set_data = create_test_set_data() + test_set = models.TestSet( + **test_set_data, organization_id=test_org_id, user_id=authenticated_user_id + ) + test_db.add(test_set) + test_db.commit() + + crud.delete_test_set( + test_db, test_set.id, organization_id=test_org_id, user_id=authenticated_user_id + ) + + result = test_service.remove_test_set_associations( + db=test_db, + test_set_id=str(test_set.id), + test_ids=[str(uuid.uuid4())], + organization_id=test_org_id, + user_id=authenticated_user_id, + ) + + assert result["success"] is False + assert "deleted" in result["message"] + def test_remove_test_set_associations_no_existing_associations( self, test_db: Session, authenticated_user_id, test_org_id ): diff --git a/tests/backend/services/test_test_set.py b/tests/backend/services/test_test_set.py index fe4c22c6d1..e10b28b830 100644 --- a/tests/backend/services/test_test_set.py +++ b/tests/backend/services/test_test_set.py @@ -678,3 +678,34 @@ def test_get_test_sets_excludes_explorer_metadata_requirement( ids = {ts.id for ts in results} assert regular.id in ids assert explorer.id not in ids + + +class TestGetTestSetSoftDeleteContract: + """services.test_set.get_test_set must raise ItemDeletedException for a + soft-deleted row, like every other entity's single-item fetch -- not + silently collapse it into "not found" like get_item does for None.""" + + def test_raises_for_deleted_test_set( + self, test_db: Session, test_org_id, authenticated_user_id + ): + from rhesis.backend.app.utils.database_exceptions import ItemDeletedException + + test_set = models.TestSet( + name="Soft Delete Services Test Set", + organization_id=test_org_id, + user_id=authenticated_user_id, + ) + test_db.add(test_set) + test_db.commit() + test_db.refresh(test_set) + + crud.delete_test_set( + test_db, test_set.id, organization_id=test_org_id, user_id=authenticated_user_id + ) + + with pytest.raises(ItemDeletedException): + test_set_service.get_test_set(test_db, test_set.id, str(test_org_id)) + + def test_returns_none_for_nonexistent(self, test_db: Session, test_org_id): + result = test_set_service.get_test_set(test_db, uuid.uuid4(), str(test_org_id)) + assert result is None diff --git a/tests/backend/tasks/test_attach_tests_to_existing_test_set.py b/tests/backend/tasks/test_attach_tests_to_existing_test_set.py new file mode 100644 index 0000000000..0667a554eb --- /dev/null +++ b/tests/backend/tasks/test_attach_tests_to_existing_test_set.py @@ -0,0 +1,73 @@ +""" +Regression coverage for _attach_tests_to_existing_test_set's soft-delete +handling. + +The pre-created TestSet row is fetched with bypass_tenant_filter() (the row +was created by the router before this Celery task runs), using the standard +with_deleted() + _check_and_raise_if_deleted() pattern. A soft-deleted row +raises ItemDeletedException, same as everywhere else -- it's listed in +BaseTask.dont_autoretry_for, so Celery treats it as terminal instead of +retrying forever against a row that will never come back. +""" + +from contextlib import contextmanager +from unittest.mock import MagicMock + +import pytest +from sqlalchemy.orm import Session + +from rhesis.backend.app import crud, models +from rhesis.backend.app.utils.database_exceptions import ItemDeletedException + + +def _make_sdk_test_set(): + """Minimal SDK-like test set object with one test.""" + ts = MagicMock() + ts.test_set_type = "single_turn" + ts.tests = [ + { + "prompt": {"content": "hello", "language_code": "en"}, + "requirement": "Harmful", + "category": "Security", + "topic": "Injection", + "metadata": {"generated_by": "ConfigSynthesizer", "additional_info": {}}, + } + ] + return ts + + +@pytest.mark.unit +class TestAttachTestsToExistingTestSetSoftDelete: + def test_raises_item_deleted_exception_for_deleted_test_set( + self, test_db: Session, test_org_id: str, authenticated_user_id: str + ): + from rhesis.backend.tasks.test_set import _attach_tests_to_existing_test_set + + test_set = models.TestSet( + name="Pre-created Test Set", + organization_id=test_org_id, + user_id=authenticated_user_id, + ) + test_db.add(test_set) + test_db.commit() + test_db.refresh(test_set) + + crud.delete_test_set( + test_db, test_set.id, organization_id=test_org_id, user_id=authenticated_user_id + ) + + @contextmanager + def fake_get_db_session(): + yield test_db + + mock_task = MagicMock() + mock_task.get_db_session = fake_get_db_session + + with pytest.raises(ItemDeletedException): + _attach_tests_to_existing_test_set( + mock_task, + _make_sdk_test_set(), + test_set_id=str(test_set.id), + org_id=test_org_id, + user_id=authenticated_user_id, + ) diff --git a/tests/backend/tasks/test_batch_per_test_metrics.py b/tests/backend/tasks/test_batch_per_test_metrics.py index 90b09e4aa0..27f339d944 100644 --- a/tests/backend/tasks/test_batch_per_test_metrics.py +++ b/tests/backend/tasks/test_batch_per_test_metrics.py @@ -34,9 +34,7 @@ def _make_metric_config( mc.class_name = class_name # Batch evaluation drops configs that declare no scope, so default to both # turn types: these tests are about which configs are selected, not scope. - mc.metric_scope = ( - metric_scope if metric_scope is not None else ["Single-Turn", "Multi-Turn"] - ) + mc.metric_scope = metric_scope if metric_scope is not None else ["Single-Turn", "Multi-Turn"] return mc @@ -110,9 +108,7 @@ def test_false_when_empty(self): assert ctx.has_metrics is False def test_true_with_shared(self): - ctx = _make_execution_context( - metric_configs=[_make_metric_config("M", "J")] - ) + ctx = _make_execution_context(metric_configs=[_make_metric_config("M", "J")]) assert ctx.has_metrics is True def test_true_with_per_test(self): @@ -157,26 +153,16 @@ def _make_test(self, test_id, requirement_name, metric_name, metric_class): return test, metric @patch("rhesis.backend.metrics.metric_config.metric_model_to_config") - @patch( - "rhesis.backend.tasks.execution.executors.data.get_test_metrics" - ) - def test_requirement_metrics_resolved_per_test( - self, mock_get_metrics, mock_to_config - ): + @patch("rhesis.backend.tasks.execution.executors.data.get_test_metrics") + def test_requirement_metrics_resolved_per_test(self, mock_get_metrics, mock_to_config): """When using requirement metrics (P3), each test gets its own configs.""" - test_1, metric_1 = self._make_test( - "t1", "Requirement A", "Metric A", "JudgeA" - ) - test_2, metric_2 = self._make_test( - "t2", "Requirement B", "Metric B", "JudgeB" - ) + test_1, metric_1 = self._make_test("t1", "Requirement A", "Metric A", "JudgeA") + test_2, metric_2 = self._make_test("t2", "Requirement B", "Metric B", "JudgeB") # get_test_metrics returns different metrics per test; the sample-test # call passes return_source=True and expects a (metrics, source) tuple. def side_effect(test, *args, **kwargs): - metrics = {str(test_1.id): [metric_1], str(test_2.id): [metric_2]}[ - str(test.id) - ] + metrics = {str(test_1.id): [metric_1], str(test_2.id): [metric_2]}[str(test.id)] if kwargs.get("return_source"): return metrics, "requirement" return metrics @@ -214,26 +200,21 @@ def to_config(m): endpoint.environment = None session = MagicMock() - session.query.return_value.filter.return_value.first.return_value = endpoint with ( - patch( - "rhesis.backend.app.database.bind_scope_to_session" - ), + patch("rhesis.backend.app.database.bind_scope_to_session"), patch( "rhesis.backend.app.services.test_set.get_test_set", return_value=test_set, ), patch( - "rhesis.backend.app.services.invokers.auth.manager" - ".AuthenticationManager" - ), - patch( - "rhesis.backend.app.config.settings.get_model_settings" + "rhesis.backend.app.crud.get_endpoint", + return_value=endpoint, ), + patch("rhesis.backend.app.services.invokers.auth.manager.AuthenticationManager"), + patch("rhesis.backend.app.config.settings.get_model_settings"), patch( - "rhesis.backend.tasks.execution.executors.data" - ".get_test_and_prompt", + "rhesis.backend.tasks.execution.executors.data.get_test_and_prompt", side_effect=lambda s, tid, org: ( test_1 if tid == str(test_1.id) else test_2, "prompt", @@ -245,9 +226,7 @@ def to_config(m): prefetch_execution_context, ) - ctx = prefetch_execution_context( - session, test_config, test_run, [test_1, test_2] - ) + ctx = prefetch_execution_context(session, test_config, test_run, [test_1, test_2]) # Shared should be empty; per-test should have entries assert ctx.metric_configs == [] @@ -262,9 +241,7 @@ def to_config(m): assert configs_2[0].class_name == "JudgeB" @patch("rhesis.backend.metrics.metric_config.metric_model_to_config") - @patch( - "rhesis.backend.tasks.execution.executors.data.get_test_metrics" - ) + @patch("rhesis.backend.tasks.execution.executors.data.get_test_metrics") def test_test_set_metrics_shared(self, mock_get_metrics, mock_to_config): """When using test_set metrics (P2), all tests share the same configs.""" test_1, _ = self._make_test("t1", "Requirement A", "Metric A", "JudgeA") @@ -309,26 +286,21 @@ def to_config(m): endpoint.environment = None session = MagicMock() - session.query.return_value.filter.return_value.first.return_value = endpoint with ( - patch( - "rhesis.backend.app.database.bind_scope_to_session" - ), + patch("rhesis.backend.app.database.bind_scope_to_session"), patch( "rhesis.backend.app.services.test_set.get_test_set", return_value=test_set, ), patch( - "rhesis.backend.app.services.invokers.auth.manager" - ".AuthenticationManager" - ), - patch( - "rhesis.backend.app.config.settings.get_model_settings" + "rhesis.backend.app.crud.get_endpoint", + return_value=endpoint, ), + patch("rhesis.backend.app.services.invokers.auth.manager.AuthenticationManager"), + patch("rhesis.backend.app.config.settings.get_model_settings"), patch( - "rhesis.backend.tasks.execution.executors.data" - ".get_test_and_prompt", + "rhesis.backend.tasks.execution.executors.data.get_test_and_prompt", side_effect=lambda s, tid, org: ( test_1 if tid == str(test_1.id) else test_2, "prompt", @@ -340,9 +312,7 @@ def to_config(m): prefetch_execution_context, ) - ctx = prefetch_execution_context( - session, test_config, test_run, [test_1, test_2] - ) + ctx = prefetch_execution_context(session, test_config, test_run, [test_1, test_2]) # Shared should be populated; per-test should be empty assert len(ctx.metric_configs) == 1 @@ -396,9 +366,7 @@ def test_configured_but_invalid_execution_time_metrics_falls_back_per_test( # resolved to nothing valid, so it fell through to P3 (requirement) — # which differs per test. def side_effect(test, *args, **kwargs): - metrics = {str(test_1.id): [metric_1], str(test_2.id): [metric_2]}[ - str(test.id) - ] + metrics = {str(test_1.id): [metric_1], str(test_2.id): [metric_2]}[str(test.id)] if kwargs.get("return_source"): return metrics, "requirement" return metrics @@ -436,7 +404,6 @@ def to_config(m): endpoint.environment = None session = MagicMock() - session.query.return_value.filter.return_value.first.return_value = endpoint with ( patch("rhesis.backend.app.database.bind_scope_to_session"), @@ -445,13 +412,13 @@ def to_config(m): return_value=test_set, ), patch( - "rhesis.backend.app.services.invokers.auth.manager" - ".AuthenticationManager" + "rhesis.backend.app.crud.get_endpoint", + return_value=endpoint, ), + patch("rhesis.backend.app.services.invokers.auth.manager.AuthenticationManager"), patch("rhesis.backend.app.config.settings.get_model_settings"), patch( - "rhesis.backend.tasks.execution.executors.data" - ".get_test_and_prompt", + "rhesis.backend.tasks.execution.executors.data.get_test_and_prompt", side_effect=lambda s, tid, org: ( test_1 if tid == str(test_1.id) else test_2, "prompt", @@ -463,9 +430,7 @@ def to_config(m): prefetch_execution_context, ) - ctx = prefetch_execution_context( - session, test_config, test_run, [test_1, test_2] - ) + ctx = prefetch_execution_context(session, test_config, test_run, [test_1, test_2]) # Even though P1 config was present, it didn't actually win — the # per-test path must be used, not a single shared resolution from @@ -504,9 +469,7 @@ async def test_evaluate_uses_per_test_configs(self): ) mock_evaluator = MagicMock() - mock_evaluator.a_evaluate = MagicMock( - return_value={"MetricA": {"is_successful": True}} - ) + mock_evaluator.a_evaluate = MagicMock(return_value={"MetricA": {"is_successful": True}}) # Make it awaitable import asyncio @@ -546,8 +509,6 @@ async def test_evaluate_uses_per_test_configs(self): # The evaluator should have been called with test-1's metric (MetricA), # not test-2's (MetricB) or an empty shared list. call_kwargs = mock_evaluator.a_evaluate.call_args - metrics_passed = call_kwargs.kwargs.get("metrics") or call_kwargs[1].get( - "metrics" - ) + metrics_passed = call_kwargs.kwargs.get("metrics") or call_kwargs[1].get("metrics") assert len(metrics_passed) == 1 assert metrics_passed[0].class_name == "JudgeA" diff --git a/tests/backend/tasks/test_validation.py b/tests/backend/tasks/test_validation.py index d60c9ccd9d..90cf52d865 100644 --- a/tests/backend/tasks/test_validation.py +++ b/tests/backend/tasks/test_validation.py @@ -12,25 +12,32 @@ import pytest from rhesis.backend.app.constants import TestType +from rhesis.backend.app.utils.database_exceptions import ItemDeletedException from rhesis.backend.tasks.execution.executors.data import get_test_and_prompt def _mock_query_builder(mocker, test_obj): - """Mock the QueryBuilder chain get_test_and_prompt uses to fetch the test. + """Mock get_item_detail, which get_test_and_prompt uses to fetch the test.""" + mocker.patch("rhesis.backend.app.utils.crud_utils.get_item_detail", return_value=test_obj) - QueryBuilder(db, Test).with_related(...).with_organization_filter(...) - .with_custom_filter(...).first() -- each chained call returns the same - mock so .first() can be configured once. + +def test_deleted_test_raises_clear_value_error(mocker): + """A soft-deleted test must raise a clear ValueError, not ItemDeletedException. + + get_test_and_prompt's contract is ValueError for every failure mode -- + background executors only catch/re-raise generically, so the exception + type doesn't need to change, but the message should say "deleted" rather + than the generic "not found". """ - builder = MagicMock() - builder.with_related.return_value = builder - builder.with_organization_filter.return_value = builder - builder.with_custom_filter.return_value = builder - builder.first.return_value = test_obj + mock_db = MagicMock() mocker.patch( - "rhesis.backend.tasks.execution.executors.data.QueryBuilder", return_value=builder + "rhesis.backend.app.utils.crud_utils.get_item_detail", + side_effect=ItemDeletedException("Test", "some-id"), ) + with pytest.raises(ValueError, match="has been deleted"): + get_test_and_prompt(mock_db, str(uuid4())) + def test_single_turn_requires_prompt(mocker): """Test that single-turn tests require a prompt."""