From 5261e1ad4146bb48fc9d6c231a30c9fb0e9b4df7 Mon Sep 17 00:00:00 2001 From: "@darshanmandge" Date: Fri, 8 May 2026 08:18:18 +0200 Subject: [PATCH 1/7] Add emodel_circuit derivation type with name field --- .gitignore | 1 + ...e_to_derivation_and_emodel_circuit_type.py | 54 +++++++++++++++++++ app/db/model.py | 1 + app/db/types.py | 5 ++ app/schemas/derivation.py | 2 + scripts/export/build_database_archive.sh | 4 +- tests/test_derivation.py | 26 +++++++++ 7 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 alembic/versions/20260508_043423_92d1f0c96bfd_add_name_to_derivation_and_emodel_circuit_type.py diff --git a/.gitignore b/.gitignore index c63007044..f8226ab61 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ coverage.xml .vscode *.egg-info tmp +temp/ .DS_Store export.tar files.txt diff --git a/alembic/versions/20260508_043423_92d1f0c96bfd_add_name_to_derivation_and_emodel_circuit_type.py b/alembic/versions/20260508_043423_92d1f0c96bfd_add_name_to_derivation_and_emodel_circuit_type.py new file mode 100644 index 000000000..7bbeea956 --- /dev/null +++ b/alembic/versions/20260508_043423_92d1f0c96bfd_add_name_to_derivation_and_emodel_circuit_type.py @@ -0,0 +1,54 @@ +"""Add name column to derivation and emodel_circuit DerivationType + +Revision ID: 92d1f0c96bfd +Revises: c8cdf20bbb0d +Create Date: 2026-05-08 04:34:23.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from alembic_postgresql_enum import TableReference + +import app.db.types + +# revision identifiers, used by Alembic. +revision: str = "92d1f0c96bfd" +down_revision: Union[str, None] = "c8cdf20bbb0d" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Add nullable name column to derivation table. + op.add_column("derivation", sa.Column("name", sa.String(), nullable=True)) + + # Extend DerivationType enum with emodel_circuit. + op.sync_enum_values( + enum_schema="public", + enum_name="derivationtype", + new_values=["circuit_extraction", "circuit_rewiring", "emodel_circuit", "unspecified"], + affected_columns=[ + TableReference( + table_schema="public", table_name="derivation", column_name="derivation_type" + ) + ], + enum_values_to_rename=[], + ) + + +def downgrade() -> None: + op.sync_enum_values( + enum_schema="public", + enum_name="derivationtype", + new_values=["circuit_extraction", "circuit_rewiring", "unspecified"], + affected_columns=[ + TableReference( + table_schema="public", table_name="derivation", column_name="derivation_type" + ) + ], + enum_values_to_rename=[], + ) + op.drop_column("derivation", "name") diff --git a/app/db/model.py b/app/db/model.py index d1a73dd97..d84c0ee3f 100644 --- a/app/db/model.py +++ b/app/db/model.py @@ -1671,6 +1671,7 @@ class Derivation(Base): used: Mapped["Entity"] = relationship(foreign_keys=[used_id]) generated: Mapped["Entity"] = relationship(foreign_keys=[generated_id], passive_deletes=True) derivation_type: Mapped[DerivationType] + name: Mapped[str | None] = mapped_column(nullable=True) class ScientificArtifactPublicationLink(Identifiable): diff --git a/app/db/types.py b/app/db/types.py index 29926251f..ed2541adf 100644 --- a/app/db/types.py +++ b/app/db/types.py @@ -252,11 +252,16 @@ class DerivationType(StrEnum): a circuit. circuit_rewiring: Indicates that the entity was derived by rewiring the connectivity of a circuit. + emodel_circuit: Indicates that an emodel (used) was assigned to neurons of a circuit + (generated). The optional ``name`` field on the derivation carries the SONATA + ``model_template`` entry, by convention ``hoc:`` + (e.g. ``hoc:cADpyr_L5TPC``, ``hoc:bAC_L23BC``). unspecified: Indicates a derivation that does not require a specific type. """ circuit_extraction = auto() circuit_rewiring = auto() + emodel_circuit = auto() unspecified = auto() diff --git a/app/schemas/derivation.py b/app/schemas/derivation.py index 87f2d7f98..3d004231f 100644 --- a/app/schemas/derivation.py +++ b/app/schemas/derivation.py @@ -14,9 +14,11 @@ class DerivationCreate(DerivationBase): used_id: uuid.UUID generated_id: uuid.UUID derivation_type: DerivationType + name: str | None = None class DerivationRead(DerivationBase): used: BasicEntityRead generated: BasicEntityRead derivation_type: DerivationType + name: str | None = None diff --git a/scripts/export/build_database_archive.sh b/scripts/export/build_database_archive.sh index 8c784eac5..c06f79844 100755 --- a/scripts/export/build_database_archive.sh +++ b/scripts/export/build_database_archive.sh @@ -2,7 +2,7 @@ # Automatically generated, do not edit! set -euo pipefail SCRIPT_VERSION="1" -SCRIPT_DB_VERSION="c8cdf20bbb0d" +SCRIPT_DB_VERSION="92d1f0c96bfd" echo "DB dump (version $SCRIPT_VERSION for db version $SCRIPT_DB_VERSION)" @@ -271,7 +271,7 @@ install -m 755 /dev/stdin "$WORK_DIR/load.sh" <<'EOF_LOAD_SCRIPT' # Automatically generated, do not edit! set -euo pipefail SCRIPT_VERSION="1" -SCRIPT_DB_VERSION="c8cdf20bbb0d" +SCRIPT_DB_VERSION="92d1f0c96bfd" echo "DB load (version $SCRIPT_VERSION for db version $SCRIPT_DB_VERSION)" diff --git a/tests/test_derivation.py b/tests/test_derivation.py index 5f6ab596b..a68733898 100644 --- a/tests/test_derivation.py +++ b/tests/test_derivation.py @@ -141,6 +141,7 @@ def test_get_derived_from( [ "circuit_extraction", "circuit_rewiring", + "emodel_circuit", "unspecified", ], ) @@ -158,6 +159,31 @@ def test_create_one(client, derivation_type, root_circuit, circuit): "used": {"type": "circuit", "id": str(root_circuit.id)}, "generated": {"type": "circuit", "id": str(circuit.id)}, "derivation_type": derivation_type, + "name": None, + } + + +def test_create_emodel_circuit_with_name(client, root_circuit, circuit): + """Link an emodel (used) to a circuit (generated) with the SONATA model_template label. + + The ``used_id`` should be an EModel id in real usage; here we reuse a circuit fixture + since the generic /derivation endpoint does not constrain the entity types. + """ + data = assert_request( + client.post, + url="/derivation", + json={ + "used_id": str(root_circuit.id), + "generated_id": str(circuit.id), + "derivation_type": "emodel_circuit", + "name": "hoc:cADpyr_L5TPC", + }, + ).json() + assert data == { + "used": {"type": "circuit", "id": str(root_circuit.id)}, + "generated": {"type": "circuit", "id": str(circuit.id)}, + "derivation_type": "emodel_circuit", + "name": "hoc:cADpyr_L5TPC", } From 7fbbc7d297275b14326a5ad42f1016710d7a26f6 Mon Sep 17 00:00:00 2001 From: "@darshanmandge" Date: Fri, 8 May 2026 08:37:33 +0200 Subject: [PATCH 2/7] Add emodel_circuit derivation type with label field ' --- ...label_to_derivation_and_emodel_circuit_type.py} | 14 +++++++------- app/db/model.py | 2 +- app/db/types.py | 2 +- app/schemas/derivation.py | 4 ++-- scripts/export/build_database_archive.sh | 4 ++-- tests/test_derivation.py | 8 ++++---- 6 files changed, 17 insertions(+), 17 deletions(-) rename alembic/versions/{20260508_043423_92d1f0c96bfd_add_name_to_derivation_and_emodel_circuit_type.py => 20260508_063047_1caa2fc3d44e_add_label_to_derivation_and_emodel_circuit_type.py} (78%) diff --git a/alembic/versions/20260508_043423_92d1f0c96bfd_add_name_to_derivation_and_emodel_circuit_type.py b/alembic/versions/20260508_063047_1caa2fc3d44e_add_label_to_derivation_and_emodel_circuit_type.py similarity index 78% rename from alembic/versions/20260508_043423_92d1f0c96bfd_add_name_to_derivation_and_emodel_circuit_type.py rename to alembic/versions/20260508_063047_1caa2fc3d44e_add_label_to_derivation_and_emodel_circuit_type.py index 7bbeea956..a3ea69d8c 100644 --- a/alembic/versions/20260508_043423_92d1f0c96bfd_add_name_to_derivation_and_emodel_circuit_type.py +++ b/alembic/versions/20260508_063047_1caa2fc3d44e_add_label_to_derivation_and_emodel_circuit_type.py @@ -1,8 +1,8 @@ -"""Add name column to derivation and emodel_circuit DerivationType +"""Add label column to derivation and emodel_circuit DerivationType -Revision ID: 92d1f0c96bfd +Revision ID: 1caa2fc3d44e Revises: c8cdf20bbb0d -Create Date: 2026-05-08 04:34:23.000000 +Create Date: 2026-05-08 06:30:47.000000 """ @@ -15,15 +15,15 @@ import app.db.types # revision identifiers, used by Alembic. -revision: str = "92d1f0c96bfd" +revision: str = "1caa2fc3d44e" down_revision: Union[str, None] = "c8cdf20bbb0d" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: - # Add nullable name column to derivation table. - op.add_column("derivation", sa.Column("name", sa.String(), nullable=True)) + # Add nullable label column to derivation table. + op.add_column("derivation", sa.Column("label", sa.String(), nullable=True)) # Extend DerivationType enum with emodel_circuit. op.sync_enum_values( @@ -51,4 +51,4 @@ def downgrade() -> None: ], enum_values_to_rename=[], ) - op.drop_column("derivation", "name") + op.drop_column("derivation", "label") diff --git a/app/db/model.py b/app/db/model.py index d84c0ee3f..753de15da 100644 --- a/app/db/model.py +++ b/app/db/model.py @@ -1671,7 +1671,7 @@ class Derivation(Base): used: Mapped["Entity"] = relationship(foreign_keys=[used_id]) generated: Mapped["Entity"] = relationship(foreign_keys=[generated_id], passive_deletes=True) derivation_type: Mapped[DerivationType] - name: Mapped[str | None] = mapped_column(nullable=True) + label: Mapped[str | None] = mapped_column(nullable=True) class ScientificArtifactPublicationLink(Identifiable): diff --git a/app/db/types.py b/app/db/types.py index ed2541adf..d78936e29 100644 --- a/app/db/types.py +++ b/app/db/types.py @@ -253,7 +253,7 @@ class DerivationType(StrEnum): circuit_rewiring: Indicates that the entity was derived by rewiring the connectivity of a circuit. emodel_circuit: Indicates that an emodel (used) was assigned to neurons of a circuit - (generated). The optional ``name`` field on the derivation carries the SONATA + (generated). The optional ``label`` field on the derivation carries the SONATA ``model_template`` entry, by convention ``hoc:`` (e.g. ``hoc:cADpyr_L5TPC``, ``hoc:bAC_L23BC``). unspecified: Indicates a derivation that does not require a specific type. diff --git a/app/schemas/derivation.py b/app/schemas/derivation.py index 3d004231f..df903f7fc 100644 --- a/app/schemas/derivation.py +++ b/app/schemas/derivation.py @@ -14,11 +14,11 @@ class DerivationCreate(DerivationBase): used_id: uuid.UUID generated_id: uuid.UUID derivation_type: DerivationType - name: str | None = None + label: str | None = None class DerivationRead(DerivationBase): used: BasicEntityRead generated: BasicEntityRead derivation_type: DerivationType - name: str | None = None + label: str | None = None diff --git a/scripts/export/build_database_archive.sh b/scripts/export/build_database_archive.sh index c06f79844..69c2a86a7 100755 --- a/scripts/export/build_database_archive.sh +++ b/scripts/export/build_database_archive.sh @@ -2,7 +2,7 @@ # Automatically generated, do not edit! set -euo pipefail SCRIPT_VERSION="1" -SCRIPT_DB_VERSION="92d1f0c96bfd" +SCRIPT_DB_VERSION="1caa2fc3d44e" echo "DB dump (version $SCRIPT_VERSION for db version $SCRIPT_DB_VERSION)" @@ -271,7 +271,7 @@ install -m 755 /dev/stdin "$WORK_DIR/load.sh" <<'EOF_LOAD_SCRIPT' # Automatically generated, do not edit! set -euo pipefail SCRIPT_VERSION="1" -SCRIPT_DB_VERSION="92d1f0c96bfd" +SCRIPT_DB_VERSION="1caa2fc3d44e" echo "DB load (version $SCRIPT_VERSION for db version $SCRIPT_DB_VERSION)" diff --git a/tests/test_derivation.py b/tests/test_derivation.py index a68733898..bfd704b6c 100644 --- a/tests/test_derivation.py +++ b/tests/test_derivation.py @@ -159,11 +159,11 @@ def test_create_one(client, derivation_type, root_circuit, circuit): "used": {"type": "circuit", "id": str(root_circuit.id)}, "generated": {"type": "circuit", "id": str(circuit.id)}, "derivation_type": derivation_type, - "name": None, + "label": None, } -def test_create_emodel_circuit_with_name(client, root_circuit, circuit): +def test_create_emodel_circuit_with_label(client, root_circuit, circuit): """Link an emodel (used) to a circuit (generated) with the SONATA model_template label. The ``used_id`` should be an EModel id in real usage; here we reuse a circuit fixture @@ -176,14 +176,14 @@ def test_create_emodel_circuit_with_name(client, root_circuit, circuit): "used_id": str(root_circuit.id), "generated_id": str(circuit.id), "derivation_type": "emodel_circuit", - "name": "hoc:cADpyr_L5TPC", + "label": "hoc:cADpyr_L5TPC", }, ).json() assert data == { "used": {"type": "circuit", "id": str(root_circuit.id)}, "generated": {"type": "circuit", "id": str(circuit.id)}, "derivation_type": "emodel_circuit", - "name": "hoc:cADpyr_L5TPC", + "label": "hoc:cADpyr_L5TPC", } From e3230802b0730d945041fe31eea701c9248b8888 Mon Sep 17 00:00:00 2001 From: Christoph Pokorny Date: Fri, 8 May 2026 09:56:29 +0200 Subject: [PATCH 3/7] Added new derivation type "circuit_customization" --- ...a_add_label_to_derivation_and_circuit_.py} | 24 ++++++++++++------- app/db/types.py | 5 ++++ scripts/export/build_database_archive.sh | 4 ++-- tests/test_derivation.py | 21 ++++++++++++++++ 4 files changed, 44 insertions(+), 10 deletions(-) rename alembic/versions/{20260508_063047_1caa2fc3d44e_add_label_to_derivation_and_emodel_circuit_type.py => 20260508_095241_b8ebf9a4e3da_add_label_to_derivation_and_circuit_.py} (67%) diff --git a/alembic/versions/20260508_063047_1caa2fc3d44e_add_label_to_derivation_and_emodel_circuit_type.py b/alembic/versions/20260508_095241_b8ebf9a4e3da_add_label_to_derivation_and_circuit_.py similarity index 67% rename from alembic/versions/20260508_063047_1caa2fc3d44e_add_label_to_derivation_and_emodel_circuit_type.py rename to alembic/versions/20260508_095241_b8ebf9a4e3da_add_label_to_derivation_and_circuit_.py index a3ea69d8c..d18ad48cb 100644 --- a/alembic/versions/20260508_063047_1caa2fc3d44e_add_label_to_derivation_and_emodel_circuit_type.py +++ b/alembic/versions/20260508_095241_b8ebf9a4e3da_add_label_to_derivation_and_circuit_.py @@ -1,8 +1,8 @@ -"""Add label column to derivation and emodel_circuit DerivationType +"""Add label to derivation and circuit_customization/emodel_circuit types -Revision ID: 1caa2fc3d44e +Revision ID: b8ebf9a4e3da Revises: c8cdf20bbb0d -Create Date: 2026-05-08 06:30:47.000000 +Create Date: 2026-05-08 09:52:41.175792 """ @@ -12,24 +12,29 @@ import sqlalchemy as sa from alembic_postgresql_enum import TableReference +from sqlalchemy import Text import app.db.types # revision identifiers, used by Alembic. -revision: str = "1caa2fc3d44e" +revision: str = "b8ebf9a4e3da" down_revision: Union[str, None] = "c8cdf20bbb0d" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: - # Add nullable label column to derivation table. + # ### commands auto generated by Alembic - please adjust! ### op.add_column("derivation", sa.Column("label", sa.String(), nullable=True)) - - # Extend DerivationType enum with emodel_circuit. op.sync_enum_values( enum_schema="public", enum_name="derivationtype", - new_values=["circuit_extraction", "circuit_rewiring", "emodel_circuit", "unspecified"], + new_values=[ + "circuit_customization", + "circuit_extraction", + "circuit_rewiring", + "emodel_circuit", + "unspecified", + ], affected_columns=[ TableReference( table_schema="public", table_name="derivation", column_name="derivation_type" @@ -37,9 +42,11 @@ def upgrade() -> None: ], enum_values_to_rename=[], ) + # ### end Alembic commands ### def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### op.sync_enum_values( enum_schema="public", enum_name="derivationtype", @@ -52,3 +59,4 @@ def downgrade() -> None: enum_values_to_rename=[], ) op.drop_column("derivation", "label") + # ### end Alembic commands ### diff --git a/app/db/types.py b/app/db/types.py index d78936e29..f4958c8fc 100644 --- a/app/db/types.py +++ b/app/db/types.py @@ -248,6 +248,10 @@ class DerivationType(StrEnum): """Represents the type of derivation relationship between two entities. Attributes: + circuit_customization: Indicates that a circuit entity was derived from another circuit by + customizing certain components. The optional ``label`` field specifies the type of + customization, such as ``synaptic_modification``, ``new_emodels``, + ``emodel_modification``, ``population_modification``. circuit_extraction: Indicates that the entity was derived by extracting a set of nodes from a circuit. circuit_rewiring: Indicates that the entity was derived by rewiring the connectivity of @@ -259,6 +263,7 @@ class DerivationType(StrEnum): unspecified: Indicates a derivation that does not require a specific type. """ + circuit_customization = auto() circuit_extraction = auto() circuit_rewiring = auto() emodel_circuit = auto() diff --git a/scripts/export/build_database_archive.sh b/scripts/export/build_database_archive.sh index 69c2a86a7..b52ecd9bc 100755 --- a/scripts/export/build_database_archive.sh +++ b/scripts/export/build_database_archive.sh @@ -2,7 +2,7 @@ # Automatically generated, do not edit! set -euo pipefail SCRIPT_VERSION="1" -SCRIPT_DB_VERSION="1caa2fc3d44e" +SCRIPT_DB_VERSION="b8ebf9a4e3da" echo "DB dump (version $SCRIPT_VERSION for db version $SCRIPT_DB_VERSION)" @@ -271,7 +271,7 @@ install -m 755 /dev/stdin "$WORK_DIR/load.sh" <<'EOF_LOAD_SCRIPT' # Automatically generated, do not edit! set -euo pipefail SCRIPT_VERSION="1" -SCRIPT_DB_VERSION="1caa2fc3d44e" +SCRIPT_DB_VERSION="b8ebf9a4e3da" echo "DB load (version $SCRIPT_VERSION for db version $SCRIPT_DB_VERSION)" diff --git a/tests/test_derivation.py b/tests/test_derivation.py index bfd704b6c..cda7fb0e0 100644 --- a/tests/test_derivation.py +++ b/tests/test_derivation.py @@ -139,6 +139,7 @@ def test_get_derived_from( @pytest.mark.parametrize( "derivation_type", [ + "circuit_customization", "circuit_extraction", "circuit_rewiring", "emodel_circuit", @@ -187,6 +188,26 @@ def test_create_emodel_circuit_with_label(client, root_circuit, circuit): } +def test_create_circuit_customization_with_label(client, root_circuit, circuit): + """Derive a circuit from another by customizing components, with a label for the type.""" + data = assert_request( + client.post, + url="/derivation", + json={ + "used_id": str(root_circuit.id), + "generated_id": str(circuit.id), + "derivation_type": "circuit_customization", + "label": "synaptic_modification", + }, + ).json() + assert data == { + "used": {"type": "circuit", "id": str(root_circuit.id)}, + "generated": {"type": "circuit", "id": str(circuit.id)}, + "derivation_type": "circuit_customization", + "label": "synaptic_modification", + } + + def test_create_invalid_data(client, root_circuit, circuit): # test that the derivation type is mandatory data = assert_request( From f55c643475b18afae1f810d0b710b54ce3ab2b91 Mon Sep 17 00:00:00 2001 From: Christoph Pokorny Date: Mon, 11 May 2026 09:30:57 +0200 Subject: [PATCH 4/7] Changed "new_emodels" to "emodel_addition" --- app/db/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/db/types.py b/app/db/types.py index f4958c8fc..117f049e6 100644 --- a/app/db/types.py +++ b/app/db/types.py @@ -250,7 +250,7 @@ class DerivationType(StrEnum): Attributes: circuit_customization: Indicates that a circuit entity was derived from another circuit by customizing certain components. The optional ``label`` field specifies the type of - customization, such as ``synaptic_modification``, ``new_emodels``, + customization, such as ``synaptic_modification``, ``emodel_addition``, ``emodel_modification``, ``population_modification``. circuit_extraction: Indicates that the entity was derived by extracting a set of nodes from a circuit. From 6119f2d1d29dacb97f558f8ce61180748de2fb2b Mon Sep 17 00:00:00 2001 From: Darshan Mandge <8694327+darshanmandge@users.noreply.github.com> Date: Mon, 11 May 2026 09:32:27 +0200 Subject: [PATCH 5/7] Update app/db/model.py Co-authored-by: Gianluca Ficarelli <26835404+GianlucaFicarelli@users.noreply.github.com> --- app/db/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/db/model.py b/app/db/model.py index 753de15da..9069a4d66 100644 --- a/app/db/model.py +++ b/app/db/model.py @@ -1671,7 +1671,7 @@ class Derivation(Base): used: Mapped["Entity"] = relationship(foreign_keys=[used_id]) generated: Mapped["Entity"] = relationship(foreign_keys=[generated_id], passive_deletes=True) derivation_type: Mapped[DerivationType] - label: Mapped[str | None] = mapped_column(nullable=True) + label: Mapped[str | None] class ScientificArtifactPublicationLink(Identifiable): From 917d319f7c040fb5d79036d72b3b0f2e7496e390 Mon Sep 17 00:00:00 2001 From: "@darshanmandge" Date: Mon, 11 May 2026 09:56:12 +0200 Subject: [PATCH 6/7] Update tests and Add label validation for DerivationCreate --- app/schemas/derivation.py | 40 ++++++++++++++++++++++++++++++++++++++- tests/test_derivation.py | 40 +++++++++++++++++++++++++++++++-------- 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/app/schemas/derivation.py b/app/schemas/derivation.py index df903f7fc..dabfc4d98 100644 --- a/app/schemas/derivation.py +++ b/app/schemas/derivation.py @@ -1,10 +1,24 @@ +import re import uuid -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, model_validator from app.db.types import DerivationType from app.schemas.base import BasicEntityRead +# Allowed label values per derivation type. +# - emodel_circuit: SONATA ``model_template`` entry, by convention ``hoc:``. +# - circuit_customization: type of customization applied to the source circuit. +_HOC_TEMPLATE_RE = re.compile(r"^hoc:[A-Za-z0-9_]+$") +_CIRCUIT_CUSTOMIZATION_LABELS = frozenset( + { + "synaptic_modification", + "emodel_addition", + "emodel_modification", + "population_modification", + } +) + class DerivationBase(BaseModel): model_config = ConfigDict(from_attributes=True) @@ -16,6 +30,30 @@ class DerivationCreate(DerivationBase): derivation_type: DerivationType label: str | None = None + @model_validator(mode="after") + def label_matches_derivation_type(self): + """Validate the label against the derivation type when provided.""" + if self.label is None: + return self + if self.derivation_type == DerivationType.emodel_circuit and not _HOC_TEMPLATE_RE.fullmatch( + self.label + ): + msg = ( + "label for derivation_type 'emodel_circuit' must match " + "'hoc:' (e.g. 'hoc:cADpyr_L5TPC')" + ) + raise ValueError(msg) + if ( + self.derivation_type == DerivationType.circuit_customization + and self.label not in _CIRCUIT_CUSTOMIZATION_LABELS + ): + msg = ( + "label for derivation_type 'circuit_customization' must be one of " + f"{sorted(_CIRCUIT_CUSTOMIZATION_LABELS)}" + ) + raise ValueError(msg) + return self + class DerivationRead(DerivationBase): used: BasicEntityRead diff --git a/tests/test_derivation.py b/tests/test_derivation.py index cda7fb0e0..e8eb626b6 100644 --- a/tests/test_derivation.py +++ b/tests/test_derivation.py @@ -164,24 +164,20 @@ def test_create_one(client, derivation_type, root_circuit, circuit): } -def test_create_emodel_circuit_with_label(client, root_circuit, circuit): - """Link an emodel (used) to a circuit (generated) with the SONATA model_template label. - - The ``used_id`` should be an EModel id in real usage; here we reuse a circuit fixture - since the generic /derivation endpoint does not constrain the entity types. - """ +def test_create_emodel_circuit_with_label(client, emodel_id, circuit): + """Link an emodel (used) to a circuit (generated) with the SONATA model_template label.""" data = assert_request( client.post, url="/derivation", json={ - "used_id": str(root_circuit.id), + "used_id": str(emodel_id), "generated_id": str(circuit.id), "derivation_type": "emodel_circuit", "label": "hoc:cADpyr_L5TPC", }, ).json() assert data == { - "used": {"type": "circuit", "id": str(root_circuit.id)}, + "used": {"type": "emodel", "id": str(emodel_id)}, "generated": {"type": "circuit", "id": str(circuit.id)}, "derivation_type": "emodel_circuit", "label": "hoc:cADpyr_L5TPC", @@ -208,6 +204,34 @@ def test_create_circuit_customization_with_label(client, root_circuit, circuit): } +@pytest.mark.parametrize( + ("derivation_type", "label"), + [ + ("emodel_circuit", "cADpyr_L5TPC"), # missing hoc: prefix + ("emodel_circuit", "hoc:"), # empty template name + ("emodel_circuit", "nml:cADpyr_L5TPC"), # wrong prefix + ("circuit_customization", "unknown_modification"), + ("circuit_customization", ""), + ], +) +def test_create_invalid_label_for_derivation_type( + client, emodel_id, circuit, derivation_type, label +): + used_id = emodel_id if derivation_type == "emodel_circuit" else str(circuit.id) + data = assert_request( + client.post, + url="/derivation", + json={ + "used_id": str(used_id), + "generated_id": str(circuit.id), + "derivation_type": derivation_type, + "label": label, + }, + expected_status_code=422, + ).json() + assert data["error_code"] == "INVALID_REQUEST" + + def test_create_invalid_data(client, root_circuit, circuit): # test that the derivation type is mandatory data = assert_request( From aabfb9599784f3bfcae083b3a57d0c98e9f56862 Mon Sep 17 00:00:00 2001 From: "@darshanmandge" Date: Mon, 11 May 2026 14:29:25 +0200 Subject: [PATCH 7/7] Validate used/generated entity types per derivation_type --- app/service/derivation.py | 65 +++++++++++++++++++ tests/conftest.py | 2 +- tests/test_derivation.py | 131 ++++++++++++++++++++++++++++---------- 3 files changed, 164 insertions(+), 34 deletions(-) diff --git a/app/service/derivation.py b/app/service/derivation.py index a6db6c46a..c7d904451 100644 --- a/app/service/derivation.py +++ b/app/service/derivation.py @@ -1,17 +1,21 @@ """Generic derivation service.""" import uuid +from http import HTTPStatus import sqlalchemy as sa from sqlalchemy import and_ from sqlalchemy.orm import aliased, joinedload, raiseload from app.db.model import Derivation, DerivationType, Entity +from app.db.types import EntityType from app.db.utils import ENTITY_TYPE_TO_CLASS, load_db_model_from_pydantic from app.dependencies.auth import AdminContextDep, UserContextDep, UserContextWithProjectIdDep from app.dependencies.common import DerivationQueryDep, PaginationQuery from app.dependencies.db import SessionDep from app.errors import ( + ApiError, + ApiErrorCode, ensure_authorized_references, ensure_foreign_keys_integrity, ensure_result, @@ -27,6 +31,66 @@ from app.schemas.types import ListResponse from app.utils.routers import entity_route_to_type +# Allowed entity types per derivation_type for (used, generated). +# A value of ``None`` means "no type-specific check" (any entity allowed). +# ``unspecified`` is intentionally unconstrained: it is currently used as a +# placeholder for emodel/memodel derivations and will be replaced by dedicated +# derivation types later. +_ALLOWED_ENTITY_TYPES: dict[ + DerivationType, tuple[frozenset[EntityType] | None, frozenset[EntityType] | None] +] = { + DerivationType.circuit_extraction: ( + frozenset({EntityType.circuit}), + frozenset({EntityType.circuit}), + ), + DerivationType.circuit_rewiring: ( + frozenset({EntityType.circuit}), + frozenset({EntityType.circuit}), + ), + DerivationType.circuit_customization: ( + frozenset({EntityType.circuit}), + frozenset({EntityType.circuit}), + ), + DerivationType.emodel_circuit: ( + frozenset({EntityType.emodel}), + frozenset({EntityType.circuit}), + ), + DerivationType.unspecified: (None, None), +} + + +def _validate_entity_types( + derivation_type: DerivationType, + used_entity: Entity, + generated_entity: Entity, +) -> None: + """Validate that the used/generated entity types match the derivation_type. + + Raises: + ApiError: with ``INVALID_REQUEST`` (HTTP 422) if the types do not match. + """ + allowed_used, allowed_generated = _ALLOWED_ENTITY_TYPES[derivation_type] + if allowed_used is not None and used_entity.type not in allowed_used: + raise ApiError( + message=( + f"derivation_type '{derivation_type.value}' requires used entity type " + f"to be one of {sorted(t.value for t in allowed_used)}, " + f"got '{used_entity.type.value}'" + ), + error_code=ApiErrorCode.INVALID_REQUEST, + http_status_code=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + if allowed_generated is not None and generated_entity.type not in allowed_generated: + raise ApiError( + message=( + f"derivation_type '{derivation_type.value}' requires generated entity type " + f"to be one of {sorted(t.value for t in allowed_generated)}, " + f"got '{generated_entity.type.value}'" + ), + error_code=ApiErrorCode.INVALID_REQUEST, + http_status_code=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + def _read_many( *, @@ -158,6 +222,7 @@ def create_one( json_model.generated_id, user_context.project_id, ) + _validate_entity_types(json_model.derivation_type, used_entity, generated_entity) db_model_class = Derivation db_model_instance = load_db_model_from_pydantic( json_model=json_model, diff --git a/tests/conftest.py b/tests/conftest.py index 85284c10d..f5c81c7be 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1289,7 +1289,7 @@ def root_circuit_json_data(brain_atlas_id, subject_id, brain_region_id, license_ "subject_id": str(subject_id), "build_category": "em_reconstruction", "authorized_project_id": PROJECT_ID, - "authorized_public": True, + "authorized_public": False, "created_by_id": str(person_id), "updated_by_id": str(person_id), "brain_region_id": str(brain_region_id), diff --git a/tests/test_derivation.py b/tests/test_derivation.py index e8eb626b6..fa014f1e2 100644 --- a/tests/test_derivation.py +++ b/tests/test_derivation.py @@ -1,6 +1,6 @@ import pytest -from app.db.model import Derivation +from app.db.model import Circuit, Derivation from app.errors import ApiErrorCode from app.schemas.api import ErrorResponse @@ -8,43 +8,66 @@ PROJECT_ID, UNRELATED_PROJECT_ID, add_all_db, + add_db, assert_request, assert_response, - create_electrical_cell_recording_id, ) +def _add_source_circuit(db, root_circuit_json_data, person_id, name): + return add_db( + db, + Circuit( + **root_circuit_json_data + | { + "name": name, + "created_by_id": person_id, + "updated_by_id": person_id, + "authorized_project_id": PROJECT_ID, + "authorized_public": True, + } + ), + ) + + def test_get_derived_from( - db, clients, emodel_id, public_emodel_id, electrical_cell_recording_json_data + db, + clients, + person_id, + public_root_circuit, + root_circuit, + root_circuit_json_data, ): - # create two emodels, one with derivations and one without - trace_ids = [ - create_electrical_cell_recording_id( - clients.user_1, json_data=electrical_cell_recording_json_data | {"name": f"name-{i}"} - ) + # Create source circuits (used) for the typed derivations. + # Source/target circuits use circuit_extraction / circuit_rewiring + # (circuit -> circuit), and a single unspecified derivation goes to a + # private target. Direct DB inserts via add_all_db bypass the create-time + # type validator, but the data still satisfies it for consistency. + source_ids = [ + _add_source_circuit(db, root_circuit_json_data, person_id, f"source-{i}").id for i in range(6) ] derivations = ( [ Derivation( - used_id=ecr_id, - generated_id=public_emodel_id, + used_id=src_id, + generated_id=public_root_circuit.id, derivation_type="circuit_extraction", ) - for ecr_id in trace_ids[:3] + for src_id in source_ids[:3] ] + [ Derivation( - used_id=ecr_id, - generated_id=public_emodel_id, + used_id=src_id, + generated_id=public_root_circuit.id, derivation_type="circuit_rewiring", ) - for ecr_id in trace_ids[3:5] + for src_id in source_ids[3:5] ] + [ Derivation( - used_id=trace_ids[5], - generated_id=emodel_id, # private + used_id=source_ids[5], + generated_id=root_circuit.id, # private derivation_type="unspecified", ) ] @@ -52,44 +75,44 @@ def test_get_derived_from( add_all_db(db, derivations) response = clients.user_1.get( - url=f"/emodel/{public_emodel_id}/derived-from", + url=f"/circuit/{public_root_circuit.id}/derived-from", params={"derivation_type": "circuit_extraction"}, ) assert_response(response, 200) data = response.json()["data"] assert len(data) == 3 - assert [d["id"] for d in data] == [str(id_) for id_ in reversed(trace_ids[:3])] - assert all(d["type"] == "electrical_cell_recording" for d in data) + assert [d["id"] for d in data] == [str(id_) for id_ in reversed(source_ids[:3])] + assert all(d["type"] == "circuit" for d in data) response = clients.user_1.get( - url=f"/emodel/{public_emodel_id}/derived-from", + url=f"/circuit/{public_root_circuit.id}/derived-from", params={"derivation_type": "circuit_rewiring"}, ) assert_response(response, 200) data = response.json()["data"] assert len(data) == 2 - assert [d["id"] for d in data] == [str(id_) for id_ in reversed(trace_ids[3:5])] - assert all(d["type"] == "electrical_cell_recording" for d in data) + assert [d["id"] for d in data] == [str(id_) for id_ in reversed(source_ids[3:5])] + assert all(d["type"] == "circuit" for d in data) response = clients.user_1.get( - url=f"/emodel/{emodel_id}/derived-from", + url=f"/circuit/{root_circuit.id}/derived-from", params={"derivation_type": "unspecified"}, ) assert_response(response, 200) data = response.json()["data"] assert len(data) == 1 - assert data[0]["id"] == str(trace_ids[5]) - assert data[0]["type"] == "electrical_cell_recording" + assert data[0]["id"] == str(source_ids[5]) + assert data[0]["type"] == "circuit" # Test error not derivation_type param - response = clients.user_1.get(url=f"/emodel/{public_emodel_id}/derived-from") + response = clients.user_1.get(url=f"/circuit/{public_root_circuit.id}/derived-from") assert_response(response, 422) error = ErrorResponse.model_validate(response.json()) assert error.error_code == ApiErrorCode.INVALID_REQUEST # Test error invalid derivation_type param response = clients.user_1.get( - url=f"/emodel/{public_emodel_id}/derived-from", + url=f"/circuit/{public_root_circuit.id}/derived-from", params={"derivation_type": "invalid_type"}, ) assert_response(response, 422) @@ -98,7 +121,7 @@ def test_get_derived_from( # Test empty result response = clients.user_1.get( - url=f"/emodel/{public_emodel_id}/derived-from", + url=f"/circuit/{public_root_circuit.id}/derived-from", params={"derivation_type": "unspecified"}, ) assert_response(response, 200) @@ -107,7 +130,7 @@ def test_get_derived_from( # Test private unreadable entity response = clients.user_2.get( - url=f"/emodel/{emodel_id}/derived-from", + url=f"/circuit/{root_circuit.id}/derived-from", params={"derivation_type": "unspecified"}, ) assert_response(response, 404) @@ -115,7 +138,7 @@ def test_get_derived_from( # Test non existing entity response = clients.user_2.get( - url="/emodel/00000000-0000-0000-0000-000000000000/derived-from", + url="/circuit/00000000-0000-0000-0000-000000000000/derived-from", params={"derivation_type": "unspecified"}, ) assert_response(response, 404) @@ -123,14 +146,14 @@ def test_get_derived_from( data = assert_request( clients.admin.get, - url=f"/admin/emodel/{public_emodel_id}/derived-from", + url=f"/admin/circuit/{public_root_circuit.id}/derived-from", params={"derivation_type": "circuit_extraction"}, ).json()["data"] assert len(data) == 3 data = assert_request( clients.admin.get, - url=f"/admin/emodel/{emodel_id}/derived-from", + url=f"/admin/circuit/{root_circuit.id}/derived-from", params={"derivation_type": "unspecified"}, ).json()["data"] assert len(data) == 1 @@ -142,11 +165,15 @@ def test_get_derived_from( "circuit_customization", "circuit_extraction", "circuit_rewiring", - "emodel_circuit", "unspecified", ], ) def test_create_one(client, derivation_type, root_circuit, circuit): + """Create derivations between two circuits (covered by validation rules). + + ``emodel_circuit`` is excluded here because it requires used=emodel, + generated=circuit; it is exercised in ``test_create_emodel_circuit_with_label``. + """ data = assert_request( client.post, url="/derivation", @@ -232,6 +259,44 @@ def test_create_invalid_label_for_derivation_type( assert data["error_code"] == "INVALID_REQUEST" +@pytest.mark.parametrize( + ("derivation_type", "use_emodel_as_used", "use_emodel_as_generated"), + [ + # circuit-only types reject emodel as the used entity + ("circuit_extraction", True, False), + ("circuit_rewiring", True, False), + ("circuit_customization", True, False), + # emodel_circuit rejects circuit as used + ("emodel_circuit", False, False), + # emodel_circuit rejects emodel as generated + ("emodel_circuit", True, True), + ], +) +def test_create_invalid_entity_types_for_derivation_type( + client, + emodel_id, + public_emodel_id, + root_circuit, + circuit, + derivation_type, + use_emodel_as_used, + use_emodel_as_generated, +): + used_id = str(emodel_id) if use_emodel_as_used else str(root_circuit.id) + generated_id = str(public_emodel_id) if use_emodel_as_generated else str(circuit.id) + data = assert_request( + client.post, + url="/derivation", + json={ + "used_id": used_id, + "generated_id": generated_id, + "derivation_type": derivation_type, + }, + expected_status_code=422, + ).json() + assert data["error_code"] == "INVALID_REQUEST" + + def test_create_invalid_data(client, root_circuit, circuit): # test that the derivation type is mandatory data = assert_request(