Removing Caching from the orchestrator (#938)

* Removing Caching from the orchestrator

* Remove more files

Author: pboers1988

docs/migration-guide/4.0.md

@@ -0,0 +1,16 @@
+---
+hide:
+  - navigation
+---
+# 4.0 Migration Guide
+
+In this document we describe the steps that should be taken to migrate from `orchestrator-core` v3 to v4.
+
+## About 4.0
+
+In this release we have removed the caching of domain models. Domain models will always be loaded from the database.
+
+## Steps
+
+To use 4.0 all workflows must have run to completion. The `cache_domain_models` step no longer is part of the codebase
+therfore `in flight` workflows will fail.

mkdocs.yml

@@ -192,6 +192,7 @@ nav:
           - Migration guides:
                 - 2.x: migration-guide/2.0.md
                 - 3.x: migration-guide/3.0.md
+                - 4.x: migration-guide/4.0.md
 
     - Workshops:
           #    - Beginner:

orchestrator/api/api_v1/endpoints/subscription_customer_descriptions.py

@@ -28,7 +28,6 @@
 from orchestrator.schemas import SubscriptionDescriptionBaseSchema, SubscriptionDescriptionSchema
 from orchestrator.schemas.subscription_descriptions import UpdateSubscriptionDescriptionSchema
 from orchestrator.utils.errors import StaleDataError
-from orchestrator.utils.redis import delete_from_redis
 
 router = APIRouter()
 
@@ -55,7 +54,6 @@ def delete_subscription_customer_descriptions(_id: UUID) -> None:
     description = db.session.get(SubscriptionCustomerDescriptionTable, _id)
     if description:
         delete(SubscriptionCustomerDescriptionTable, _id)
-        delete_from_redis(description.subscription_id)
 
 
 @router.get("/{_id}", response_model=SubscriptionDescriptionSchema)

orchestrator/domain/base.py

@@ -1294,27 +1294,13 @@ def from_other_lifecycle(
     # Some common functions shared by from_other_product and from_subscription
     @classmethod
     def _get_subscription(cls: type[S], subscription_id: UUID | UUIDstr) -> SubscriptionTable | None:
-        from orchestrator.settings import app_settings
 
         if not isinstance(subscription_id, UUID | UUIDstr):
             raise TypeError(f"subscription_id is of type {type(subscription_id)} instead of UUID | UUIDstr")
 
-        if app_settings.ENABLE_SUBSCRIPTION_MODEL_OPTIMIZATIONS:
-            # TODO #900 remove toggle and make this path the default
-            loaders = [
-                joinedload(SubscriptionTable.product).selectinload(ProductTable.fixed_inputs),
-            ]
-
-        else:
-            loaders = [
-                selectinload(SubscriptionTable.instances)
-                .joinedload(SubscriptionInstanceTable.product_block)
-                .selectinload(ProductBlockTable.resource_types),
-                selectinload(SubscriptionTable.instances).selectinload(
-                    SubscriptionInstanceTable.in_use_by_block_relations
-                ),
-                selectinload(SubscriptionTable.instances).selectinload(SubscriptionInstanceTable.values),
-            ]
+        loaders = [
+            joinedload(SubscriptionTable.product).selectinload(ProductTable.fixed_inputs),
+        ]
 
         return db.session.get(SubscriptionTable, subscription_id, options=loaders)
 
@@ -1394,7 +1380,6 @@ def from_other_product(
     def from_subscription(cls: type[S], subscription_id: UUID | UUIDstr) -> S:
         """Use a subscription_id to return required fields of an existing subscription."""
         from orchestrator.domain.context_cache import get_from_cache, store_in_cache
-        from orchestrator.settings import app_settings
 
         if cached_model := get_from_cache(subscription_id):
             return cast(S, cached_model)
@@ -1421,12 +1406,7 @@ def from_subscription(cls: type[S], subscription_id: UUID | UUIDstr) -> S:
 
         fixed_inputs = {fi.name: fi.value for fi in subscription.product.fixed_inputs}
 
-        instances: dict[str, Any]
-        if app_settings.ENABLE_SUBSCRIPTION_MODEL_OPTIMIZATIONS:
-            # TODO #900 remove toggle and make this path the default
-            instances = cls._load_root_instances(subscription_id)
-        else:
-            instances = cls._load_instances(subscription.instances, status, match_domain_attr=False)
+        instances = cls._load_root_instances(subscription_id)
 
         try:
             model = cls(

orchestrator/domain/customer_description.py

@@ -21,7 +21,6 @@
 from orchestrator.api.models import delete
 from orchestrator.db import SubscriptionCustomerDescriptionTable, db
 from orchestrator.utils.errors import StaleDataError
-from orchestrator.utils.redis import delete_subscription_from_redis
 from orchestrator.utils.validate_data_version import validate_data_version
 from orchestrator.websocket import invalidate_subscription_cache
 
@@ -38,7 +37,6 @@ def get_customer_description_by_customer_subscription(
     return db.session.scalars(stmt).one_or_none()
 
 
-@delete_subscription_from_redis()
 async def create_subscription_customer_description(
     customer_id: str, subscription_id: UUID, description: str
 ) -> SubscriptionCustomerDescriptionTable:
@@ -53,7 +51,6 @@ async def create_subscription_customer_description(
     return customer_description
 
 
-@delete_subscription_from_redis()
 async def update_subscription_customer_description(
     customer_description: SubscriptionCustomerDescriptionTable,
     description: str,
@@ -70,7 +67,6 @@ async def update_subscription_customer_description(
     return customer_description
 
 
-@delete_subscription_from_redis()
 async def delete_subscription_customer_description_by_customer_subscription(
     customer_id: str, subscription_id: UUID
 ) -> SubscriptionCustomerDescriptionTable | None:

orchestrator/graphql/mutations/customer_description.py

@@ -60,7 +60,7 @@ async def resolve_remove_customer_description(
     )
     if not description:
         return NotFoundError(message="Customer description not found")
-    return CustomerDescription.from_pydantic(description)
+    return CustomerDescription.from_pydantic(description)  # type: ignore
 
 
 @strawberry.type(description="Customer subscription description mutations")

orchestrator/services/processes.py

@@ -441,6 +441,7 @@ def create_process(
     }
 
     try:
+
         state = post_form(workflow.initial_input_form, initial_state, user_inputs)
     except FormValidationError:
         logger.exception("Validation errors", user_inputs=user_inputs)

orchestrator/services/subscriptions.py

@@ -605,7 +605,6 @@ def build_domain_model(subscription_model: SubscriptionModel) -> dict:
 
 def build_extended_domain_model(subscription_model: SubscriptionModel) -> dict:
     """Create a subscription dict from the SubscriptionModel with additional keys."""
-    from orchestrator.settings import app_settings
 
     stmt = select(SubscriptionCustomerDescriptionTable).where(
         SubscriptionCustomerDescriptionTable.subscription_id == subscription_model.subscription_id
@@ -615,27 +614,12 @@ def build_extended_domain_model(subscription_model: SubscriptionModel) -> dict:
     with cache_subscription_models():
         subscription = subscription_model.model_dump()
 
-    def inject_in_use_by_ids(path_to_block: str) -> None:
-        if not (in_use_by_subs := getattr_in(subscription_model, f"{path_to_block}.in_use_by")):
-            return
-
-        in_use_by_ids = [obj.in_use_by_id for obj in in_use_by_subs.col]
-        in_use_by_relations = [convert_to_in_use_by_relation(instance) for instance in in_use_by_subs]
-        update_in(subscription, f"{path_to_block}.in_use_by_ids", in_use_by_ids)
-        update_in(subscription, f"{path_to_block}.in_use_by_relations", in_use_by_relations)
-
-    if app_settings.ENABLE_SUBSCRIPTION_MODEL_OPTIMIZATIONS:
-        # TODO #900 remove toggle and make this path the default
-        # query all subscription instances and inject the in_use_by_ids/in_use_by_relations into the subscription dict.
-        instance_to_in_use_by = {
-            instance.subscription_instance_id: instance.in_use_by
-            for instance in eagerload_all_subscription_instances_only_inuseby(subscription_model.subscription_id)
-        }
-        inject_in_use_by_ids_v2(subscription, instance_to_in_use_by)
-    else:
-        # find all product blocks, check if they have in_use_by and inject the in_use_by_ids into the subscription dict.
-        for path in product_block_paths(subscription):
-            inject_in_use_by_ids(path)
+    # query all subscription instances and inject the in_use_by_ids/in_use_by_relations into the subscription dict.
+    instance_to_in_use_by = {
+        instance.subscription_instance_id: instance.in_use_by
+        for instance in eagerload_all_subscription_instances_only_inuseby(subscription_model.subscription_id)
+    }
+    inject_in_use_by_ids_v2(subscription, instance_to_in_use_by)
 
     subscription["customer_descriptions"] = customer_descriptions
 

orchestrator/settings.py

@@ -55,7 +55,6 @@ class AppSettings(BaseSettings):
     MAIL_PORT: int = 25
     MAIL_STARTTLS: bool = False
     CACHE_URI: RedisDsn = "redis://localhost:6379/0"  # type: ignore
-    CACHE_DOMAIN_MODELS: bool = False
     CACHE_HMAC_SECRET: str | None = None  # HMAC signing key, used when pickling results in the cache
     REDIS_RETRY_COUNT: NonNegativeInt = Field(
         2, description="Number of retries for redis connection errors/timeouts, 0 to disable"
@@ -87,9 +86,6 @@ class AppSettings(BaseSettings):
     ENABLE_GRAPHQL_STATS_EXTENSION: bool = False
     VALIDATE_OUT_OF_SYNC_SUBSCRIPTIONS: bool = False
     FILTER_BY_MODE: Literal["partial", "exact"] = "exact"
-    ENABLE_SUBSCRIPTION_MODEL_OPTIMIZATIONS: bool = (
-        True  # True=ignore cache + optimized DB queries; False=use cache + unoptimized DB queries. Remove in #900
-    )
 
 
 app_settings = AppSettings()

orchestrator/utils/get_subscription_dict.py

@@ -2,15 +2,11 @@
 
 from orchestrator.domain.base import SubscriptionModel
 from orchestrator.services.subscriptions import _generate_etag, build_domain_model, build_extended_domain_model
-from orchestrator.utils.redis import from_redis
 
 
 async def get_subscription_dict(subscription_id: UUID, inject_inuseby: bool = True) -> tuple[dict, str]:
     """Helper function to get subscription dict by uuid from db or cache."""
 
-    if cached_model := from_redis(subscription_id):
-        return cached_model  # type: ignore
-
     subscription_model = SubscriptionModel.from_subscription(subscription_id)
 
     if not inject_inuseby:

orchestrator/utils/redis.py

@@ -10,21 +10,17 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import functools
 from collections.abc import AsyncGenerator
 from contextlib import asynccontextmanager
-from os import getenv
-from typing import Any, Callable
+from typing import Any
 from uuid import UUID
 
 from anyio import CancelScope, get_cancelled_exc_class
 from redis.asyncio import Redis as AIORedis
 from redis.asyncio.client import Pipeline, PubSub
 from structlog import get_logger
 
-from orchestrator.services.subscriptions import _generate_etag
 from orchestrator.settings import app_settings
-from orchestrator.utils.json import PY_JSON_TYPES, json_dumps, json_loads
 from orchestrator.utils.redis_client import (
     create_redis_asyncio_client,
     create_redis_client,
@@ -37,52 +33,6 @@
 ONE_WEEK = 3600 * 24 * 7
 
 
-def caching_models_enabled() -> bool:
-    return getenv("AIOCACHE_DISABLE", "0") == "0" and app_settings.CACHE_DOMAIN_MODELS
-
-
-def to_redis(subscription: dict[str, Any]) -> str | None:
-    if caching_models_enabled():
-        logger.info("Setting cache for subscription", subscription=subscription["subscription_id"])
-        etag = _generate_etag(subscription)
-        cache.set(f"orchestrator:domain:{subscription['subscription_id']}", json_dumps(subscription), ex=ONE_WEEK)
-        cache.set(f"orchestrator:domain:etag:{subscription['subscription_id']}", etag, ex=ONE_WEEK)
-        return etag
-
-    logger.warning("Caching disabled, not caching subscription", subscription=subscription["subscription_id"])
-    return None
-
-
-def from_redis(subscription_id: UUID) -> tuple[PY_JSON_TYPES, str] | None:
-    log = logger.bind(subscription_id=subscription_id)
-
-    if app_settings.ENABLE_SUBSCRIPTION_MODEL_OPTIMIZATIONS:
-        # TODO #900 remove toggle and remove usage of this function in get_subscription_dict
-        log.info("Using SubscriptionModel optimization, not loading subscription from redis cache")
-        return None
-
-    if caching_models_enabled():
-        log.debug("Try to retrieve subscription from cache")
-        obj = cache.get(f"orchestrator:domain:{subscription_id}")
-        etag = cache.get(f"orchestrator:domain:etag:{subscription_id}")
-        if obj and etag:
-            log.info("Retrieved subscription from cache")
-            return json_loads(obj), etag.decode("utf-8")
-        log.info("Subscription not found in cache")
-        return None
-    log.warning("Caching disabled, not loading subscription")
-    return None
-
-
-def delete_from_redis(subscription_id: UUID) -> None:
-    if caching_models_enabled():
-        logger.info("Deleting subscription object from cache", subscription_id=subscription_id)
-        cache.delete(f"orchestrator:domain:{subscription_id}")
-        cache.delete(f"orchestrator:domain:etag:{subscription_id}")
-    else:
-        logger.warning("Caching disabled, not deleting subscription", subscription=subscription_id)
-
-
 def default_get_subscription_id(data: Any) -> UUID:
     if hasattr(data, "subscription_id"):
         return data.subscription_id
@@ -91,22 +41,6 @@ def default_get_subscription_id(data: Any) -> UUID:
     return data
 
 
-def delete_subscription_from_redis(
-    extract_fn: Callable[..., UUID] = default_get_subscription_id,
-) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
-    def _delete_subscription_from_redis(func: Callable[..., Any]) -> Callable[..., Any]:
-        @functools.wraps(func)
-        async def wrapper(*args: tuple, **kwargs: dict[str, Any]) -> Any:
-            data = await func(*args, **kwargs)
-            key = extract_fn(data)
-            delete_from_redis(key)
-            return data
-
-        return wrapper
-
-    return _delete_subscription_from_redis
-
-
 async def delete_keys_matching_pattern(_cache: AIORedis, pattern: str, chunksize: int = 5000) -> int:
     """Delete all keys matching the given pattern.
 

orchestrator/workflows/modify_note.py

@@ -13,11 +13,10 @@
 from orchestrator.db import db
 from orchestrator.forms import SubmitFormPage
 from orchestrator.services import subscriptions
-from orchestrator.settings import app_settings
 from orchestrator.targets import Target
 from orchestrator.utils.json import to_serializable
-from orchestrator.workflow import StepList, conditional, done, init, step, workflow
-from orchestrator.workflows.steps import cache_domain_models, store_process_subscription
+from orchestrator.workflow import StepList, done, init, step, workflow
+from orchestrator.workflows.steps import store_process_subscription
 from orchestrator.workflows.utils import wrap_modify_initial_input_form
 from pydantic_forms.types import FormGenerator, State, UUIDstr
 from pydantic_forms.validators import LongText
@@ -54,11 +53,4 @@ def store_subscription_note(subscription_id: UUIDstr, note: str) -> State:
 
 @workflow("Modify Note", initial_input_form=wrap_modify_initial_input_form(initial_input_form), target=Target.MODIFY)
 def modify_note() -> StepList:
-    push_subscriptions = conditional(lambda _: app_settings.CACHE_DOMAIN_MODELS)
-    return (
-        init
-        >> store_process_subscription(Target.MODIFY)
-        >> store_subscription_note
-        >> push_subscriptions(cache_domain_models)
-        >> done
-    )
+    return init >> store_process_subscription(Target.MODIFY) >> store_subscription_note >> done