🚸 Set up Django on every import of lamindb via a "mock instance"

docs/hub-cloud/08-test-multi-session.ipynb

@@ -54,6 +54,15 @@
     "ln_setup.init(storage=\"./testsetup\")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert ln_setup.settings.instance.slug == \"testuser1/testsetup\""
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -111,43 +120,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Let us try to init another instance in the same Python session:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from lamindb_setup._init_instance import CannotSwitchDefaultInstance\n",
-    "\n",
-    "with pytest.raises(CannotSwitchDefaultInstance):\n",
-    "    ln_setup.init(storage=\"./testsetup2\")\n",
-    "with pytest.raises(CannotSwitchDefaultInstance):\n",
-    "    ln_setup.connect(\"testsetup\")\n",
-    "with pytest.raises(RuntimeError):\n",
-    "    ln_setup.migrate.create()\n",
-    "with pytest.raises(RuntimeError):\n",
-    "    ln_setup.migrate.deploy()\n",
-    "\n",
-    "assert ln_setup.settings.instance.slug == \"testuser1/testsetup\""
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Reset `django` and connect to another instance:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "ln_setup.core.django.reset_django()"
+    "Connect to another instance in the same process:"
    ]
   },
   {

lamindb_setup/_cache.py

@@ -8,6 +8,7 @@
 
 from .core._settings_save import save_platform_user_storage_settings
 from .core._settings_store import system_settings_file
+from .errors import CurrentInstanceNotConfigured
 
 
 def clear_cache_dir():
@@ -19,9 +20,8 @@ def clear_cache_dir():
                 "disconnecting the current instance to update the cloud sqlite database."
             )
             disconnect()
-    except SystemExit as e:
-        if str(e) != "No instance connected! Call `lamin connect` or `lamin init`":
-            raise e
+    except CurrentInstanceNotConfigured:
+        pass
 
     cache_dir = settings.cache_dir
     if cache_dir.exists():

lamindb_setup/_check_setup.py

@@ -4,7 +4,9 @@
 import importlib as il
 import inspect
 import os
+from importlib.metadata import distributions
 from typing import TYPE_CHECKING
+from uuid import UUID
 
 from lamin_utils import logger
 
@@ -25,6 +27,7 @@
 
 
 CURRENT_ISETTINGS: InstanceSettings | None = None
+MODULE_CANDIDATES: set[str] | None = None
 IS_LOADING: bool = False
 
 
@@ -42,7 +45,25 @@ def wrapper(*args, **kwargs):
     return wrapper
 
 
-def _get_current_instance_settings() -> InstanceSettings | None:
+def find_module_candidates():
+    """Find all local packages that depend on lamindb."""
+    global MODULE_CANDIDATES
+    if MODULE_CANDIDATES is not None:
+        return MODULE_CANDIDATES
+    all_dists = list(distributions())
+    lamindb_deps = {
+        dist.metadata["Name"].lower()
+        for dist in all_dists
+        if dist.requires and any("lamindb" in req.lower() for req in dist.requires)
+    }
+    lamindb_deps.remove("lamindb")
+    MODULE_CANDIDATES = lamindb_deps
+    return lamindb_deps
+
+
+def _get_current_instance_settings(from_module: str | None = None) -> InstanceSettings:
+    from .core._settings_instance import InstanceSettings
+
     global CURRENT_ISETTINGS
 
     if CURRENT_ISETTINGS is not None:
@@ -60,9 +81,17 @@ def _get_current_instance_settings() -> InstanceSettings | None:
                 " command line: `lamin connect <instance>` or `lamin init <...>`"
             )
             raise e
-        return isettings
     else:
-        return None
+        module_candidates = find_module_candidates()
+        isettings = InstanceSettings(
+            id=UUID("00000000-0000-0000-0000-000000000000"),
+            owner="none",
+            name="none",
+            storage=None,
+            modules=",".join(module_candidates),
+        )
+    CURRENT_ISETTINGS = isettings
+    return isettings
 
 
 def _normalize_module_name(module_name: str) -> str:
@@ -132,12 +161,7 @@ def _check_instance_setup(from_module: str | None = None) -> bool:
         return True
     isettings = _get_current_instance_settings()
     if isettings is not None:
-        if (
-            from_module is not None
-            and settings.auto_connect
-            and not django_lamin.IS_SETUP
-            and not IS_LOADING
-        ):
+        if from_module is not None and not django_lamin.IS_SETUP and not IS_LOADING:
             if from_module != "lamindb":
                 _check_module_in_instance_modules(from_module, isettings)
 
@@ -146,10 +170,13 @@ def _check_instance_setup(from_module: str | None = None) -> bool:
                 il.reload(il.import_module(from_module))
             else:
                 django_lamin.setup_django(isettings)
-                logger.important(f"connected lamindb: {isettings.slug}")
+                if isettings.slug != "none/none":
+                    logger.important(f"connected lamindb: {isettings.slug}")
+                else:
+                    logger.warning("not connected, call: ln.connect('account/name')")
         return django_lamin.IS_SETUP
     else:
-        if from_module is not None and settings.auto_connect:
+        if from_module is not None:
             # the below enables users to auto-connect to an instance
             # simply by setting an environment variable, bypassing the
             # need of calling connect() manually

lamindb_setup/_connect_instance.py

@@ -2,17 +2,19 @@
 
 import importlib
 import os
+import sys
 from typing import TYPE_CHECKING, Any
 from uuid import UUID
 
 from lamin_utils import logger
 
-from ._check_setup import _check_instance_setup, _get_current_instance_settings
-from ._disconnect import disconnect
-from ._init_instance import (
-    MESSAGE_CANNOT_SWITCH_DEFAULT_INSTANCE,
-    load_from_isettings,
+from ._check_setup import (
+    _check_instance_setup,
+    _get_current_instance_settings,
+    find_module_candidates,
 )
+from ._disconnect import disconnect
+from ._init_instance import load_from_isettings
 from ._silence_loggers import silence_loggers
 from .core._hub_core import connect_instance_hub
 from .core._hub_utils import (
@@ -188,6 +190,51 @@ def _connect_instance(
     return isettings
 
 
+def reset_django_module_variables():
+    import types
+
+    from django.apps import apps
+
+    app_names = {app.name for app in apps.get_app_configs()}
+
+    for name, module in sys.modules.items():
+        if (
+            module is not None
+            and (not name.startswith("__") or name == "__main__")
+            and name not in sys.builtin_module_names
+            and not (
+                hasattr(module, "__file__")
+                and module.__file__
+                and any(
+                    path in module.__file__ for path in ["/lib/python", "\\lib\\python"]
+                )
+            )
+        ):
+            try:
+                for k, v in vars(module).items():
+                    if (
+                        isinstance(v, types.ModuleType)
+                        and not k.startswith("_")
+                        and getattr(v, "__name__", None) in app_names
+                    ):
+                        if v.__name__ in sys.modules:
+                            vars(module)[k] = sys.modules[v.__name__]
+                    # Also reset classes from Django apps - but check if the class module starts with any app name
+                    elif hasattr(v, "__module__") and getattr(v, "__module__", None):
+                        class_module = v.__module__
+                        # Check if the class module starts with any of our app names
+                        if any(
+                            class_module.startswith(app_name) for app_name in app_names
+                        ):
+                            if class_module in sys.modules:
+                                fresh_module = sys.modules[class_module]
+                                attr_name = getattr(v, "__name__", k)
+                                if hasattr(fresh_module, attr_name):
+                                    vars(module)[k] = getattr(fresh_module, attr_name)
+            except (AttributeError, TypeError):
+                continue
+
+
 def _connect_cli(instance: str) -> None:
     from lamindb_setup import settings as settings_
 
@@ -244,12 +291,7 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
 
     try:
         if instance is None:
-            isettings_or_none = _get_current_instance_settings()
-            if isettings_or_none is None:
-                raise ValueError(
-                    "No instance was connected through the CLI, pass a value to `instance` or connect via the CLI."
-                )
-            isettings = isettings_or_none
+            isettings = _get_current_instance_settings()
         else:
             owner, name = get_owner_name_from_identifier(instance)
             if _check_instance_setup() and not _test:
@@ -260,9 +302,9 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
                     logger.important(f"connected lamindb: {settings.instance.slug}")
                     return None
                 else:
-                    raise CannotSwitchDefaultInstance(
-                        MESSAGE_CANNOT_SWITCH_DEFAULT_INSTANCE
-                    )
+                    from lamindb_setup.core.django import reset_django
+
+                    reset_django()
             elif (
                 _write_settings
                 and settings._instance_exists
@@ -315,7 +357,9 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
         load_from_isettings(isettings, user=_user, write_settings=_write_settings)
         if _reload_lamindb:
             importlib.reload(importlib.import_module("lamindb"))
-        logger.important(f"connected lamindb: {isettings.slug}")
+            reset_django_module_variables()
+        if isettings.slug != "none/none":
+            logger.important(f"connected lamindb: {isettings.slug}")
     except Exception as e:
         if isettings is not None:
             if _write_settings:

lamindb_setup/_init_instance.py

@@ -215,16 +215,6 @@ def validate_init_args(
     return name_str, instance_id, instance_state, instance_slug
 
 
-MESSAGE_CANNOT_SWITCH_DEFAULT_INSTANCE = """
-You cannot write to different instances in the same Python session.
-
-Do you want to read from another instance via `SQLRecord.using()`? For example:
-
-ln.Artifact.using("laminlabs/cellxgene").filter()
-
-Or do you want to switch off auto-connect via `lamin settings set auto-connect false`?
-"""
-
 DOC_STORAGE_ARG = "A local or remote folder (`'s3://...'` or `'gs://...'`). Defaults to current working directory."
 DOC_INSTANCE_NAME = (
     "Instance name. If not passed, it will equal the folder name passed to `storage`."
@@ -272,9 +262,12 @@ def init(
         from ._check_setup import _check_instance_setup
 
         if _check_instance_setup() and not _test:
-            raise CannotSwitchDefaultInstance(MESSAGE_CANNOT_SWITCH_DEFAULT_INSTANCE)
+            from lamindb_setup.core.django import reset_django
+
+            reset_django()
         elif _write_settings:
             disconnect(mute=True)
+        from ._connect_instance import reset_django_module_variables
         from .core._hub_core import init_instance_hub
 
         name_str, instance_id, instance_state, _ = validate_init_args(
@@ -287,8 +280,6 @@ def init(
             _user=_user,  # will get from settings.user if _user is None
         )
         if instance_state == "connected":
-            if _write_settings:
-                settings.auto_connect = True  # we can also debate this switch here
             return None
         prevent_register_hub = is_local_db_url(db) if db is not None else False
         ssettings, _ = init_storage(
@@ -344,9 +335,8 @@ def init(
             from ._schema_metadata import update_schema_in_hub
 
             update_schema_in_hub(access_token=access_token)
-        if _write_settings:
-            settings.auto_connect = True
         importlib.reload(importlib.import_module("lamindb"))
+        reset_django_module_variables()
         logger.important(f"initialized lamindb: {isettings.slug}")
     except Exception as e:
         from ._delete import delete_by_isettings

lamindb_setup/core/_settings.py

@@ -2,11 +2,14 @@
 
 import os
 import sys
+import warnings
 from typing import TYPE_CHECKING
 
 from lamin_utils import logger
 from platformdirs import user_cache_dir
 
+from lamindb_setup.errors import CurrentInstanceNotConfigured
+
 from ._settings_load import (
     load_cache_path_from_settings,
     load_instance_settings,
@@ -84,10 +87,13 @@ def auto_connect(self) -> bool:
         - in Python: `ln.setup.settings.auto_connect = True/False`
         - via the CLI: `lamin settings set auto-connect true/false`
         """
-        return self._auto_connect_path.exists()
+        return True
 
     @auto_connect.setter
     def auto_connect(self, value: bool) -> None:
+        logger.warning(
+            "setting auto_connect to `False` no longer has an effect and the setting will likely be removed in the future; since lamindb 1.7, auto_connect `True` no longer clashes with connecting in a Python session",
+        )
         if value:
             self._auto_connect_path.touch()
         else:
@@ -188,9 +194,10 @@ def is_connected(self) -> bool:
         If `True`, the current instance is connected, meaning that the db and other settings
         are properly configured for use.
         """
-        from .django import IS_SETUP  # always import to protect from assignment
-
-        return IS_SETUP
+        if self._instance_exists:
+            return self.instance.slug != "none/none"
+        else:
+            return False
 
     @property
     def private_django_api(self) -> bool:
@@ -253,7 +260,7 @@ def _instance_exists(self):
             self.instance  # noqa
             return True
         # this is implicit logic that catches if no instance is loaded
-        except SystemExit:
+        except CurrentInstanceNotConfigured:
             return False
 
     @property