diff --git a/docs/hub-cloud/08-test-multi-session.ipynb b/docs/hub-cloud/08-test-multi-session.ipynb index 7585dd066..facd2b540 100644 --- a/docs/hub-cloud/08-test-multi-session.ipynb +++ b/docs/hub-cloud/08-test-multi-session.ipynb @@ -54,6 +54,15 @@ "ln_setup.init(storage=\"./testsetup\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert ln_setup.settings.instance.slug == \"testuser1/testsetup\"" + ] + }, { "cell_type": "code", "execution_count": null, @@ -111,43 +120,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Let us try to init another instance in the same Python session:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from lamindb_setup._init_instance import CannotSwitchDefaultInstance\n", - "\n", - "with pytest.raises(CannotSwitchDefaultInstance):\n", - " ln_setup.init(storage=\"./testsetup2\")\n", - "with pytest.raises(CannotSwitchDefaultInstance):\n", - " ln_setup.connect(\"testsetup\")\n", - "with pytest.raises(RuntimeError):\n", - " ln_setup.migrate.create()\n", - "with pytest.raises(RuntimeError):\n", - " ln_setup.migrate.deploy()\n", - "\n", - "assert ln_setup.settings.instance.slug == \"testuser1/testsetup\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Reset `django` and connect to another instance:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ln_setup.core.django.reset_django()" + "Connect to another instance in the same process:" ] }, { diff --git a/lamindb_setup/_cache.py b/lamindb_setup/_cache.py index 636bdf4c1..384120809 100644 --- a/lamindb_setup/_cache.py +++ b/lamindb_setup/_cache.py @@ -8,6 +8,7 @@ from .core._settings_save import save_platform_user_storage_settings from .core._settings_store import system_settings_file +from .errors import CurrentInstanceNotConfigured def clear_cache_dir(): @@ -19,9 +20,8 @@ def clear_cache_dir(): "disconnecting the current instance to update the cloud sqlite database." ) disconnect() - except SystemExit as e: - if str(e) != "No instance connected! Call `lamin connect` or `lamin init`": - raise e + except CurrentInstanceNotConfigured: + pass cache_dir = settings.cache_dir if cache_dir.exists(): diff --git a/lamindb_setup/_check_setup.py b/lamindb_setup/_check_setup.py index a8b7ba65e..0250488e0 100644 --- a/lamindb_setup/_check_setup.py +++ b/lamindb_setup/_check_setup.py @@ -4,7 +4,9 @@ import importlib as il import inspect import os +from importlib.metadata import distributions from typing import TYPE_CHECKING +from uuid import UUID from lamin_utils import logger @@ -25,6 +27,7 @@ CURRENT_ISETTINGS: InstanceSettings | None = None +MODULE_CANDIDATES: set[str] | None = None IS_LOADING: bool = False @@ -42,7 +45,25 @@ def wrapper(*args, **kwargs): return wrapper -def _get_current_instance_settings() -> InstanceSettings | None: +def find_module_candidates(): + """Find all local packages that depend on lamindb.""" + global MODULE_CANDIDATES + if MODULE_CANDIDATES is not None: + return MODULE_CANDIDATES + all_dists = list(distributions()) + lamindb_deps = { + dist.metadata["Name"].lower() + for dist in all_dists + if dist.requires and any("lamindb" in req.lower() for req in dist.requires) + } + lamindb_deps.remove("lamindb") + MODULE_CANDIDATES = lamindb_deps + return lamindb_deps + + +def _get_current_instance_settings(from_module: str | None = None) -> InstanceSettings: + from .core._settings_instance import InstanceSettings + global CURRENT_ISETTINGS if CURRENT_ISETTINGS is not None: @@ -60,9 +81,17 @@ def _get_current_instance_settings() -> InstanceSettings | None: " command line: `lamin connect ` or `lamin init <...>`" ) raise e - return isettings else: - return None + module_candidates = find_module_candidates() + isettings = InstanceSettings( + id=UUID("00000000-0000-0000-0000-000000000000"), + owner="none", + name="none", + storage=None, + modules=",".join(module_candidates), + ) + CURRENT_ISETTINGS = isettings + return isettings def _normalize_module_name(module_name: str) -> str: @@ -132,12 +161,7 @@ def _check_instance_setup(from_module: str | None = None) -> bool: return True isettings = _get_current_instance_settings() if isettings is not None: - if ( - from_module is not None - and settings.auto_connect - and not django_lamin.IS_SETUP - and not IS_LOADING - ): + if from_module is not None and not django_lamin.IS_SETUP and not IS_LOADING: if from_module != "lamindb": _check_module_in_instance_modules(from_module, isettings) @@ -146,10 +170,13 @@ def _check_instance_setup(from_module: str | None = None) -> bool: il.reload(il.import_module(from_module)) else: django_lamin.setup_django(isettings) - logger.important(f"connected lamindb: {isettings.slug}") + if isettings.slug != "none/none": + logger.important(f"connected lamindb: {isettings.slug}") + else: + logger.warning("not connected, call: ln.connect('account/name')") return django_lamin.IS_SETUP else: - if from_module is not None and settings.auto_connect: + if from_module is not None: # the below enables users to auto-connect to an instance # simply by setting an environment variable, bypassing the # need of calling connect() manually diff --git a/lamindb_setup/_connect_instance.py b/lamindb_setup/_connect_instance.py index 9c2e01c81..65336ae51 100644 --- a/lamindb_setup/_connect_instance.py +++ b/lamindb_setup/_connect_instance.py @@ -2,17 +2,19 @@ import importlib import os +import sys from typing import TYPE_CHECKING, Any from uuid import UUID from lamin_utils import logger -from ._check_setup import _check_instance_setup, _get_current_instance_settings -from ._disconnect import disconnect -from ._init_instance import ( - MESSAGE_CANNOT_SWITCH_DEFAULT_INSTANCE, - load_from_isettings, +from ._check_setup import ( + _check_instance_setup, + _get_current_instance_settings, + find_module_candidates, ) +from ._disconnect import disconnect +from ._init_instance import load_from_isettings from ._silence_loggers import silence_loggers from .core._hub_core import connect_instance_hub from .core._hub_utils import ( @@ -188,6 +190,51 @@ def _connect_instance( return isettings +def reset_django_module_variables(): + import types + + from django.apps import apps + + app_names = {app.name for app in apps.get_app_configs()} + + for name, module in sys.modules.items(): + if ( + module is not None + and (not name.startswith("__") or name == "__main__") + and name not in sys.builtin_module_names + and not ( + hasattr(module, "__file__") + and module.__file__ + and any( + path in module.__file__ for path in ["/lib/python", "\\lib\\python"] + ) + ) + ): + try: + for k, v in vars(module).items(): + if ( + isinstance(v, types.ModuleType) + and not k.startswith("_") + and getattr(v, "__name__", None) in app_names + ): + if v.__name__ in sys.modules: + vars(module)[k] = sys.modules[v.__name__] + # Also reset classes from Django apps - but check if the class module starts with any app name + elif hasattr(v, "__module__") and getattr(v, "__module__", None): + class_module = v.__module__ + # Check if the class module starts with any of our app names + if any( + class_module.startswith(app_name) for app_name in app_names + ): + if class_module in sys.modules: + fresh_module = sys.modules[class_module] + attr_name = getattr(v, "__name__", k) + if hasattr(fresh_module, attr_name): + vars(module)[k] = getattr(fresh_module, attr_name) + except (AttributeError, TypeError): + continue + + def _connect_cli(instance: str) -> None: from lamindb_setup import settings as settings_ @@ -244,12 +291,7 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None: try: if instance is None: - isettings_or_none = _get_current_instance_settings() - if isettings_or_none is None: - raise ValueError( - "No instance was connected through the CLI, pass a value to `instance` or connect via the CLI." - ) - isettings = isettings_or_none + isettings = _get_current_instance_settings() else: owner, name = get_owner_name_from_identifier(instance) if _check_instance_setup() and not _test: @@ -260,9 +302,9 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None: logger.important(f"connected lamindb: {settings.instance.slug}") return None else: - raise CannotSwitchDefaultInstance( - MESSAGE_CANNOT_SWITCH_DEFAULT_INSTANCE - ) + from lamindb_setup.core.django import reset_django + + reset_django() elif ( _write_settings and settings._instance_exists @@ -315,7 +357,9 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None: load_from_isettings(isettings, user=_user, write_settings=_write_settings) if _reload_lamindb: importlib.reload(importlib.import_module("lamindb")) - logger.important(f"connected lamindb: {isettings.slug}") + reset_django_module_variables() + if isettings.slug != "none/none": + logger.important(f"connected lamindb: {isettings.slug}") except Exception as e: if isettings is not None: if _write_settings: diff --git a/lamindb_setup/_init_instance.py b/lamindb_setup/_init_instance.py index 25a70959a..dec97d788 100644 --- a/lamindb_setup/_init_instance.py +++ b/lamindb_setup/_init_instance.py @@ -215,16 +215,6 @@ def validate_init_args( return name_str, instance_id, instance_state, instance_slug -MESSAGE_CANNOT_SWITCH_DEFAULT_INSTANCE = """ -You cannot write to different instances in the same Python session. - -Do you want to read from another instance via `SQLRecord.using()`? For example: - -ln.Artifact.using("laminlabs/cellxgene").filter() - -Or do you want to switch off auto-connect via `lamin settings set auto-connect false`? -""" - DOC_STORAGE_ARG = "A local or remote folder (`'s3://...'` or `'gs://...'`). Defaults to current working directory." DOC_INSTANCE_NAME = ( "Instance name. If not passed, it will equal the folder name passed to `storage`." @@ -272,9 +262,12 @@ def init( from ._check_setup import _check_instance_setup if _check_instance_setup() and not _test: - raise CannotSwitchDefaultInstance(MESSAGE_CANNOT_SWITCH_DEFAULT_INSTANCE) + from lamindb_setup.core.django import reset_django + + reset_django() elif _write_settings: disconnect(mute=True) + from ._connect_instance import reset_django_module_variables from .core._hub_core import init_instance_hub name_str, instance_id, instance_state, _ = validate_init_args( @@ -287,8 +280,6 @@ def init( _user=_user, # will get from settings.user if _user is None ) if instance_state == "connected": - if _write_settings: - settings.auto_connect = True # we can also debate this switch here return None prevent_register_hub = is_local_db_url(db) if db is not None else False ssettings, _ = init_storage( @@ -344,9 +335,8 @@ def init( from ._schema_metadata import update_schema_in_hub update_schema_in_hub(access_token=access_token) - if _write_settings: - settings.auto_connect = True importlib.reload(importlib.import_module("lamindb")) + reset_django_module_variables() logger.important(f"initialized lamindb: {isettings.slug}") except Exception as e: from ._delete import delete_by_isettings diff --git a/lamindb_setup/core/_settings.py b/lamindb_setup/core/_settings.py index a3156e880..6e9f413ad 100644 --- a/lamindb_setup/core/_settings.py +++ b/lamindb_setup/core/_settings.py @@ -2,11 +2,14 @@ import os import sys +import warnings from typing import TYPE_CHECKING from lamin_utils import logger from platformdirs import user_cache_dir +from lamindb_setup.errors import CurrentInstanceNotConfigured + from ._settings_load import ( load_cache_path_from_settings, load_instance_settings, @@ -84,10 +87,13 @@ def auto_connect(self) -> bool: - in Python: `ln.setup.settings.auto_connect = True/False` - via the CLI: `lamin settings set auto-connect true/false` """ - return self._auto_connect_path.exists() + return True @auto_connect.setter def auto_connect(self, value: bool) -> None: + # logger.warning( + # "setting auto_connect to `False` no longer has an effect and the setting will likely be removed in the future; since lamindb 1.7, auto_connect `True` no longer clashes with connecting in a Python session", + # ) if value: self._auto_connect_path.touch() else: @@ -188,9 +194,10 @@ def is_connected(self) -> bool: If `True`, the current instance is connected, meaning that the db and other settings are properly configured for use. """ - from .django import IS_SETUP # always import to protect from assignment - - return IS_SETUP + if self._instance_exists: + return self.instance.slug != "none/none" + else: + return False @property def private_django_api(self) -> bool: @@ -253,7 +260,7 @@ def _instance_exists(self): self.instance # noqa return True # this is implicit logic that catches if no instance is loaded - except SystemExit: + except CurrentInstanceNotConfigured: return False @property diff --git a/lamindb_setup/core/_settings_instance.py b/lamindb_setup/core/_settings_instance.py index ce52fa563..0c3a156c4 100644 --- a/lamindb_setup/core/_settings_instance.py +++ b/lamindb_setup/core/_settings_instance.py @@ -58,7 +58,7 @@ def __init__( id: UUID, # instance id/uuid owner: str, # owner handle name: str, # instance name - storage: StorageSettings, # storage location + storage: StorageSettings | None, # storage location keep_artifacts_local: bool = False, # default to local storage uid: str | None = None, # instance uid/lnid db: str | None = None, # DB URI @@ -77,7 +77,7 @@ def __init__( self._owner: str = owner self._name: str = name self._uid: str | None = uid - self._storage: StorageSettings = storage + self._storage: StorageSettings | None = storage validate_db_arg(db) self._db: str | None = db self._schema_str: str | None = modules @@ -218,7 +218,7 @@ def storage(self) -> StorageSettings: For a cloud instance, this is cloud storage. For a local instance, this is a local directory. """ - return self._storage + return self._storage # type: ignore @property def local_storage(self) -> StorageSettings: @@ -415,6 +415,10 @@ def db(self) -> str: "It overwrites all db connections and is used instead of `instance.db`." ) if self._db is None: + from .django import IS_SETUP + + if self._storage is None and self.slug == "none/none": + return "sqlite:///:memory:" # here, we want the updated sqlite file # hence, we don't use self._sqlite_file_local() # error_no_origin=False because on instance init @@ -507,7 +511,7 @@ def _persist(self, write_to_disk: bool = True) -> None: write_to_disk: Save these instance settings to disk and overwrite the current instance settings file. """ - if write_to_disk: + if write_to_disk and self.slug != "none/none": assert self.name is not None filepath = self._get_settings_file() # persist under filepath for later reference diff --git a/lamindb_setup/core/_settings_load.py b/lamindb_setup/core/_settings_load.py index beed078d0..9b16451ed 100644 --- a/lamindb_setup/core/_settings_load.py +++ b/lamindb_setup/core/_settings_load.py @@ -9,7 +9,7 @@ from lamin_utils import logger from pydantic import ValidationError -from lamindb_setup.errors import SettingsEnvFileOutdated +from lamindb_setup.errors import CurrentInstanceNotConfigured, SettingsEnvFileOutdated from ._settings_instance import InstanceSettings from ._settings_storage import StorageSettings @@ -50,7 +50,7 @@ def load_instance_settings(instance_settings_file: Path | None = None): if instance_settings_file is None: instance_settings_file = current_instance_settings_file() if not instance_settings_file.exists(): - raise SystemExit("No instance connected! Call `lamin connect` or `lamin init`") + raise CurrentInstanceNotConfigured try: settings_store = InstanceSettingsStore(_env_file=instance_settings_file) except (ValidationError, TypeError) as error: diff --git a/lamindb_setup/core/django.py b/lamindb_setup/core/django.py index 84202edf1..5dfd1bed3 100644 --- a/lamindb_setup/core/django.py +++ b/lamindb_setup/core/django.py @@ -155,6 +155,7 @@ def setup_django( import dj_database_url import django + from django.apps import apps from django.conf import settings from django.core.management import call_command @@ -213,6 +214,9 @@ def setup_django( STATIC_URL="static/", ) settings.configure(**kwargs) + # this isn't needed the first time django.setup() is called, but for unknown reason it's needed the second time + # the first time, it already defaults to true + apps.apps_ready = True django.setup(set_prefix=False) # https://laminlabs.slack.com/archives/C04FPE8V01W/p1698239551460289 from django.db.backends.base.base import BaseDatabaseWrapper @@ -273,6 +277,7 @@ def reset_django(): app_names = {"django"} | {app.name for app in apps.get_app_configs()} apps.app_configs.clear() + apps.all_models.clear() apps.apps_ready = apps.models_ready = apps.ready = apps.loading = False apps.clear_cache() diff --git a/lamindb_setup/errors.py b/lamindb_setup/errors.py index dc92e1c57..527e20314 100644 --- a/lamindb_setup/errors.py +++ b/lamindb_setup/errors.py @@ -3,6 +3,7 @@ .. autosummary:: :toctree: . + CurrentInstanceNotConfigured InstanceNotSetupError ModuleWasntConfigured StorageAlreadyManaged @@ -27,6 +28,7 @@ def __init__(self, message: str | None = None): super().__init__(message) +# TODO: remove this exception sooner or later because we don't have a need for it anymore class InstanceNotSetupError(DefaultMessageException): default_message = """\ To use lamindb, you need to connect to an instance. @@ -37,6 +39,14 @@ class InstanceNotSetupError(DefaultMessageException): """ +class CurrentInstanceNotConfigured(DefaultMessageException): + default_message = """\ +No instance is connected! Call +- CLI: lamin connect / lamin init +- Python: ln.connect() / ln.setup.init() +- R: ln$connect() / ln$setup$init()""" + + MODULE_WASNT_CONFIGURED_MESSAGE_TEMPLATE = ( "'{}' wasn't configured for this instance -- " "if you want it, go to your instance settings page and add it under 'schema modules' (or ask an admin to do so)" diff --git a/tests/hub-cloud/test_connect_instance.py b/tests/hub-cloud/test_connect_instance.py index 7660553eb..6d20b15f4 100644 --- a/tests/hub-cloud/test_connect_instance.py +++ b/tests/hub-cloud/test_connect_instance.py @@ -13,12 +13,8 @@ def test_connect_pass_none(): - with pytest.raises(ValueError) as err: - ln_setup.connect(_test=True) - assert ( - err.exconly() - == "ValueError: No instance was connected through the CLI, pass a value to `instance` or connect via the CLI." - ) + # this doesn't log anything and connects to the mock instance + ln_setup.connect(_test=True) # do not call hub if the owner is set to anonymous