Database Layer Overhaul inclduing Thread Safety, Connection Management & Configuration

dementor/assets/Dementor.toml

@@ -204,17 +204,39 @@ UPnP = true
 # =============================================================================
 [DB]
 
-# If true, allows duplicate credentials to be stored. If false, only unique
-# credentials will be stored and printed once.
-# The default value is: true
-
-DuplicateCreds = true
+# --- Url (advanced) ----------------------------------------------------------
+# Full SQLAlchemy database URL.  When set, Path is ignored.
+# Use this to connect to an external database server for multi-session or
+# shared access.  Leave empty (the default) to use the SQLite Path below.
+#
+# Examples:
+#   Url = "mysql+pymysql://user:pass@127.0.0.1/dementor"       # MySQL / MariaDB
+#   Url = "postgresql+psycopg2://user:pass@127.0.0.1/dementor"  # PostgreSQL
+#
+# Url =
 
-# Dialect = "sqlite"
-# Driver = "pysqlite"
-# Url = "sqlite:///:memory:"
+# --- Path (default backend) --------------------------------------------------
+# Path to the SQLite database file.  Only used when Url is empty.
+#
+#   Relative paths are resolved from the workspace directory:
+#     Path = "Dementor.db"           (default -- file in workspace)
+#     Path = "data/captures.db"      (subfolder, created automatically)
+#
+#   Absolute paths are used as-is:
+#     Path = "/opt/dementor/creds.db"
+#
+#   In-memory database (fast, but all data is lost when Dementor exits;
+#   the TUI can still query captured creds while running):
+#     Path = ":memory:"
+#
 # Path = "Dementor.db"
 
+# --- DuplicateCreds -----------------------------------------------------------
+# When true, every captured hash is stored even if an identical credential
+# (same domain + username + type + protocol) was seen before.  When false,
+# only the first capture is kept and repeats are silently skipped.
+DuplicateCreds = true
+
 # =============================================================================
 # mDNS
 # =============================================================================

dementor/db/__init__.py

@@ -18,18 +18,36 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+"""Dementor database package -- constants, helpers, and ORM models.
+
+Provides the :class:`~dementor.db.model.DementorDB` wrapper for thread-safe
+credential storage, the :class:`~dementor.db.connector.DatabaseConfig` for
+``[DB]`` TOML configuration, and engine initialization via
+:func:`~dementor.db.connector.create_db`.
+"""
+
+__all__ = ["CLEARTEXT", "HOST_INFO", "NO_USER", "normalize_client_address"]
+
 # --------------------------------------------------------------------------- #
 # Public constants
 # --------------------------------------------------------------------------- #
-_CLEARTEXT = "Cleartext"
+CLEARTEXT = "Cleartext"
 """Constant indicating plaintext credentials (as opposed to hashes)."""
 
-_NO_USER = "<missing-user>"
+NO_USER = "<missing-user>"
 """Placeholder string used when username is absent or invalid in credential logging."""
 
-_HOST_INFO = "_host_info"
+HOST_INFO = "_host_info"
 """Key used in extras dict to store host information for credential logging."""
 
+# Backward-compatible aliases so existing imports like
+#   from dementor.db import _CLEARTEXT
+# keep working without a mass-rename across all protocol files.
+# New code should use the unprefixed names above.
+_CLEARTEXT = CLEARTEXT
+_NO_USER = NO_USER
+_HOST_INFO = HOST_INFO
+
 
 def normalize_client_address(client: str) -> str:
     """Normalize IPv6-mapped IPv4 addresses by stripping IPv6 prefix.

dementor/db/connector.py

@@ -18,70 +18,56 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 # pyright: reportUninitializedInstanceVariable=false
+"""Database engine initialization and configuration.
+
+Reads the ``[DB]`` TOML section via :class:`DatabaseConfig`, builds a
+SQLAlchemy :class:`~sqlalchemy.engine.Engine` with backend-specific pool
+settings, and exposes :func:`create_db` as the single entry point used
+by :func:`~dementor.standalone.serve` at startup.
+"""
+
 import typing
+from typing import Any
 
 from sqlalchemy import Engine, create_engine
+from sqlalchemy.pool import StaticPool
 
 from dementor.config.session import SessionConfig
-from dementor.db.model import DementorDB, ModelBase
+from dementor.db.model import DementorDB
 from dementor.log.logger import dm_logger
 from dementor.config.toml import TomlConfig, Attribute as A
 
 
 class DatabaseConfig(TomlConfig):
-    """
-    Configuration mapping for the ``[DB]`` TOML section.
+    """Configuration mapping for the ``[DB]`` TOML section.
 
-    The attributes correspond to the most common SQLAlchemy connection
-    parameters.  All fields are optional - sensible defaults are applied
-    when a key is missing.
+    Users set EITHER ``Url`` (a full SQLAlchemy DSN for any backend,
+    e.g. ``mysql+pymysql://user:pass@host/db``) OR ``Path`` (a file
+    path for the default SQLite backend, e.g. ``Dementor.db``).
+
+    When ``Url`` is omitted, ``Path`` is resolved relative to the
+    session workspace and wrapped into a ``sqlite+pysqlite://`` URL.
     """
 
     _section_: typing.ClassVar[str] = "DB"
     _fields_: typing.ClassVar[list[A]] = [
-        A("db_raw_path", "Url", None),
+        A("db_url", "Url", None),
         A("db_path", "Path", "Dementor.db"),
         A("db_duplicate_creds", "DuplicateCreds", False),
-        A("db_dialect", "Dialect", None),
-        A("db_driver", "Driver", None),
     ]
 
     if typing.TYPE_CHECKING:  # pragma: no cover - only for static analysis
-        db_raw_path: str | None
+        db_url: str | None
         db_path: str
         db_duplicate_creds: bool
-        db_dialect: str | None
-        db_driver: str | None
-
-
-def init_dementor_db(session: SessionConfig) -> Engine | None:
-    """
-    Initialise the database engine and create all tables.
-
-    :param session: The active :class:`~dementor.config.session.SessionConfig`
-        containing the ``db_config`` attribute.
-    :type session: SessionConfig
-    :return: The created SQLAlchemy ``Engine`` or ``None`` if an error
-        prevented initialisation.
-    :rtype: Engine | None
-    """
-    engine = init_engine(session)
-    if engine is not None:
-        ModelBase.metadata.create_all(engine)
-    return engine
 
 
 def init_engine(session: SessionConfig) -> Engine | None:
-    """
-    Build a SQLAlchemy ``Engine`` from a :class:`DatabaseConfig`.
-
-    The logic follows the rules laid out in the SQLAlchemy documentation
-    (see https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls).
+    """Build a SQLAlchemy ``Engine`` from a :class:`DatabaseConfig`.
 
-    * If ``db_raw_path`` is supplied it is used verbatim.
-    * Otherwise a URL is composed from ``dialect``, ``driver`` and ``path``.
-      For SQLite the path is resolved relative to the session's
-      ``resolve_path`` helper; missing directories are created on the fly.
+    * If ``db_url`` (TOML ``Url``) is supplied it is used verbatim.
+    * Otherwise ``db_path`` (TOML ``Path``) is resolved relative to the
+      session workspace and wrapped into a ``sqlite+pysqlite://`` URL.
 
     Sensitive information (user/password) is hidden in the debug output.
 
@@ -91,18 +77,19 @@ def init_engine(session: SessionConfig) -> Engine | None:
     :rtype: Engine | None
     """
     # --------------------------------------------------------------- #
-    # 1.  Resolve "raw" URL - either provided by the user or built.
+    # 1.  Resolve URL -- either user-supplied DSN or built from Path.
     # --------------------------------------------------------------- #
-    raw_path = session.db_config.db_raw_path
+    raw_path = session.db_config.db_url
     if raw_path is None:
-        # Build the URL manually when the user didn't provide a full DSN.
-        dialect = session.db_config.db_dialect or "sqlite"
-        driver = session.db_config.db_driver or "pysqlite"
+        # No Url configured -- use the SQLite Path default.
+        dialect = "sqlite"
+        driver = "pysqlite"
         path = session.db_config.db_path
         if not path:
             return dm_logger.error("Database path not specified!")
-        # :memory: is a special SQLite in-memory database.
-        if dialect == "sqlite" and path != ":memory:":
+        if path == ":memory:":
+            path = "/:memory:"
+        else:
             real_path = session.resolve_path(path)
             if not real_path.parent.exists():
                 dm_logger.debug(f"Creating database directory {real_path.parent}")
@@ -118,26 +105,98 @@ def init_engine(session: SessionConfig) -> Engine | None:
             dialect = sql_type
             driver = "<default>"
 
+    # --------------------------------------------------------------- #
+    # 2.  Mask credentials in the debug log output.
+    # --------------------------------------------------------------- #
+    # For non-SQLite URLs like mysql+pymysql://user:pass@host/db,
+    # replace the user:pass portion with stars so passwords don't
+    # appear in log files.
     if dialect != "sqlite":
         first_element, *parts = path.split("/")
         if "@" in first_element:
-            # keep only the “host:port” part, replace user:pass with stars
             first_element = first_element.split("@")[1]
             path = "***:***@" + "/".join([first_element, *parts])
 
     dm_logger.debug("Using database [%s:%s] at: %s", dialect, driver, path)
-    return create_engine(raw_path, isolation_level="AUTOCOMMIT", future=True)
+
+    # --------------------------------------------------------------- #
+    # 3.  Build the engine with backend-specific pool settings.
+    # --------------------------------------------------------------- #
+    # All backends use AUTOCOMMIT -- Dementor does individual INSERT/SELECT
+    # operations, not multi-statement transactions.
+    #
+    # pool_reset_on_return=None: the pool's default is to ROLLBACK on
+    # every connection checkin, which is wasted work under AUTOCOMMIT.
+    #
+    # skip_autocommit_rollback=True: tells the dialect itself not to
+    # emit ROLLBACK either (SQLAlchemy 2.0.43+).  Together these two
+    # settings eliminate every unnecessary ROLLBACK round-trip.
+    common: dict[str, Any] = {
+        "isolation_level": "AUTOCOMMIT",
+        "pool_reset_on_return": None,
+        "skip_autocommit_rollback": True,
+    }
+
+    # Three pool strategies, one per backend constraint:
+    #
+    # :memory: SQLite  -> StaticPool  (DB exists only inside one connection;
+    #                     a second connection = empty DB.  DementorDB.lock
+    #                     serializes all access to that one connection.)
+    #
+    # File SQLite      -> QueuePool   (SQLAlchemy 2.0 default for file SQLite.
+    #                     Each thread checks out its own connection;
+    #                     _release() returns it after each operation.)
+    #
+    # MySQL/PostgreSQL -> QueuePool   (Connection reuse avoids the ~10-50ms
+    #                     TCP+auth overhead of opening a new connection per
+    #                     query.  LIFO keeps idle connections at the front so
+    #                     the server's wait_timeout can expire the rest.)
+    if dialect == "sqlite":
+        if path == ":memory:" or path.endswith("/:memory:"):
+            return create_engine(
+                raw_path,
+                **common,
+                poolclass=StaticPool,
+                connect_args={"check_same_thread": False},
+            )
+        # File-based SQLite: QueuePool is the SQLAlchemy 2.0 default.
+        # check_same_thread=False is set automatically by the dialect.
+        # DementorDB._release() returns connections after each operation.
+        return create_engine(raw_path, **common)
+
+    # MySQL / MariaDB / PostgreSQL: QueuePool.
+    #   pool_pre_ping  – detect dead connections before checkout.
+    #   pool_use_lifo  – reuse most-recent connection so idle ones expire
+    #                    naturally via server-side wait_timeout.
+    #   pool_recycle   – hard ceiling: close connections older than 1 hour.
+    #   pool_timeout=5 – fail fast on exhaustion (PoolTimeoutError caught
+    #                    in model.py); hash file is the primary capture path.
+    return create_engine(
+        raw_path,
+        **common,
+        pool_pre_ping=True,
+        pool_use_lifo=True,
+        pool_size=20,
+        max_overflow=40,
+        pool_timeout=5,
+        pool_recycle=3600,
+    )
 
 
 def create_db(session: SessionConfig) -> DementorDB:
-    """
-    High-level helper that returns a fully-initialised :class:`DementorDB`.
+    """Create a fully initialised :class:`DementorDB` ready for use.
 
-    :param session: Current session configuration.
+    Builds the SQLAlchemy engine via :func:`init_engine` and passes it
+    to the :class:`~dementor.db.model.DementorDB` constructor, which
+    creates the tables and sets up the scoped session.
+
+    :param session: Current session configuration holding the
+        :class:`DatabaseConfig` at ``session.db_config``.
     :type session: SessionConfig
-    :return: Ready-to-use :class:`DementorDB` instance.
+    :return: Ready-to-use database wrapper.
     :rtype: DementorDB
-    :raises Exception: If the engine cannot be created.
+    :raises RuntimeError: If the engine cannot be created (e.g. empty
+        ``Path`` with no ``Url``).
     """
     engine = init_engine(session)
     if not engine: