From bb51e10185e734f6151857b589bc055082478891 Mon Sep 17 00:00:00 2001 From: Renan Souza Date: Thu, 30 Apr 2026 13:45:03 -0400 Subject: [PATCH] Adding mq-only + offline flushes --- README.md | 2 +- docs/README.md | 16 +++++++++++++--- docs/cli-reference.rst | 11 +++++++++++ docs/default_user_guide.rst | 1 + docs/setup.rst | 1 + src/flowcept/cli.py | 14 +++++++++++++- src/flowcept/commons/daos/mq_dao/mq_dao_base.py | 2 ++ src/flowcept/configs.py | 15 +++++++++------ src/flowcept/flowcept_api/flowcept_controller.py | 3 ++- 9 files changed, 53 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 207cf47c..91182710 100644 --- a/README.md +++ b/README.md @@ -389,7 +389,7 @@ Flowcept looks for its settings in the following order: Important: - environment variables can override settings values -- use profiles for mode switches such as `full-online`, `full-offline`, `mq-only`, `full-telemetry` +- use profiles for mode switches such as `full-online`, `full-offline`, `mq-only`, `mq-only-no-flush`, `full-telemetry` - adapter flags are additive: ```bash diff --git a/docs/README.md b/docs/README.md index 8c00cf6b..5c37631e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -87,6 +87,7 @@ flowcept --config-profile full-online flowcept --config-profile full-telemetry flowcept --config-profile mq-only flowcept --config-profile full-offline +flowcept --config-profile mq-only-no-flush ``` What this does: @@ -123,6 +124,14 @@ Current profile behavior: - `kv_db.enabled: false` - `databases.mongodb.enabled: false` - `databases.lmdb.enabled: false` +- `mq-only-no-flush`: + - `project.db_flush_mode: offline` + - `project.dump_buffer.enabled: true` + - `mq.enabled: true` + - `kv_db.enabled: false` + - `databases.mongodb.enabled: false` + - `databases.lmdb.enabled: false` + - Tasks accumulate locally and are bulk-published to MQ in a single end-of-run flush. Also dumps to local JSONL. Use `Flowcept(check_safe_stops=False)`. - `full-telemetry`: - enables CPU, per-CPU, process, memory, disk, network, machine telemetry - keeps `telemetry_capture.gpu: null` @@ -554,10 +563,11 @@ Read more: - Fix: ensure `project.db_flush_mode: offline` and `project.dump_buffer.enabled: true` in settings - Symptom: `ValueError` about `db_flush_mode` vs MQ/DB settings - Fix: keep config consistent: - - Offline mode: disable MQ/KV/DBs - - Online mode: use `flowcept --config-profile full-online -y` or `flowcept --config-profile mq-only -y` + - Offline mode (no MQ/KV/DBs): `flowcept --config-profile full-offline -y` + - Offline mode with end-of-run MQ flush: `flowcept --config-profile mq-only-no-flush -y` + - Online mode: `flowcept --config-profile full-online -y` or `flowcept --config-profile mq-only -y` - Symptom: `ValueError` about `check_safe_stops=True` requiring KV while MQ is enabled - - Fix: either use `flowcept --config-profile full-online -y`, or keep `mq-only` and instantiate `Flowcept(check_safe_stops=False)` + - Fix: either use `flowcept --config-profile full-online -y`, or use `mq-only` / `mq-only-no-flush` and instantiate `Flowcept(check_safe_stops=False)` - Symptom: REST API import/start failures (`fastapi`/`uvicorn` missing) - Fix: `pip install flowcept[webservice,mongo]` - Symptom: `Flowcept.db` queries fail due to missing Mongo deps diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 28c8bbba..8ee54401 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -37,6 +37,7 @@ Flowcept provides quick settings profiles to switch between common runtime modes flowcept --config-profile full-telemetry flowcept --config-profile mq-only flowcept --config-profile full-offline + flowcept --config-profile mq-only-no-flush Settings bootstrap ------------------ @@ -102,6 +103,16 @@ Current profile values: - ``kv_db.enabled: false`` - ``databases.mongodb.enabled: false`` - ``databases.lmdb.enabled: false`` +- ``mq-only-no-flush``: + - ``project.db_flush_mode: offline`` + - ``project.dump_buffer.enabled: true`` + - ``mq.enabled: true`` + - ``kv_db.enabled: false`` + - ``databases.mongodb.enabled: false`` + - ``databases.lmdb.enabled: false`` + - Tasks accumulate locally during the run and are bulk-published to MQ in a single + end-of-run flush. A local JSONL copy is also written. No persistent DB is required. + - Use ``Flowcept(check_safe_stops=False)`` with this profile. Environment variables can override settings values at runtime. This matters for keys such as ``MONGO_ENABLED``, ``LMDB_ENABLED``, ``MQ_ENABLED``, ``MQ_TYPE``, ``MQ_PORT``, and ``DB_FLUSH_MODE``. diff --git a/docs/default_user_guide.rst b/docs/default_user_guide.rst index f1676525..ec3bfe82 100644 --- a/docs/default_user_guide.rst +++ b/docs/default_user_guide.rst @@ -40,6 +40,7 @@ Use CLI profiles to switch between common runtime settings quickly: flowcept --config-profile full-telemetry flowcept --config-profile mq-only flowcept --config-profile full-offline + flowcept --config-profile mq-only-no-flush The CLI prints exactly which keys will change, asks for confirmation, and writes to ``FLOWCEPT_SETTINGS_PATH`` (if set) or ``~/.flowcept/settings.yaml``. diff --git a/docs/setup.rst b/docs/setup.rst index 5d1ebf20..8530b351 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -238,6 +238,7 @@ Common profiles: - ``full-online``: Redis MQ + Redis KV + Mongo + online flush - ``full-offline``: offline flush + dump buffer + MQ/KV/DB disabled - ``mq-only``: MQ only, no KV/Mongo/LMDB +- ``mq-only-no-flush``: MQ enabled, tasks accumulate locally and are bulk-published to MQ in a single end-of-run flush; also dumps to local JSONL; use with ``Flowcept(check_safe_stops=False)`` - ``full-telemetry``: telemetry on except GPU Adapter flags are additive: diff --git a/src/flowcept/cli.py b/src/flowcept/cli.py index 03768a62..c143975d 100644 --- a/src/flowcept/cli.py +++ b/src/flowcept/cli.py @@ -81,6 +81,14 @@ "databases.mongodb.enabled": False, "databases.lmdb.enabled": False, }, + "mq-only-no-flush": { + "project.db_flush_mode": "offline", + "project.dump_buffer.enabled": True, + "mq.enabled": True, + "kv_db.enabled": False, + "databases.mongodb.enabled": False, + "databases.lmdb.enabled": False, + }, } @@ -232,7 +240,7 @@ def apply_config_profile(config_profile: str, yes: bool = False): ---------- config_profile : str Profile name. Supported values: full-online, full-telemetry, mq-only, - full-offline. + full-offline, mq-only-no-flush. yes : bool, optional If true, skip confirmation prompt and apply changes immediately. @@ -1012,6 +1020,10 @@ def main(): # noqa: D103 print(" Configure settings for MQ-only mode (MQ enabled; KV and DocDBs disabled).") print(" flowcept --config-profile full-offline [-y]") print(" Configure settings for fully offline mode (MQ + KV + Mongo disabled).") + print(" flowcept --config-profile mq-only-no-flush [-y]") + print(" MQ enabled, no persistent DBs. Tasks accumulate locally and are bulk-published") + print(" to MQ in a single end-of-run flush. Also dumps to local JSONL.") + print(" Use with Flowcept(check_safe_stops=False).") print("") for group, funcs in COMMAND_GROUPS: print(f"{group}:\n") diff --git a/src/flowcept/commons/daos/mq_dao/mq_dao_base.py b/src/flowcept/commons/daos/mq_dao/mq_dao_base.py index 39e22af7..3d952523 100644 --- a/src/flowcept/commons/daos/mq_dao/mq_dao_base.py +++ b/src/flowcept/commons/daos/mq_dao/mq_dao_base.py @@ -186,6 +186,8 @@ def _close_buffer(self): else: self.logger.error("MQ time-based flushing is not started") else: + if MQ_ENABLED and self.buffer: + self.bulk_publish(self.buffer) # end-of-run single flush to Redis self.buffer = list() self.logger.debug("Buffer closed.") diff --git a/src/flowcept/configs.py b/src/flowcept/configs.py index 75ad5e08..d9163c2e 100644 --- a/src/flowcept/configs.py +++ b/src/flowcept/configs.py @@ -295,12 +295,15 @@ def validate_config(): " flowcept --config-profile full-online -y\n" " flowcept --config-profile full-offline -y" ) - if DB_FLUSH_MODE == "offline" and (MQ_ENABLED or MONGO_ENABLED or LMDB_ENABLED or KVDB_ENABLED): + if DB_FLUSH_MODE == "offline" and (MONGO_ENABLED or LMDB_ENABLED or KVDB_ENABLED): raise ValueError( - "Invalid configuration: project.db_flush_mode is 'offline' but MQ/DBs are enabled.\n" - f"mq.enabled={MQ_ENABLED}, kv_db.enabled={KVDB_ENABLED}, " + "Invalid configuration: project.db_flush_mode is 'offline' but persistent DBs are enabled.\n" + f"kv_db.enabled={KVDB_ENABLED}, " f"databases.mongodb.enabled={MONGO_ENABLED}, databases.lmdb.enabled={LMDB_ENABLED}.\n" - "Disable mq.enabled, kv_db.enabled, and databases when running offline.\n" - "Quick fix with profile:\n" - " flowcept --config-profile full-offline -y" + "Disable kv_db and databases when running offline.\n" + "Note: mq.enabled=true is allowed with db_flush_mode=offline — tasks accumulate locally\n" + "and are bulk-published to MQ in a single flush at the end of the run.\n" + "Quick fix with profiles:\n" + " flowcept --config-profile full-offline -y\n" + " flowcept --config-profile mq-only-no-flush -y" ) diff --git a/src/flowcept/flowcept_api/flowcept_controller.py b/src/flowcept/flowcept_api/flowcept_controller.py index 3e709772..04b5f1c9 100644 --- a/src/flowcept/flowcept_api/flowcept_controller.py +++ b/src/flowcept/flowcept_api/flowcept_controller.py @@ -134,7 +134,8 @@ def __init__( "\n" "Quick fix with profiles:\n" " flowcept --config-profile full-online -y\n" - " flowcept --config-profile mq-only -y # and instantiate Flowcept(check_safe_stops=False)" + " flowcept --config-profile mq-only -y # and instantiate Flowcept(check_safe_stops=False)\n" + " flowcept --config-profile mq-only-no-flush -y # end-of-run bulk flush, check_safe_stops=False" ) self._enable_persistence = start_persistence self._db_inserters: List = []