diff --git a/graphify/__main__.py b/graphify/__main__.py index 924ae986d..4ba382521 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -539,6 +539,7 @@ def _run_cli() -> None: print(" --force overwrite graph.json even if the rebuild has fewer nodes") print(" (also: GRAPHIFY_FORCE=1 env var; use after refactors that delete code)") print(" --no-cluster skip clustering, write raw extraction only") + print(" --one-file-system don't cross mount points (find -xdev); auto-on for a / or $HOME root") print(" cluster-only rerun clustering on an existing graph.json and regenerate report") print(" --no-viz skip graph.html generation (useful for >5000 node graphs / CI)") print(" --graph path to graph.json (default /graphify-out/graph.json)") @@ -585,6 +586,10 @@ def _run_cli() -> None: print(" --half-life-days N signal weight halves every N days (default 30)") print(" --min-corroboration N distinct useful results to prefer a node (default 2)") print(" check-update check needs_update flag and notify if semantic re-extraction is pending (cron-safe)") + print(" serve-html serve graphify-out/ over HTTP and print the graph.html URL (headless hosts)") + print(" --host H interface to bind (default 127.0.0.1; --host 0.0.0.0 exposes it on the network)") + print(" --port N port to listen on (default 8899)") + print(" --dir DIR directory to serve (default graphify-out)") print(" tree emit a D3 v7 collapsible-tree HTML for graph.json") print(" --graph PATH path to graph.json (default graphify-out/graph.json)") print(" --output HTML output path (default graphify-out/GRAPH_TREE.html)") @@ -610,6 +615,8 @@ def _run_cli() -> None: print(" --out DIR, --output DIR output dir (default: ); writes /graphify-out/") print(" --google-workspace export .gdoc/.gsheet/.gslides shortcuts via gws before extraction") print(" --no-gitignore ignore .gitignore and .git/info/exclude (prioritizes .graphifyignore)") + print(" --one-file-system don't cross mount points (find -xdev); auto-on for a / or $HOME root") + print(" --exclude PATTERN gitignore-style exclude, anchored at the scan root (repeatable)") print(" --no-cluster skip clustering, write raw extraction only") print(" --code-only index code (local AST, no API key) and skip doc/paper/image files") print(" --postgres DSN extract schema from a live PostgreSQL database") diff --git a/graphify/cli.py b/graphify/cli.py index 441e4ca36..abfc201c1 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -6,6 +6,7 @@ import of main to avoid a cli<->__main__ import cycle. """ from __future__ import annotations +import functools import json import os import re @@ -802,6 +803,70 @@ def _reenter_main() -> None: main() +def _is_high_scan_root(root: Path) -> str | None: + """Return a human label when *root* is a dangerously high scan root, else None. + + A whole-machine scan of a filesystem root (`/`, or a drive root on Windows) + or the user's `$HOME` walks vendored caches, virtual filesystems and other + machines' mounts — the root cause of a 2-hour `/` scan that ingested 1.18M + refs from the Go module cache. The extract/update paths use this to auto-arm + protections and warn (FIX 2/FIX 3). + """ + try: + resolved = root.resolve() + except (OSError, RuntimeError): + resolved = root + if resolved == Path(resolved.anchor) and resolved.anchor: + return "filesystem root" + home = Path(os.path.expanduser("~")) + try: + if home and resolved == home.resolve(): + return "home directory ($HOME)" + except (OSError, RuntimeError): + pass + return None + + +def _warn_high_root_scan(root: Path, label: str, cli_excludes: list[str], one_file_system: bool) -> None: + """Print a loud, unmissable warning before a whole-machine scan (FIX 3). + + Only fired when the user passed no ``--exclude`` and there is no + ``.graphifyignore`` at the root, i.e. nothing scoping the scan. Never blocks + (kept scriptable) — it just makes the active protections visible. + """ + if cli_excludes: + return + try: + if (root / ".graphifyignore").is_file(): + return + except OSError: + return + print( + f"[graphify] WARNING: starting a whole-machine scan of {root} ({label}).", + file=sys.stderr, + ) + print( + "[graphify] This can take hours and ingest vendored caches / other " + "mounts. Active protections:", + file=sys.stderr, + ) + print( + "[graphify] - default excludes: /proc /sys /dev /run + Go/.cargo/" + ".rustup/.m2/.gradle/.pnpm-store caches (disable: GRAPHIFY_NO_DEFAULT_EXCLUDES=1)", + file=sys.stderr, + ) + print( + f"[graphify] - one-file-system: {'ON' if one_file_system else 'OFF'} " + "(mount points not crossed when ON)", + file=sys.stderr, + ) + print( + "[graphify] Scope it with --exclude PATTERN or a .graphifyignore file " + "to scan faster.", + file=sys.stderr, + ) + + def dispatch_command(cmd: str) -> None: if cmd == "provider": from graphify.llm import _custom_providers_path, BACKENDS @@ -2077,6 +2142,10 @@ def dispatch_command(cmd: str) -> None: elif cmd == "update": force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes") no_cluster = False + # --one-file-system (find -xdev). update re-runs detect() via watch's + # _rebuild_code, which has no kwarg for this, so it is plumbed through the + # GRAPHIFY_ONE_FILE_SYSTEM env var that detect() already honours (FIX 2). + one_file_system: bool | None = None args = sys.argv[2:] watch_arg: str | None = None for a in args: @@ -2086,6 +2155,9 @@ def dispatch_command(cmd: str) -> None: if a == "--no-cluster": no_cluster = True continue + if a in ("--one-file-system", "--xdev"): + one_file_system = True + continue if a.startswith("-"): print(f"error: unknown update option: {a}", file=sys.stderr) sys.exit(2) @@ -2106,6 +2178,24 @@ def dispatch_command(cmd: str) -> None: if not watch_path.exists(): print(f"error: path not found: {watch_path}", file=sys.stderr) sys.exit(1) + + # FIX 2/FIX 3: high-root guardrails (see the extract path). Auto-enable + # --one-file-system for a `/` or $HOME scan root, then warn loudly if + # nothing scopes a whole-machine scan. + _high_root_label = _is_high_scan_root(watch_path) + if _high_root_label is not None: + if one_file_system is None: + one_file_system = True + print( + f"[graphify update] {_high_root_label} scan root: " + f"auto-enabling --one-file-system (won't cross mount points)" + ) + _warn_high_root_scan( + watch_path.resolve(), _high_root_label, [], bool(one_file_system) + ) + if one_file_system: + os.environ["GRAPHIFY_ONE_FILE_SYSTEM"] = "1" + from graphify.watch import _rebuild_code print(f"Re-extracting code files in {watch_path} (no LLM needed)...") @@ -2155,6 +2245,87 @@ def dispatch_command(cmd: str) -> None: check_update(Path(sys.argv[2]).resolve()) sys.exit(0) + elif cmd == "serve-html": + # Serve graphify-out/ over HTTP so headless hosts (no browser, remote + # boxes) can open graph.html in a local browser (#headless). Uses only + # the stdlib http.server — no extra deps. + import http.server + + host = "127.0.0.1" + port = 8899 + serve_dir = Path(_GRAPHIFY_OUT) + args = sys.argv[2:] + i_arg = 0 + while i_arg < len(args): + a = args[i_arg] + if a == "--host" and i_arg + 1 < len(args): + host = args[i_arg + 1]; i_arg += 2 + elif a.startswith("--host="): + host = a.split("=", 1)[1]; i_arg += 1 + elif a == "--port" and i_arg + 1 < len(args): + port = int(args[i_arg + 1]); i_arg += 2 + elif a.startswith("--port="): + port = int(a.split("=", 1)[1]); i_arg += 1 + elif a == "--dir" and i_arg + 1 < len(args): + serve_dir = Path(args[i_arg + 1]); i_arg += 2 + elif a.startswith("--dir="): + serve_dir = Path(a.split("=", 1)[1]); i_arg += 1 + elif a in ("-h", "--help"): + print("Usage: graphify serve-html [--host H] [--port N] [--dir DIR]") + print(" --host H interface to bind (default 127.0.0.1; use 0.0.0.0 to expose on the network)") + print(" --port N port to listen on (default 8899)") + print(" --dir DIR directory to serve (default graphify-out)") + return + else: + print(f"error: unknown serve-html option: {a}", file=sys.stderr) + sys.exit(2) + + serve_dir = serve_dir.resolve() + if not serve_dir.is_dir(): + print(f"error: directory not found: {serve_dir}", file=sys.stderr) + print("Run `graphify extract ` first to generate graphify-out/.", file=sys.stderr) + sys.exit(1) + + # Serve serve_dir as the document root regardless of CWD (Python 3.7+ + # SimpleHTTPRequestHandler honours the `directory` kwarg). + handler = functools.partial( + http.server.SimpleHTTPRequestHandler, directory=str(serve_dir) + ) + + # ThreadingHTTPServer keeps the browser responsive when it opens several + # requests (HTML + JSON + assets) at once. + http.server.ThreadingHTTPServer.allow_reuse_address = True + try: + httpd = http.server.ThreadingHTTPServer((host, port), handler) + except OSError as exc: + print(f"error: could not bind {host}:{port} ({exc})", file=sys.stderr) + sys.exit(1) + + # 0.0.0.0/:: binds every interface — surface that it is reachable off-box. + display_host = "localhost" if host in ("127.0.0.1", "0.0.0.0", "::") else host + graph_html = serve_dir / "graph.html" + print(f"Serving {serve_dir} at http://{display_host}:{port}/") + if graph_html.is_file(): + print(f"Open the graph: http://{display_host}:{port}/graph.html") + else: + print( + f"note: graph.html not found in {serve_dir} " + "(run `graphify extract ` to generate it)" + ) + if host in ("0.0.0.0", "::"): + print( + f"warning: bound to {host} — this exposes the server to your " + "network, not just this machine.", + file=sys.stderr, + ) + print("Press Ctrl+C to stop.") + try: + httpd.serve_forever() + except KeyboardInterrupt: + print("\nStopping server.") + finally: + httpd.server_close() + return elif cmd == "tree": # Emit a D3 v7 collapsible-tree HTML view of graph.json: # expand-all / collapse-all / reset-view buttons, multi-line @@ -2854,6 +3025,9 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": cli_exclude_hubs: float | None = None cli_excludes: list[str] = [] cli_timing: bool = False + # --one-file-system (find -xdev): prune child dirs on a different mount. + # Auto-enabled below for a `/` or $HOME scan root. None -> library default. + cli_one_file_system: bool | None = None # --force parity with `graphify update`: the flag or GRAPHIFY_FORCE=1 # disables the incremental gate and skips semantic-cache reads (#1894). force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes") @@ -2945,6 +3119,8 @@ def _parse_float(name: str, raw: str) -> float: cli_excludes.append(args[i + 1]); i += 2 elif a.startswith("--exclude="): cli_excludes.append(a.split("=", 1)[1]); i += 1 + elif a in ("--one-file-system", "--xdev"): + cli_one_file_system = True; i += 1 elif a == "--postgres" and i + 1 < len(args): cli_postgres_dsn = args[i + 1]; i += 2 elif a.startswith("--postgres="): @@ -2977,6 +3153,22 @@ def _parse_float(name: str, raw: str) -> float: if deep_mode: print("[graphify extract] deep mode enabled: richer semantic extraction") + # FIX 2/FIX 3: high-root guardrails. Auto-enable --one-file-system for a + # `/` or $HOME scan root (with a notice), then warn loudly if nothing is + # scoping a whole-machine scan. + if has_path: + _high_root_label = _is_high_scan_root(target) + if _high_root_label is not None: + if cli_one_file_system is None: + cli_one_file_system = True + print( + f"[graphify extract] {_high_root_label} scan root: " + f"auto-enabling --one-file-system (won't cross mount points)" + ) + _warn_high_root_scan( + target, _high_root_label, cli_excludes, bool(cli_one_file_system) + ) + # CLI flag wins over env var. Setting GRAPHIFY_API_TIMEOUT here so # _call_openai_compat picks it up without needing a new kwarg path. if cli_api_timeout is not None: @@ -3062,6 +3254,7 @@ def _parse_float(name: str, raw: str) -> float: google_workspace=google_workspace or None, extra_excludes=_effective_excludes or None, gitignore=_effective_gitignore, + one_file_system=cli_one_file_system, ) files_by_type = detection.get("files", {}) new_by_type = detection.get("new_files", {}) @@ -3108,6 +3301,7 @@ def _parse_float(name: str, raw: str) -> float: extra_excludes=_effective_excludes or None, cache_root=out_root, gitignore=_effective_gitignore, + one_file_system=cli_one_file_system, ) files_by_type = detection.get("files", {}) code_files = [Path(p) for p in files_by_type.get("code", [])] diff --git a/graphify/detect.py b/graphify/detect.py index 23e9198bf..3b72af0ba 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -812,6 +812,56 @@ def count_words(path: Path) -> int: ".worktrees", # git worktree convention (#947) — sibling checkouts, always redundant } +# Absolute paths never worth descending: kernel virtual filesystems. A high-root +# scan (e.g. `graphify extract /`) that walks /proc alone ingests a per-PID tree +# that changes under the scanner and is architecturally meaningless; /sys, /dev +# and /run are the same class. Matched by the directory's resolved absolute path, +# not its basename, so an unrelated project dir named "proc" is never pruned. +# Disable with GRAPHIFY_NO_DEFAULT_EXCLUDES=1 (mirrors the vendored-cache gate). +_SKIP_ABS_PATHS = frozenset({ + "/proc", "/sys", "/dev", "/run", +}) + +# Well-known vendored dependency caches. These hold thousands-to-millions of +# extracted upstream sources that are never this project's code — the Go module +# cache under `go/pkg/mod` alone ingested 1.18M refs on a `/` scan (#root-cause). +# Basenames like `.cargo`/`.m2` are unambiguous enough to prune by name, but +# `pkg`/`mod` are common real directory names, so the Go cache is gated on the +# `.../go/pkg/mod` path shape below in `_is_vendored_cache_dir` rather than a +# bare basename. Disable all of these with GRAPHIFY_NO_DEFAULT_EXCLUDES=1. +_SKIP_VENDORED_CACHE_DIRS = frozenset({ + ".cargo", ".rustup", ".m2", ".gradle", ".pnpm-store", +}) + + +def _default_excludes_enabled() -> bool: + """False when the user opted out of the built-in high-root protections. + + GRAPHIFY_NO_DEFAULT_EXCLUDES=1 disables the virtual-filesystem and + vendored-cache pruning added for whole-machine scans, for the rare case a + user really does want to index one of those trees. Explicit ``--exclude`` / + ``.graphifyignore`` ``!`` negations still take precedence regardless. + """ + return os.environ.get("GRAPHIFY_NO_DEFAULT_EXCLUDES", "").lower() not in ("1", "true", "yes") + + +def _is_vendored_cache_dir(name: str, parent: "Path | None") -> bool: + """True when *name* under *parent* is a well-known vendored dependency cache. + + Basename caches (`.cargo`, `.rustup`, `.m2`, `.gradle`, `.pnpm-store`) match + directly. The Go module cache is gated on the `.../go/pkg/mod` path structure + — pruning a bare `mod`/`pkg` by name would drop legitimate source — so it + matches only when the directory is `mod` whose parent is `pkg` whose parent + is `go`. + """ + if name in _SKIP_VENDORED_CACHE_DIRS: + return True + # Go module cache: /pkg/mod (parent basename "pkg", grandparent "go"). + if name == "mod" and parent is not None and parent.name == "pkg" and parent.parent.name == "go": + return True + return False + + # Large generated files that are never useful to extract _SKIP_FILES = { "package-lock.json", "yarn.lock", "pnpm-lock.yaml", @@ -1216,6 +1266,43 @@ def _matches(rel: str, p: str, path_relative: bool) -> bool: return _eval(path) +def _explicitly_reincluded(path: Path, patterns: list[tuple[Path, str]]) -> bool: + """True when a user ``!`` negation pattern explicitly matches *path*. + + Lets an explicit ``--exclude '!go/pkg/mod'`` / ``.graphifyignore`` negation + re-include a directory the built-in default excludes (virtual filesystems, + vendored caches) would otherwise prune (#root-cause). Only negation patterns + are considered — a plain include has no bearing on default-exclude pruning. + """ + for anchor, pattern in patterns: + if not pattern.startswith("!"): + continue + raw = pattern[1:] + directory_only = raw.endswith("/") + path_relative = "/" in raw.rstrip("/") + p = raw.strip("/") + if not p: + continue + try: + rel = _nfc(str(path.relative_to(anchor)).replace(os.sep, "/")) + except ValueError: + continue + if rel == ".": + continue + if directory_only and not path.is_dir(): + continue + if path_relative: + if _match_anchored_ignore_pattern(rel, p): + return True + else: + if fnmatch.fnmatch(_nfc(path.name), p) or fnmatch.fnmatch(rel, p): + return True + for part in rel.split("/"): + if fnmatch.fnmatch(part, p): + return True + return False + + def ignored_predicate( root: Path, *, @@ -1304,8 +1391,23 @@ def _resolves_under_root(path: Path, root: Path) -> bool: return True -def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: bool | None = None, extra_excludes: list[str] | None = None, cache_root: Path | None = None, gitignore: bool = True) -> dict: +def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: bool | None = None, extra_excludes: list[str] | None = None, cache_root: Path | None = None, gitignore: bool = True, one_file_system: bool | None = None) -> dict: root = root.resolve() + # --one-file-system / GRAPHIFY_ONE_FILE_SYSTEM=1 (find -xdev): capture the + # root's device now and prune any child dir on a different filesystem so a + # high-root scan never crosses a mount point (network shares, bind mounts, + # /proc, external disks). Auto-enabled by the CLI for `/` and $HOME (#root). + if one_file_system is None: + one_file_system = os.environ.get("GRAPHIFY_ONE_FILE_SYSTEM", "").lower() in ("1", "true", "yes") + root_dev: int | None = None + if one_file_system: + try: + root_dev = os.stat(_os_path(root)).st_dev + except OSError: + root_dev = None + # Built-in high-root protections: virtual filesystems + vendored dep caches. + # Escapable via GRAPHIFY_NO_DEFAULT_EXCLUDES=1 (#root-cause). + default_excludes = _default_excludes_enabled() configured_out_dir = root / GRAPHIFY_OUT configured_out_names = {configured_out_dir.name} try: @@ -1436,6 +1538,29 @@ def _on_walk_error(err: OSError) -> None: if is_configured_out: pruned_noise.append(str(child) + os.sep) continue + # Built-in high-root protections (#root-cause). An explicit + # user `!` negation re-includes and takes precedence, so a + # user who really wants to index /proc or the Go cache can. + if default_excludes and not _explicitly_reincluded(child, ignore_patterns): + try: + child_abs = os.path.realpath(_os_path(child)) + except OSError: + child_abs = str(child) + if child_abs in _SKIP_ABS_PATHS: + pruned_noise.append(str(child) + os.sep + " [virtual filesystem]") + continue + if _is_vendored_cache_dir(d, dp): + pruned_noise.append(str(child) + os.sep + " [vendored dependency cache]") + continue + # --one-file-system: never cross a mount point (#root). + if root_dev is not None and not _explicitly_reincluded(child, ignore_patterns): + try: + child_dev = os.stat(_os_path(child)).st_dev + except OSError: + child_dev = root_dev + if child_dev != root_dev: + pruned_noise.append(str(child) + os.sep + " [different filesystem (--one-file-system)]") + continue if _is_noise_dir(d, dp): # Record pruned-as-noise dirs so a wrongly-pruned real # source dir is at least traceable in the output rather @@ -1916,6 +2041,7 @@ def detect_incremental( kind: str = "semantic", extra_excludes: list[str] | None = None, gitignore: bool = True, + one_file_system: bool | None = None, ) -> dict: """Like detect(), but returns only new or modified files since the last run. @@ -1945,6 +2071,7 @@ def detect_incremental( google_workspace=google_workspace, extra_excludes=extra_excludes, gitignore=gitignore, + one_file_system=one_file_system, ) # Pass ``root`` so a manifest written with relative keys (post-#777) is # re-anchored to the absolute form the rest of this function compares diff --git a/graphify/export.py b/graphify/export.py index 0a1a0bb65..708828289 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -315,8 +315,23 @@ def _json_sort_key(item: dict) -> str: if true_src is not None and true_tgt is not None: link["source"] = true_src link["target"] = true_tgt - data["nodes"].sort(key=_json_sort_key) - data["links"].sort(key=_json_sort_key) + # Deterministic ordering keyed on stable identity fields only. Avoid a full + # json.dumps of every element (#: at ~844k edges that re-serializes the whole + # graph an extra time). Nodes are keyed by id; links by their endpoint + + # relation identity. Total and stable across runs — need not byte-match the + # old json-dump ordering. + def _node_sort_key(node: dict): + return str(node.get("id")) + + def _link_sort_key(link: dict): + return ( + str(link.get("source")), + str(link.get("target")), + str(link.get("relation") or link.get("key") or ""), + ) + + data["nodes"].sort(key=_node_sort_key) + data["links"].sort(key=_link_sort_key) if "hyperedges" not in getattr(G, "graph", {}): # Hardening (#2485): a graph with NO hyperedges key at all was built by # a path that never engaged hyperedge metadata — distinct from an diff --git a/graphify/exporters/html.py b/graphify/exporters/html.py index 372dd397f..9486e76cf 100644 --- a/graphify/exporters/html.py +++ b/graphify/exporters/html.py @@ -27,6 +27,62 @@ def _viz_node_limit() -> int: except ValueError: return MAX_NODES_FOR_VIZ +_VIS_NETWORK_CDN = "https://unpkg.com/vis-network@9.1.6/standalone/umd/vis-network.min.js" +_VIS_NETWORK_SRI = "sha384-Ux6phic9PEHJ38YtrijhkzyJ8yQlH8i/+buBR8s3mAZOJrP1gwyvAcIYl3GWtpX1" +_VIS_NETWORK_VENDOR_NAME = "vis-network.min.js" + + +def _vis_network_script_tag(output_path: str) -> str: + """Return the ' + ) + if os.environ.get("GRAPHIFY_VIZ_OFFLINE") != "1": + return cdn_tag + + out = Path(output_path) + vendored = out.parent / _VIS_NETWORK_VENDOR_NAME + try: + if not vendored.exists(): + # Try a copy bundled inside the package (vendored offline asset). + bundled = Path(__file__).resolve().parent / _VIS_NETWORK_VENDOR_NAME + if bundled.exists(): + out.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(bundled, vendored) + if vendored.exists(): + # Local reference — relative so the report stays portable. + return f'' + except OSError: + pass + print( + "[graphify] GRAPHIFY_VIZ_OFFLINE=1 but no local " + f"{_VIS_NETWORK_VENDOR_NAME} could be produced next to {out.name}; " + "falling back to the SRI-pinned CDN (requires network to view).", + ) + return cdn_tag + + def _html_styles() -> str: return """