diff --git a/.gitignore b/.gitignore index 0a6775b2a..3e447e3a5 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ skills/ !tools/skillgen/fragments/core/** docs/superpowers/ .vscode/ +.idea/ .kilo openspec/ # Local benchmark scripts — never commit diff --git a/CHANGELOG.md b/CHANGELOG.md index c029b472a..586ac8ecc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,6 +80,8 @@ Full release notes with details on each version: [GitHub Releases](https://githu - Fix: Python extraction no longer crashes resolving an over-deep relative import (`from ....x import y` above the package root) (#2605, thanks @SinghAman21). - Fix: a Go qualified type (`pkg.Type`) resolves by import path instead of losing the qualifier and binding by bare name to an unrelated same-named symbol (#2608, thanks @gnukeno). - Fix: `graph.html`'s document title no longer embeds the generator's absolute host path (#2598, thanks @michaelxer); it keeps the path from the output-dir marker onward. +- Feat: verbose LLM tracing for community labeling and the dedup tiebreaker. `graphify cluster-only . --verbose` (or `graphify label . --verbose`, or `GRAPHIFY_LLM_VERBOSE=1` on any command that calls the LLM) prints each call's prompt, thinking, response, and per-call token counts to stderr, followed by a run-level token total with a cost estimate. For the `claude` backend, verbose mode enables extended thinking (1024-token budget, billed as output tokens) so there is reasoning to show; for `claude-cli` it switches to `--output-format stream-json --verbose` and reads thinking blocks from the streamed assistant events; OpenAI-compatible and Bedrock backends surface `reasoning_content`/`reasoningContent` when the deployment returns it. Non-verbose runs are byte-for-byte unchanged in behavior and cost. +- Feat: token-economics-only mode. `graphify cluster-only . --tokens` (or `graphify label . --tokens`, or `GRAPHIFY_LLM_TOKENS=1` on any command) prints one line of per-call token counts plus the run total with a cost estimate — without the prompt/thinking/response dumps, and without verbose's call-shaping side effects (no extended thinking on `claude`, no stream-json on `claude-cli`), so the accounting is billing-neutral. `--verbose` implies it: the full exchange already ends with the token line. ## 0.9.39 (2026-08-10) diff --git a/README.md b/README.md index e3a2cf867..d5f933d77 100644 --- a/README.md +++ b/README.md @@ -533,6 +533,8 @@ These are only needed for **headless / CI extraction** (`graphify extract`). Whe | `GRAPHIFY_MAX_GRAPH_BYTES` | Override the 512 MiB graph.json size cap — e.g. `700MB`, `2GB`, or plain bytes | optional — useful for very large corpora | | `GRAPHIFY_MAX_CONTEXTS` | Maximum number of non-default project graphs retained by one multi-project MCP server | optional — default: `8`; invalid values use `8`, and values below `1` use `1` | | `GRAPHIFY_LLM_TEMPERATURE` | Override LLM temperature for semantic extraction — e.g. `0.7`, or `none` to omit | optional — auto-omitted for o1/o3/o4/gpt-5 reasoning models | +| `GRAPHIFY_LLM_VERBOSE` | Print every labeling/dedup LLM call's prompt, thinking, response, and per-call token counts to stderr | optional; also `--verbose` on `cluster-only`/`label` | +| `GRAPHIFY_LLM_TOKENS` | Print only the token economics: one line of per-call token counts plus a run total — no prompt/thinking/response dumps | optional; also `--tokens` on `cluster-only`/`label` | --- @@ -784,12 +786,18 @@ graphify cluster-only ./my-project --exclude-hubs 99 # exclude p99 deg graphify cluster-only ./my-project --no-label # keep "Community N" placeholders graphify cluster-only ./my-project --backend=gemini # backend for community naming graphify cluster-only ./my-project --backend=gemini --model gemini-2.5-pro # specific model +graphify cluster-only ./my-project --verbose # trace labeling LLM calls: prompt, thinking, response, tokens +graphify cluster-only ./my-project --tokens # token economics only: per-call counts and a run total graphify label ./my-project # (re)name communities with the configured backend graphify label ./my-project --backend=openai --model gpt-4o # force a specific backend and model ``` > **Community names:** inside an agent (Claude Code, Gemini CLI) the agent names communities itself. When you run the bare CLI, `cluster-only` auto-names them with the configured backend (built-in or custom OpenAI-compatible provider) — pass `--no-label` to keep `Community N`, or run `graphify label` to (re)generate names on demand. +> **Verbose labeling:** `--verbose` on `cluster-only`/`label` (or `GRAPHIFY_LLM_VERBOSE=1` on any command) prints each labeling call's prompt, thinking, response, and per-call token counts to stderr, then a run total with a cost estimate. On the `claude` backend verbose mode enables extended thinking so the reasoning is visible (billed as output tokens); on `claude-cli` it reads thinking from the CLI's streamed assistant events. Non-verbose runs are unchanged. + +> **Token economics only:** `--tokens` (or `GRAPHIFY_LLM_TOKENS=1` on any command) prints just one line of per-call token counts plus the run total — no prompt/thinking/response dumps. Unlike `--verbose` it changes nothing about the calls themselves (no extended thinking, no stream-json), so the accounting is side-effect free. + --- ## Learn more diff --git a/graphify/__main__.py b/graphify/__main__.py index 155501a98..c73763501 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -546,12 +546,16 @@ def _run_cli() -> None: print(" --model= model to use for community naming") print(" --max-concurrency=N parallel community-labeling LLM calls (default 4; forced to 1 for ollama/claude-cli)") print(" --batch-size=N communities per labeling LLM call (default 100)") + print(" --verbose trace labeling LLM calls: prompt, thinking, response, tokens") + print(" --tokens token economics only: per-call token counts and a run total") print(" label (re)name communities with the configured LLM backend, regenerate report") print(" --missing-only keep existing labels and only name missing/placeholder communities") print(" --backend= backend to use (default: auto-detect from API keys)") print(" --model= model to use for community naming") print(" --max-concurrency=N parallel labeling LLM calls (default 4; forced to 1 for ollama/claude-cli)") print(" --batch-size=N communities per labeling LLM call (default 100)") + print(" --verbose trace labeling LLM calls: prompt, thinking, response, tokens") + print(" --tokens token economics only: per-call token counts and a run total") print(" query \"\" BFS traversal of graph.json for a question") print(" --dfs use depth-first instead of breadth-first") print(" --context C explicit edge-context filter (repeatable)") diff --git a/graphify/cli.py b/graphify/cli.py index 95adad4b9..14f9a22ab 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -1724,6 +1724,8 @@ def dispatch_command(cmd: str) -> None: no_label = "--no-label" in sys.argv missing_only = "--missing-only" in sys.argv co_timing = "--timing" in sys.argv + co_verbose = "--verbose" in sys.argv + co_tokens = "--tokens" in sys.argv _backend_arg = next((a for a in sys.argv if a.startswith("--backend=")), None) label_backend = _backend_arg.split("=", 1)[1] if _backend_arg else None _model_arg = next((a for a in sys.argv if a.startswith("--model=")), None) @@ -1772,7 +1774,7 @@ def dispatch_command(cmd: str) -> None: label_batch_size = int(args[i_arg + 1]); label_batch_size_explicit = True; i_arg += 2 elif a.startswith("--batch-size="): label_batch_size = int(a.split("=", 1)[1]); label_batch_size_explicit = True; i_arg += 1 - elif a in ("--no-viz", "--missing-only") or a.startswith("--min-community-size="): + elif a in ("--no-viz", "--missing-only", "--verbose", "--tokens") or a.startswith("--min-community-size="): i_arg += 1 elif a.startswith("--"): i_arg += 1 @@ -1782,6 +1784,20 @@ def dispatch_command(cmd: str) -> None: i_arg += 1 if watch_path is None: watch_path = Path(".") + if co_verbose: + # Trace every labeling LLM exchange (prompt, thinking, reply, + # per-call tokens) to stderr. Env-var equivalent: + # GRAPHIFY_LLM_VERBOSE=1, which also covers paths with no flag + # parsing (watch, update, dedup tiebreaker). + from graphify.llm import set_llm_verbose + set_llm_verbose(True) + if co_tokens: + # Token-economics-only: per-call token lines plus the run total, + # without prompt/thinking/response dumps and without verbose's + # call-shaping side effects (extended thinking, stream-json). + # Env-var equivalent: GRAPHIFY_LLM_TOKENS=1. + from graphify.llm import set_llm_tokens + set_llm_tokens(True) graph_json = graph_override if graph_override is not None else watch_path / _GRAPHIFY_OUT / "graph.json" if not graph_json.exists(): print( @@ -1887,6 +1903,8 @@ def dispatch_command(cmd: str) -> None: ("--model", label_model is not None), ("--batch-size", label_batch_size_explicit), ("--max-concurrency", label_max_concurrency_explicit), + ("--verbose", co_verbose), + ("--tokens", co_tokens), ) if given] if _ignored_label_flags: print( @@ -1996,6 +2014,32 @@ def dispatch_command(cmd: str) -> None: if v and v != f"Community {cid}" and v != str(cid) }) stages.mark("label") + # Gate on the EFFECTIVE tracing state, not just the CLI flags: with + # GRAPHIFY_LLM_VERBOSE/GRAPHIFY_LLM_TOKENS set in the env but no flag, + # every per-call line prints yet this run total used to be skipped. + # The flags imply the effective state via set_llm_verbose/set_llm_tokens. + from graphify.llm import _llm_tokens as _tracing_on_tokens, _llm_verbose as _tracing_on_verbose + if (_tracing_on_verbose() or _tracing_on_tokens()) and (label_token_usage["input"] or label_token_usage["output"]): + # Run-level token accountability: per-call numbers were already + # traced by llm.py's verbose/tokens hook; this is the spendable + # total, with a cost estimate when the backend's pricing is known. + from graphify.llm import detect_backend as _detect_backend, estimate_cost as _est_cost + _cost_backend = label_backend + if _cost_backend is None: + try: + _cost_backend = _detect_backend() + except Exception: + _cost_backend = None + _cost = ( + _est_cost(_cost_backend, label_token_usage["input"], label_token_usage["output"]) + if _cost_backend else 0.0 + ) + print( + f"[graphify llm] label run total: {label_token_usage['input']:,} in · " + f"{label_token_usage['output']:,} out " + f"(~${_cost:.4f} @ {_cost_backend or 'unknown backend'})", + file=sys.stderr, + ) questions = suggest_questions(G, communities, labels) # cluster-only re-clusters an EXISTING graph: the code content is exactly # what extract saw, so keep the extract-time commit stamp instead of diff --git a/graphify/llm.py b/graphify/llm.py index 46a4785d3..e203c0e6c 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -10,6 +10,7 @@ import os import re import sys +import threading import time from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor, as_completed @@ -447,6 +448,96 @@ def _thinking_disabled_via_env() -> bool: empty content otherwise.""" return os.environ.get("GRAPHIFY_DISABLE_THINKING", "").strip().lower() in ("1", "true", "yes", "on") + +# Process-local verbose switch for LLM-call tracing. Set by the CLI's +# `--verbose` flag (set_llm_verbose) or by GRAPHIFY_LLM_VERBOSE=1 in the +# environment. The env var covers paths with no flag parsing (watch, update, +# dedup tiebreaker), the flag covers a single `cluster-only`/`label` run. +_LLM_VERBOSE = False + +# Serialized stderr writes: labeling fans batches out across threads, and +# without a lock two batches' exchanges interleave mid-line. +_VERBOSE_LOCK = threading.Lock() + + +def set_llm_verbose(enabled: bool = True) -> None: + """Enable verbose LLM-call tracing for this process (see ``--verbose``).""" + global _LLM_VERBOSE + _LLM_VERBOSE = enabled + + +def _llm_verbose() -> bool: + return _LLM_VERBOSE or os.environ.get("GRAPHIFY_LLM_VERBOSE", "").strip().lower() in ("1", "true", "yes", "on") + + +# Token-economics-only mode: same per-call accounting as verbose, but without +# dumping prompt/thinking/response bodies. Unlike verbose it changes NOTHING +# about the calls themselves — no extended thinking on claude, no stream-json +# on claude-cli — so it is pure accounting with zero billing side effects. +# Set by `--tokens` (set_llm_tokens) or GRAPHIFY_LLM_TOKENS=1. Verbose mode +# implies it: the full exchange already ends with the token line. +_LLM_TOKENS = False + + +def set_llm_tokens(enabled: bool = True) -> None: + """Enable token-only LLM-call accounting for this process (see ``--tokens``).""" + global _LLM_TOKENS + _LLM_TOKENS = enabled + + +def _llm_tokens() -> bool: + return _LLM_TOKENS or os.environ.get("GRAPHIFY_LLM_TOKENS", "").strip().lower() in ("1", "true", "yes", "on") + + +def _verbose_llm_exchange( + *, + backend: str, + model: str, + prompt: str, + thinking: str | None, + text: str, + usage: dict, +) -> None: + """Dump one LLM exchange (prompt, thinking, response, tokens) to stderr. + + Thinking is only shown when the backend surfaced it (claude with extended + thinking enabled, claude-cli stream-json assistant events, OpenAI-compat + models returning ``reasoning_content``); otherwise the section says so + explicitly, so "no thinking shown" is distinguishable from "verbose off". + """ + with _VERBOSE_LOCK: + + def _emit(line: str = "") -> None: + print(f"[graphify llm] {line}", file=sys.stderr, flush=True) + + _emit(f"── call ── backend={backend} model={model} prompt={len(prompt):,} chars") + _emit("── prompt ──") + print(prompt, file=sys.stderr, flush=True) + if thinking: + _emit("── thinking ──") + print(thinking, file=sys.stderr, flush=True) + else: + _emit("── thinking: (none returned by backend) ──") + _emit("── response ──") + print(text, file=sys.stderr, flush=True) + _emit(f"── tokens: {usage.get('input', 0):,} in · {usage.get('output', 0):,} out ──") + + +def _verbose_llm_tokens(*, backend: str, model: str, prompt: str, usage: dict) -> None: + """Print one line of per-call token accounting to stderr (``--tokens``). + + The token-economics-only counterpart to ``_verbose_llm_exchange``: same + numbers, none of the prompt/thinking/response bodies. + """ + with _VERBOSE_LOCK: + print( + f"[graphify llm] ── call ── backend={backend} model={model} " + f"prompt={len(prompt):,} chars ── tokens: " + f"{usage.get('input', 0):,} in · {usage.get('output', 0):,} out ──", + file=sys.stderr, flush=True, + ) + + _EXTRACTION_SYSTEM = """\ You are a graphify semantic extraction agent. Extract a knowledge graph fragment from the files provided. Output ONLY valid JSON — no explanation, no markdown fences, no preamble. @@ -1406,6 +1497,45 @@ def _claude_cli_error(stdout: str) -> str: return "unspecified error" +def _claude_cli_stream_result(stdout: str) -> tuple[dict, str | None]: + """Parse `claude -p --output-format stream-json --verbose` stdout (NDJSON). + + Verbose mode uses stream-json instead of json because the plain json + envelope collapses each turn to the final result text, so the assistant + events that carry thinking blocks never appear. Returns ``(envelope, + thinking)`` where envelope is the final ``{"type": "result"}`` event (same + shape `_claude_cli_envelope` normalizes to) and thinking is the + concatenated thinking blocks across assistant turns, or None. + """ + events: list[dict] = [] + for line in stdout.splitlines(): + line = line.strip() + if not line: + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + continue # tolerate non-JSON chatter between events + if isinstance(event, dict): + events.append(event) + result_events = [e for e in events if e.get("type") == "result"] + if not result_events: + raise RuntimeError( + "claude -p stream-json produced no result event; " + f"first 500 chars of stdout: {stdout[:500]!r}" + ) + thinking_parts: list[str] = [] + for event in events: + if event.get("type") != "assistant": + continue + for block in (event.get("message") or {}).get("content") or []: + if isinstance(block, dict) and block.get("type") == "thinking": + part = block.get("thinking") + if part: + thinking_parts.append(str(part)) + return result_events[-1], ("\n".join(thinking_parts) or None) + + # A JSON Schema pinning the top-level shape graphify consumes. Passed to # `claude -p --json-schema` (structured output) so the CLI CONSTRAINS the model # to emit the object directly instead of relying on it CHOOSING to honour a @@ -2568,6 +2698,14 @@ def _call_llm( Previously `graphify.dedup` imported a `_call_llm` symbol that did not exist in this module, so the LLM tiebreaker silently no-op'd on `ImportError` (F-038). Adding the function here re-enables it. + + When verbose tracing is on (``set_llm_verbose`` / ``GRAPHIFY_LLM_VERBOSE``), + the full exchange (prompt, thinking where the backend surfaces it, reply, + and per-call token counts) is printed to stderr. For the ``claude`` + backend verbose mode also enables extended thinking so there is reasoning + to show. Token-only mode (``set_llm_tokens`` / ``GRAPHIFY_LLM_TOKENS``) + prints just the per-call token line and touches nothing about the call + itself — no extended thinking, no stream-json — so it is side-effect free. """ if backend not in BACKENDS: raise ValueError(f"Unknown backend {backend!r}") @@ -2583,28 +2721,58 @@ def _call_llm( ) mdl = model or _default_model_for_backend(backend) + # Per-call usage, tracked separately from the caller's accumulator so + # verbose mode can report THIS exchange's cost even when usage_out + # accumulates across batches. + call_usage: dict = {} + def _rec(inp, out) -> None: + call_usage["input"] = call_usage.get("input", 0) + int(inp or 0) + call_usage["output"] = call_usage.get("output", 0) + int(out or 0) if usage_out is not None: usage_out["input"] = usage_out.get("input", 0) + int(inp or 0) usage_out["output"] = usage_out.get("output", 0) + int(out or 0) + verbose = _llm_verbose() + # Verbose already ends its exchange dump with the token line, so the + # one-liner is only needed when verbose is off. + tokens_only = not verbose and _llm_tokens() + thinking_text: str | None = None + text: str + if backend == "claude": try: import anthropic except ImportError as exc: raise ImportError(_backend_pkg_hint("anthropic", "anthropic")) from exc client = anthropic.Anthropic(api_key=key, base_url=cfg["base_url"], timeout=_resolve_api_timeout(), max_retries=_resolve_max_retries()) - resp = client.messages.create( - model=mdl, - max_tokens=max_tokens, - messages=[{"role": "user", "content": prompt}], - ) + create_kwargs: dict = { + "model": mdl, + "max_tokens": max_tokens, + "messages": [{"role": "user", "content": prompt}], + } + if verbose: + # Extended thinking is off on this plain-text path by default; + # verbose mode turns it on so the trace shows the model's + # reasoning, not just its answer. budget_tokens must be >= 1024 + # and strictly below max_tokens, so small labeling budgets get + # bumped. Thinking tokens are billed as output tokens, exactly + # the accountability verbose mode exists to show. + budget = 1024 + create_kwargs["max_tokens"] = max(max_tokens, budget + 1024) + create_kwargs["thinking"] = {"type": "enabled", "budget_tokens": budget} + resp = client.messages.create(**create_kwargs) u = getattr(resp, "usage", None) if u is not None: _rec(getattr(u, "input_tokens", 0), getattr(u, "output_tokens", 0)) - return resp.content[0].text if resp.content else "" + # With thinking enabled the first content block is the reasoning, so + # select by block type rather than assuming content[0] is text. + thinking_text = "".join( + str(getattr(b, "thinking", "")) for b in resp.content if getattr(b, "type", "") == "thinking" + ) or None + text = "".join(str(getattr(b, "text", "")) for b in resp.content if getattr(b, "type", "") == "text") - if backend == "claude-cli": + elif backend == "claude-cli": import platform, shutil, subprocess # Mirror the extraction-path resolution: on Windows the npm shim is # claude.cmd, which CreateProcess can't resolve from a bare "claude" @@ -2618,7 +2786,12 @@ def _rec(inp, out) -> None: raise RuntimeError("Claude Code CLI not found on $PATH") elif shutil.which("claude") is None: raise RuntimeError("Claude Code CLI not found on $PATH") - cli_args = [claude_cmd, "-p", "--output-format", "json", "--no-session-persistence"] + cli_args = [claude_cmd, "-p", "--output-format", "stream-json" if verbose else "json", "--no-session-persistence"] + if verbose: + # stream-json with -p requires --verbose; it also emits per-turn + # assistant events, which is where thinking blocks appear. The + # plain json envelope keeps only the final result text. + cli_args.append("--verbose") if model is not None: cli_args.extend(["--model", mdl]) proc = subprocess.run( @@ -2632,7 +2805,20 @@ def _rec(inp, out) -> None: check=False, **_no_window_kwargs(), ) - cli_error = _claude_cli_error(proc.stdout) + envelope: dict | None = None + if verbose: + # Parse only after a clean exit: on a hard failure stdout may not + # be NDJSON at all, and stderr carries the real cause. + if proc.returncode != 0: + detail = proc.stderr.strip() or "(no stderr, no result event)" + raise RuntimeError(f"claude -p exited {proc.returncode}: {detail[:500]}") + envelope, thinking_text = _claude_cli_stream_result(proc.stdout) + cli_error = "" + if envelope.get("is_error"): + r = envelope.get("result") + cli_error = r.strip() if isinstance(r, str) and r.strip() else "unspecified error" + else: + cli_error = _claude_cli_error(proc.stdout) if proc.returncode != 0: detail = proc.stderr.strip() or cli_error or "(no stderr, no error envelope)" raise RuntimeError(f"claude -p exited {proc.returncode}: {detail[:500]}") @@ -2640,7 +2826,8 @@ def _rec(inp, out) -> None: # Without this the error text is returned as the model's reply and # the caller writes it into the graph as a community label (#2554). raise RuntimeError(f"claude -p reported an error: {cli_error[:500]}") - envelope = _claude_cli_envelope(proc.stdout) + if envelope is None: + envelope = _claude_cli_envelope(proc.stdout) cli_usage = envelope.get("usage") or {} if cli_usage: _rec( @@ -2649,10 +2836,10 @@ def _rec(inp, out) -> None: + (cli_usage.get("cache_creation_input_tokens", 0) or 0), cli_usage.get("output_tokens", 0), ) - return envelope.get("result", "") + result_text = envelope.get("result", "") + text = result_text if isinstance(result_text, str) else str(result_text or "") - - if backend == "bedrock": + elif backend == "bedrock": try: import boto3 import botocore.config @@ -2677,9 +2864,20 @@ def _rec(inp, out) -> None: bu = resp.get("usage") or {} if bu: _rec(bu.get("inputTokens", 0), bu.get("outputTokens", 0)) - return _bedrock_response_text(resp, default="") + # Reasoning models on Bedrock surface reasoningContent blocks. Nothing + # here enables reasoning, but show it when the deployment was + # configured to return it. + reasoning_parts = [ + str(rc["reasoningText"]["text"]) + for block in (resp.get("output") or {}).get("message", {}).get("content", []) or [] + if isinstance(block, dict) + for rc in [block.get("reasoningContent") or {}] + if isinstance(rc.get("reasoningText"), dict) and rc["reasoningText"].get("text") + ] + thinking_text = "\n".join(reasoning_parts) or None + text = _bedrock_response_text(resp, default="") - if backend == "azure": + elif backend == "azure": endpoint = os.environ.get("AZURE_OPENAI_ENDPOINT", "").strip() if not endpoint: raise ValueError( @@ -2700,44 +2898,60 @@ def _rec(inp, out) -> None: au = getattr(resp, "usage", None) if au is not None: _rec(getattr(au, "prompt_tokens", 0), getattr(au, "completion_tokens", 0)) - return resp.choices[0].message.content or "" + azure_msg = resp.choices[0].message + thinking_text = getattr(azure_msg, "reasoning_content", None) or None + text = azure_msg.content or "" - # OpenAI-compatible (kimi, openai, gemini, ollama) - try: - from openai import OpenAI - except ImportError as exc: - raise ImportError(_backend_pkg_hint("openai", "openai")) from exc - client = OpenAI(api_key=key, base_url=cfg["base_url"], timeout=_resolve_api_timeout(), max_retries=_resolve_max_retries()) - kwargs: dict = { - "model": mdl, - "messages": [{"role": "user", "content": prompt}], - "max_completion_tokens": max_tokens, - # Force a single non-streamed response: some OpenAI-compatible gateways - # default to SSE streaming when `stream` is omitted, but the result here - # is always read as resp.choices[0]. Same fix as _call_openai_compat - # (#1223) — this path feeds the --dedup-llm tiebreaker. - "stream": False, - } - temperature = _resolve_temperature(cfg.get("temperature", 0), mdl) - if temperature is not None: - kwargs["temperature"] = temperature - if cfg.get("reasoning_effort"): - kwargs["reasoning_effort"] = cfg["reasoning_effort"] - # Custom providers can override via providers.json `extra_body`; falls back - # to the moonshot default to preserve existing behavior. - if cfg.get("extra_body") is not None: - kwargs["extra_body"] = cfg["extra_body"] - elif "moonshot" in cfg["base_url"]: - kwargs["extra_body"] = {"thinking": {"type": "disabled"}} - elif _thinking_disabled_via_env(): - kwargs["extra_body"] = {"thinking": {"type": "disabled"}} - resp = client.chat.completions.create(**kwargs) - if not resp.choices or resp.choices[0].message is None: - raise ValueError("LLM returned empty or filtered response") - ou = getattr(resp, "usage", None) - if ou is not None: - _rec(getattr(ou, "prompt_tokens", 0), getattr(ou, "completion_tokens", 0)) - return resp.choices[0].message.content or "" + else: + # OpenAI-compatible (kimi, openai, gemini, ollama, custom providers) + try: + from openai import OpenAI + except ImportError as exc: + raise ImportError(_backend_pkg_hint("openai", "openai")) from exc + client = OpenAI(api_key=key, base_url=cfg["base_url"], timeout=_resolve_api_timeout(), max_retries=_resolve_max_retries()) + kwargs: dict = { + "model": mdl, + "messages": [{"role": "user", "content": prompt}], + "max_completion_tokens": max_tokens, + # Force a single non-streamed response: some OpenAI-compatible gateways + # default to SSE streaming when `stream` is omitted, but the result here + # is always read as resp.choices[0]. Same fix as _call_openai_compat + # (#1223); this path feeds the --dedup-llm tiebreaker. + "stream": False, + } + temperature = _resolve_temperature(cfg.get("temperature", 0), mdl) + if temperature is not None: + kwargs["temperature"] = temperature + if cfg.get("reasoning_effort"): + kwargs["reasoning_effort"] = cfg["reasoning_effort"] + # Custom providers can override via providers.json `extra_body`; falls back + # to the moonshot default to preserve existing behavior. + if cfg.get("extra_body") is not None: + kwargs["extra_body"] = cfg["extra_body"] + elif "moonshot" in cfg["base_url"]: + kwargs["extra_body"] = {"thinking": {"type": "disabled"}} + elif _thinking_disabled_via_env(): + kwargs["extra_body"] = {"thinking": {"type": "disabled"}} + resp = client.chat.completions.create(**kwargs) + if not resp.choices or resp.choices[0].message is None: + raise ValueError("LLM returned empty or filtered response") + ou = getattr(resp, "usage", None) + if ou is not None: + _rec(getattr(ou, "prompt_tokens", 0), getattr(ou, "completion_tokens", 0)) + msg = resp.choices[0].message + # DeepSeek/Kimi-style reasoning is returned out of band as + # reasoning_content; surface it when present. + thinking_text = getattr(msg, "reasoning_content", None) or None + text = msg.content or "" + + if verbose: + _verbose_llm_exchange( + backend=backend, model=mdl, prompt=prompt, + thinking=thinking_text, text=text, usage=call_usage, + ) + elif tokens_only: + _verbose_llm_tokens(backend=backend, model=mdl, prompt=prompt, usage=call_usage) + return text def estimate_cost(backend: str, input_tokens: int, output_tokens: int) -> float: diff --git a/tests/test_llm_backends.py b/tests/test_llm_backends.py index 9a9f4a2a1..1d9d7fd6e 100644 --- a/tests/test_llm_backends.py +++ b/tests/test_llm_backends.py @@ -1130,7 +1130,10 @@ def test_call_llm_claude_client_built_with_timeout_and_retries(monkeypatch): class _FakeMessages: def create(self, **_): - return types.SimpleNamespace(content=[types.SimpleNamespace(text="ok")]) + # Real Anthropic content blocks always carry a `type`; _call_llm + # selects text blocks by it (thinking blocks precede the reply + # when extended thinking is on). + return types.SimpleNamespace(content=[types.SimpleNamespace(type="text", text="ok")]) class _FakeAnthropic: def __init__(self, *_, **kwargs): diff --git a/tests/test_llm_verbose.py b/tests/test_llm_verbose.py new file mode 100644 index 000000000..c7b45c7a0 --- /dev/null +++ b/tests/test_llm_verbose.py @@ -0,0 +1,329 @@ +"""Tests for verbose LLM-call tracing (cluster-only --verbose / GRAPHIFY_LLM_VERBOSE) +and token-economics-only mode (--tokens / GRAPHIFY_LLM_TOKENS). + +Backend calls are faked; no network. Covers the verbose/tokens toggles, the +exchange and token-line printers, Anthropic thinking capture, the claude-cli +stream-json parse, and the cluster-only CLI wiring including the label-run +token total. +""" +import json +import sys +import types + +import graphify.llm as llm +from graphify.llm import ( + _call_llm, + _claude_cli_stream_result, + _verbose_llm_exchange, + _verbose_llm_tokens, +) + + +def _fake_anthropic(captured, *, thinking="reasoning here", text='{"0": "Order Management"}'): + """A minimal stand-in for the `anthropic` package.""" + class _Block: + def __init__(self, type_, **fields): + self.type = type_ + self.__dict__.update(fields) + + class _Usage: + input_tokens = 10 + output_tokens = 20 + + class _Response: + content = [ + _Block("thinking", thinking=thinking), + _Block("text", text=text), + ] + usage = _Usage() + + class _Messages: + def create(self, **kwargs): + captured.update(kwargs) + return _Response() + + class _Client: + def __init__(self, **kwargs): + captured["client_kwargs"] = kwargs + + messages = _Messages() + + return types.SimpleNamespace(Anthropic=_Client) + + +def test_llm_verbose_toggle(monkeypatch): + monkeypatch.delenv("GRAPHIFY_LLM_VERBOSE", raising=False) + monkeypatch.setattr(llm, "_LLM_VERBOSE", False) + assert not llm._llm_verbose() + monkeypatch.setenv("GRAPHIFY_LLM_VERBOSE", "1") + assert llm._llm_verbose() + monkeypatch.delenv("GRAPHIFY_LLM_VERBOSE") + llm.set_llm_verbose(True) + assert llm._llm_verbose() + + +def test_verbose_exchange_prints_all_sections(capsys): + _verbose_llm_exchange( + backend="claude", model="claude-sonnet-4-6", prompt="PROMPT BODY", + thinking="THINKING BODY", text="RESPONSE BODY", + usage={"input": 1234, "output": 56}, + ) + err = capsys.readouterr().err + assert "backend=claude" in err + assert "PROMPT BODY" in err + assert "THINKING BODY" in err + assert "RESPONSE BODY" in err + assert "1,234 in · 56 out" in err + + +def test_verbose_exchange_marks_missing_thinking(capsys): + _verbose_llm_exchange( + backend="ollama", model="qwen", prompt="p", + thinking=None, text="t", usage={"input": 1, "output": 2}, + ) + err = capsys.readouterr().err + assert "(none returned by backend)" in err + + +def test_call_llm_claude_verbose_enables_thinking(monkeypatch, capsys): + captured = {} + monkeypatch.setitem(sys.modules, "anthropic", _fake_anthropic(captured)) + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test") + monkeypatch.delenv("GRAPHIFY_LLM_VERBOSE", raising=False) + monkeypatch.setattr(llm, "_LLM_VERBOSE", True) + + text = _call_llm("name this community", backend="claude", max_tokens=300) + + # Thinking enabled, budget >= 1024 and max_tokens bumped above it. + assert captured["thinking"] == {"type": "enabled", "budget_tokens": 1024} + assert captured["max_tokens"] == 2048 + # The reply is the TEXT block, not the thinking block that precedes it. + assert text == '{"0": "Order Management"}' + err = capsys.readouterr().err + assert "reasoning here" in err + assert "── response ──" in err + assert "10 in · 20 out" in err + + +def test_call_llm_claude_non_verbose_unchanged(monkeypatch, capsys): + captured = {} + monkeypatch.setitem(sys.modules, "anthropic", _fake_anthropic(captured)) + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test") + monkeypatch.delenv("GRAPHIFY_LLM_VERBOSE", raising=False) + monkeypatch.setattr(llm, "_LLM_VERBOSE", False) + + text = _call_llm("name this community", backend="claude", max_tokens=300) + + assert "thinking" not in captured + assert captured["max_tokens"] == 300 + assert text == '{"0": "Order Management"}' + assert "[graphify llm]" not in capsys.readouterr().err + + +def test_claude_cli_stream_result_extracts_thinking(): + events = [ + {"type": "system", "subtype": "init"}, + {"type": "assistant", "message": {"content": [ + {"type": "thinking", "thinking": "hmm"}, + {"type": "text", "text": "partial"}, + ]}}, + {"type": "assistant", "message": {"content": [ + {"type": "thinking", "thinking": "more"}, + ]}}, + {"type": "result", "result": '{"0": "X"}', "is_error": False, + "usage": {"input_tokens": 5, "output_tokens": 7}}, + ] + stdout = "\n".join(json.dumps(e) for e in events) + + envelope, thinking = _claude_cli_stream_result(stdout) + + assert envelope["result"] == '{"0": "X"}' + assert envelope["usage"]["output_tokens"] == 7 + assert thinking == "hmm\nmore" + + +def test_claude_cli_stream_result_requires_result_event(): + import pytest + with pytest.raises(RuntimeError, match="no result event"): + _claude_cli_stream_result('{"type": "system"}\n{"type": "assistant"}') + + +def test_label_cli_verbose_traces_and_totals(tmp_path, monkeypatch, capsys): + import graphify.__main__ as cli + + out = tmp_path / "graphify-out" + out.mkdir() + graph = { + "directed": False, + "multigraph": False, + "nodes": [{"id": "n1", "label": "OrderService", "community": 0}], + "links": [], + } + (out / "graph.json").write_text(json.dumps(graph), encoding="utf-8") + + def fake_generate(G, communities, *, backend=None, model=None, gods=None, + quiet=False, max_concurrency=4, batch_size=100, usage_out=None): + if usage_out is not None: + usage_out["input"] = 100 + usage_out["output"] = 42 + return {0: "Orders"}, "llm" + + monkeypatch.setattr("graphify.llm.generate_community_labels", fake_generate) + monkeypatch.setattr("graphify.export.to_html", lambda *args, **kwargs: None) + # Restore the module global after the run even though the CLI flips it. + monkeypatch.setattr(llm, "_LLM_VERBOSE", False) + monkeypatch.setattr( + sys, "argv", + ["graphify", "label", str(tmp_path), "--backend", "claude", "--verbose", "--no-viz"], + ) + + cli.main() + + assert llm._LLM_VERBOSE is True + err = capsys.readouterr().err + assert "label run total: 100 in · 42 out" in err + # claude pricing: (100*3 + 42*15) / 1e6 = $0.00093 + assert "~$0.0009 @ claude" in err + + +def test_llm_tokens_toggle(monkeypatch): + monkeypatch.delenv("GRAPHIFY_LLM_TOKENS", raising=False) + monkeypatch.setattr(llm, "_LLM_TOKENS", False) + assert not llm._llm_tokens() + monkeypatch.setenv("GRAPHIFY_LLM_TOKENS", "1") + assert llm._llm_tokens() + monkeypatch.delenv("GRAPHIFY_LLM_TOKENS") + llm.set_llm_tokens(True) + assert llm._llm_tokens() + + +def test_tokens_line_prints_counts_only(capsys): + _verbose_llm_tokens( + backend="claude", model="claude-sonnet-4-6", + prompt="PROMPT BODY", usage={"input": 1234, "output": 56}, + ) + err = capsys.readouterr().err + assert "backend=claude" in err + assert "1,234 in · 56 out" in err + # The one-liner never leaks bodies. + assert "PROMPT BODY" not in err + assert "── prompt ──" not in err + + +def test_call_llm_claude_tokens_mode_is_side_effect_free(monkeypatch, capsys): + captured = {} + monkeypatch.setitem(sys.modules, "anthropic", _fake_anthropic(captured)) + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test") + monkeypatch.delenv("GRAPHIFY_LLM_VERBOSE", raising=False) + monkeypatch.delenv("GRAPHIFY_LLM_TOKENS", raising=False) + monkeypatch.setattr(llm, "_LLM_VERBOSE", False) + monkeypatch.setattr(llm, "_LLM_TOKENS", True) + + text = _call_llm("name this community", backend="claude", max_tokens=300) + + # Unlike verbose, tokens mode does not shape the call: no extended + # thinking, max_tokens untouched. + assert "thinking" not in captured + assert captured["max_tokens"] == 300 + assert text == '{"0": "Order Management"}' + err = capsys.readouterr().err + assert "10 in · 20 out" in err + # …and no bodies are dumped. + assert "name this community" not in err + assert "reasoning here" not in err + + +def test_call_llm_verbose_takes_precedence_over_tokens(monkeypatch, capsys): + captured = {} + monkeypatch.setitem(sys.modules, "anthropic", _fake_anthropic(captured)) + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test") + monkeypatch.delenv("GRAPHIFY_LLM_VERBOSE", raising=False) + monkeypatch.delenv("GRAPHIFY_LLM_TOKENS", raising=False) + monkeypatch.setattr(llm, "_LLM_VERBOSE", True) + monkeypatch.setattr(llm, "_LLM_TOKENS", True) + + _call_llm("name this community", backend="claude", max_tokens=300) + + err = capsys.readouterr().err + # Full exchange (which already carries the token line), not the one-liner. + assert "── prompt ──" in err + assert "── response ──" in err + + +def test_label_cli_tokens_prints_run_total(tmp_path, monkeypatch, capsys): + import graphify.__main__ as cli + + out = tmp_path / "graphify-out" + out.mkdir() + graph = { + "directed": False, + "multigraph": False, + "nodes": [{"id": "n1", "label": "OrderService", "community": 0}], + "links": [], + } + (out / "graph.json").write_text(json.dumps(graph), encoding="utf-8") + + def fake_generate(G, communities, *, backend=None, model=None, gods=None, + quiet=False, max_concurrency=4, batch_size=100, usage_out=None): + if usage_out is not None: + usage_out["input"] = 100 + usage_out["output"] = 42 + return {0: "Orders"}, "llm" + + monkeypatch.setattr("graphify.llm.generate_community_labels", fake_generate) + monkeypatch.setattr("graphify.export.to_html", lambda *args, **kwargs: None) + # Restore the module global after the run even though the CLI flips it. + monkeypatch.setattr(llm, "_LLM_TOKENS", False) + monkeypatch.setattr( + sys, "argv", + ["graphify", "label", str(tmp_path), "--backend", "claude", "--tokens", "--no-viz"], + ) + + cli.main() + + assert llm._LLM_TOKENS is True + assert llm._LLM_VERBOSE is False + err = capsys.readouterr().err + assert "label run total: 100 in · 42 out" in err + assert "~$0.0009 @ claude" in err + + +def test_label_cli_env_tokens_prints_run_total(tmp_path, monkeypatch, capsys): + """GRAPHIFY_LLM_TOKENS=1 with NO --tokens flag must still print the run + total: the per-call hook reads the env var, so gating the total on the CLI + flags alone would silently drop it on the env-only path.""" + import graphify.__main__ as cli + + out = tmp_path / "graphify-out" + out.mkdir() + graph = { + "directed": False, + "multigraph": False, + "nodes": [{"id": "n1", "label": "OrderService", "community": 0}], + "links": [], + } + (out / "graph.json").write_text(json.dumps(graph), encoding="utf-8") + + def fake_generate(G, communities, *, backend=None, model=None, gods=None, + quiet=False, max_concurrency=4, batch_size=100, usage_out=None): + if usage_out is not None: + usage_out["input"] = 100 + usage_out["output"] = 42 + return {0: "Orders"}, "llm" + + monkeypatch.setattr("graphify.llm.generate_community_labels", fake_generate) + monkeypatch.setattr("graphify.export.to_html", lambda *args, **kwargs: None) + monkeypatch.setattr(llm, "_LLM_TOKENS", False) + monkeypatch.setattr(llm, "_LLM_VERBOSE", False) + monkeypatch.setenv("GRAPHIFY_LLM_TOKENS", "1") + monkeypatch.delenv("GRAPHIFY_LLM_VERBOSE", raising=False) + monkeypatch.setattr( + sys, "argv", + ["graphify", "label", str(tmp_path), "--backend", "claude", "--no-viz"], + ) + + cli.main() + + err = capsys.readouterr().err + assert "label run total: 100 in · 42 out" in err + assert "~$0.0009 @ claude" in err