diff --git a/.cspell.config.yaml b/.cspell.config.yaml index 4674295e405..466769add88 100644 --- a/.cspell.config.yaml +++ b/.cspell.config.yaml @@ -18,6 +18,7 @@ dictionaries: ignoreRegExpList: - /\b[rs][1-9A-HJ-NP-Za-km-z]{25,34}/g # addresses and seeds - /\bC[A-Z0-9]{15}/g # CTIDs + - /\bXRPL_METRIC_[A-Z_]+/g # telemetry macro names; the splitter emits subwords like ISTOGRAM - /\b(XRPL|BEAST)_[A-Z_0-9]+_H_INCLUDED+/g # include guards - /\b(XRPL|BEAST)_[A-Z_0-9]+_H+/g # include guards - /::[a-z:_]+/g # things from other namespaces @@ -350,6 +351,7 @@ words: - txns - txqueue - txs + - txset - ubsan - UBSAN - ufdio diff --git a/.github/scripts/levelization/results/loops.txt b/.github/scripts/levelization/results/loops.txt index 649f1c7b95a..f4666bf6c3f 100644 --- a/.github/scripts/levelization/results/loops.txt +++ b/.github/scripts/levelization/results/loops.txt @@ -14,5 +14,5 @@ Loop: xrpld.overlay xrpld.rpc xrpld.rpc ~= xrpld.overlay Loop: xrpld.overlay xrpld.telemetry - xrpld.telemetry ~= xrpld.overlay + xrpld.overlay > xrpld.telemetry diff --git a/.github/scripts/levelization/results/ordering.txt b/.github/scripts/levelization/results/ordering.txt index d9c4f24ecf2..c19632a4728 100644 --- a/.github/scripts/levelization/results/ordering.txt +++ b/.github/scripts/levelization/results/ordering.txt @@ -193,6 +193,7 @@ tests.libxrpl > xrpl.config tests.libxrpl > xrpl.consensus tests.libxrpl > xrpl.core tests.libxrpl > xrpld.app +tests.libxrpl > xrpld.overlay tests.libxrpl > xrpld.telemetry tests.libxrpl > xrpl.json tests.libxrpl > xrpl.ledger @@ -334,6 +335,7 @@ xrpld.telemetry > xrpl.consensus xrpld.telemetry > xrpl.core xrpld.telemetry > xrpld.core xrpld.telemetry > xrpl.json +xrpld.telemetry > xrpl.ledger xrpld.telemetry > xrpl.nodestore xrpld.telemetry > xrpl.protocol xrpld.telemetry > xrpl.rdb diff --git a/.github/scripts/otel-naming/README.md b/.github/scripts/otel-naming/README.md index 660b574ed06..d0a6b9f6033 100644 --- a/.github/scripts/otel-naming/README.md +++ b/.github/scripts/otel-naming/README.md @@ -35,18 +35,29 @@ hardcoded allowlist: constants passed there (`xrpl.network.*`). A dotted key that is _declared_ in a header but never set as a resource attr is a span attribute in resource clothing — a Rule-A violation, even if it lives in the base `SpanNames.h`. +- **L1-metrics** — instrument names, label keys and bounded label values come + from the `namespace metric` / `namespace label` / `namespace lval` blocks of + every `*MetricNames.h`, read as `inline constexpr char NAME[] = "wire";`. + These headers deliberately do **not** use the `makeStr`/`StaticStr` DSL the + span headers use: the OTel C++ API takes `nostd::string_view`, which + constructs from `char const*` but has no constructor from + `std::string_view`, so a `StaticStr` will not compile in an instrument-name + or label-key position. ### Rules (each fails the build, when its inputs are present) -| Rule | Check | -| ---- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| A | No stray dotted span-attribute key (only the derived resource keys may be dotted). | -| G | Attribute keys are `lower_snake_case` (`^[a-z][a-z0-9_]*$` per dot-segment) — no camelCase, UPPERCASE, or spaces. | -| F | No string literals as attribute keys or span-name arguments in `setAttribute`/`addEvent`/`span`/`rootSpan`/`childSpan` (`rootSpan` shares `span`'s `(cat, prefix, name)` signature). Attribute _values_ are exempt (runtime data); `*SpanNames.h` definitions and test files are exempt. | -| B | Every collector `spanmetrics.dimensions` name exists in the L1 key set. | -| C | Every Tempo span-filter tag exists in the L1 key set. | -| D | Every dashboard label resolves to an L1 span attribute, a native-metric label (L6, emitted by MetricsRegistry), or a Prometheus/Grafana builtin. TraceQL scope prefixes (`span.`/`resource.`/…) are stripped before the L1 lookup. | -| E | No dotted `xrpl..` attribute key in the runbook (only the L1 resource attrs `xrpl.network.*` may be dotted). Span names, filenames, OTel-standard keys, and metric labels are not flagged. | +| Rule | Check | +| ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| A | No stray dotted span-attribute key (only the derived resource keys may be dotted). | +| G | Attribute keys are `lower_snake_case` (`^[a-z][a-z0-9_]*$` per dot-segment) — no camelCase, UPPERCASE, or spaces. | +| F | No string literals as attribute keys or span-name arguments in `setAttribute`/`addEvent`/`span`/`rootSpan`/`childSpan` (`rootSpan` shares `span`'s `(cat, prefix, name)` signature). Attribute _values_ are exempt (runtime data); `*SpanNames.h` definitions and test files are exempt. | +| B | Every collector `spanmetrics.dimensions` name exists in the L1 key set. | +| C | Every Tempo span-filter tag exists in the L1 key set. | +| D | Every dashboard label resolves to an L1 span attribute, a native-metric label (L6, emitted by MetricsRegistry), or a Prometheus/Grafana builtin. TraceQL scope prefixes (`span.`/`resource.`/…) are stripped before the L1 lookup. | +| E | No dotted `xrpl..` attribute key in the runbook (only the L1 resource attrs `xrpl.network.*` may be dotted). Span names, filenames, OTel-standard keys, and metric labels are not flagged. | +| I | No string literals as **metric** instrument names or label keys — the mirror of Rule F. Applies to the name passed to an `XRPL_METRIC_*` macro or a `meter->Create*` factory and to the label _keys_ in its label set. Label _values_, descriptions, `*MetricNames.h`, `MetricMacros.h` and test files are exempt. Scoped by metric **family** (first underscore segment): declaring a constant opts that family in, so the metric surface can be converted subsystem by subsystem. Unconverted families warn as Rule L. | +| J | Metric instrument names follow the suffix conventions: `lower_snake_case`, no `xrpld_`/`xrpl_` prefix (the exporter adds it), a counter ends `_total`, a histogram ends `_us`/`_ms`/`_seconds`, a gauge does not end `_total`. The instrument **kind** is read from the emit site, never guessed from words in the name — so a multi-series gauge carrying units in its label values (e.g. `nodestore_state` observing `write_mean_us`) is not a violation. A name created through two different factories is itself reported as a kind conflict, since no suffix can be correct for both. | +| K | Every metric named in `docker/telemetry/workload/expected_metrics.json` resolves to a declared constant, so a rename in code cannot leave the workload validator asserting a name nothing emits. PromQL selectors (`m{label="v"}`) and exporter-appended histogram suffixes (`_bucket`/`_count`/`_sum`) are normalized away first; groups fed by another emit path (`statsd_gauges`, `statsd_counters`, `spanmetrics`) are out of scope by design. | Rule F runs **unconditionally** (it is a purely syntactic check on the call-sites and needs no `*SpanNames.h`), so a code path that calls @@ -58,6 +69,7 @@ still caught. | Rule | Check | | ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | H | A namespace-qualified constant (e.g. `foo::bar::myKey`) used at a telemetry call-site is not defined in any `*SpanNames.h`. The constant should live in the proper header; defining it in-place bypasses rules A/G/F. Warns rather than fails — the argument may be a legitimately dynamic value, and the header may live on a later branch. Bare locals and `std::` names are not warned. | +| L | A literal metric name in a family that has no `*MetricNames.h` constants yet. Rule I's ratchet defers these instead of failing the build on the whole pre-existing metric surface at once; the warning keeps the outstanding conversion work visible rather than silently accepted. | ## Presence-gated diff --git a/.github/scripts/otel-naming/check_otel_naming.py b/.github/scripts/otel-naming/check_otel_naming.py index 49a4e606a7c..0ae885649fd 100644 --- a/.github/scripts/otel-naming/check_otel_naming.py +++ b/.github/scripts/otel-naming/check_otel_naming.py @@ -5,10 +5,12 @@ This script takes no parameters and can be called from any directory inside the repository (it locates the repo root via `git rev-parse`). -Enforces the OpenTelemetry span-attribute naming convention documented in -CONTRIBUTING.md ("Telemetry span attribute naming") across every layer of the -telemetry pipeline. The `*SpanNames.h` constants are the single source of truth -(L1); every other layer must agree with them. +Enforces the OpenTelemetry naming conventions documented in CONTRIBUTING.md +("Telemetry span attribute naming" and "Telemetry metric naming") across every +layer of the telemetry pipeline. The `*SpanNames.h` constants are the single +source of truth for span attributes (L1) and the `*MetricNames.h` constants for +metric names and label keys (L1-metrics); every other layer must agree with +them. Design principles ----------------- @@ -49,6 +51,10 @@ L5 runbook : docs/telemetry-runbook.md (attr tables) L6 metrics : MetricsRegistry.cpp instrument labels (native-metric label keys, a valid dashboard-label source besides L1) + L1-metrics : src/**/*MetricNames.h, include/**/*MetricNames.h (ground truth + for metric instrument names, label keys and bounded values) + L7 workload : docker/telemetry/workload/expected_metrics.json (asserted + metric names) Rules (each FAILS the build, when its inputs are present) --------------------------------------------------------- @@ -71,6 +77,24 @@ form -- xrpl.work.item/.branch/.node.role -- may be dotted). Span names, filenames, OTel-standard keys, and metric labels are not flagged. + I No string literals as METRIC instrument names or label keys. The mirror of + Rule F for metrics: the name passed to an XRPL_METRIC_* macro or to a + meter->Create* factory, and the label KEYS in its label set, must reference + a *MetricNames.h constant. Label VALUES are exempt (runtime data), as are + the descriptions, *MetricNames.h itself, MetricMacros.h, and test files. + Scoped by metric FAMILY (first underscore segment) so the metric surface + can be converted subsystem by subsystem -- declaring a constant for a + family opts that family into enforcement. See Rule L. + J Metric instrument names follow the naming and suffix conventions: + lower_snake_case, no xrpld_/xrpl_ prefix (the exporter adds it), a counter + ends in _total, a histogram ends in _us/_ms/_seconds, and a gauge does not + end in _total. The instrument KIND is read from the emit site, never + guessed from words in the name. + K Every metric named in expected_metrics.json resolves to a *MetricNames.h + constant, so a rename in code cannot leave the workload validator + asserting a name nothing emits. PromQL selectors and exporter-appended + histogram suffixes are normalized away first; groups fed by another emit + path (statsd_*, spanmetrics) are out of scope. Warnings (printed, but do NOT fail the build) ---------------------------------------------- @@ -79,11 +103,16 @@ *SpanNames.h (single source of truth); defining one in-place bypasses the naming rules. A warning (not a failure) because the argument may instead be a legitimately dynamic local (e.g. a computed span-name leaf). + L A literal metric name in a family that has no *MetricNames.h constants + yet. Rule I's ratchet defers these rather than failing the build on the + whole pre-existing metric surface at once; the warning is what keeps the + outstanding conversion work visible instead of silently accepted. Exit code is non-zero if any present-and-enforced rule finds a violation. Warnings never change the exit code. """ +import json import re import subprocess import sys @@ -161,12 +190,22 @@ def read_source(path: Path) -> str: # label set — they are not labels and must not license a dashboard filter. METRIC_LABEL_MAP = re.compile(r"\{\{(.*?)\}\}", re.DOTALL) # One key inside such a map: a string literal, or a `kFooLabel` constant name -# resolved through LABEL_CONST_DEF. Inside the map, each pair opens with `{`, -# except the first, whose `{` was consumed by the map's own `{{`. +# resolved through LABEL_CONST_DEF. A label set is a nested initializer list +# (`{{"a", x}, {"b", y}}`), so inside the map each pair opens with `{` except +# the first, whose `{` was consumed by the map's own `{{` — hence the `^` +# alternative. Matching only the doubled-brace form would derive just the first +# label of every multi-label instrument, silently under-deriving the L6 key set +# and making Rule D reject a dashboard that queries a label the code genuinely +# emits. METRIC_LABEL = re.compile(r'(?:^|\{)\s*"([a-z_][a-z0-9_]*)"\s*,') METRIC_LABEL_CONST = re.compile(r"(?:^|\{)\s*(k[A-Za-z0-9_]*)\s*,") # A label-key constant definition, with or without `inline`: # `constexpr char kHandlerLabel[] = "handler";` +# +# This resolves the flat `k`-prefixed per-subsystem header style. The +# `namespace label` style used by MetricNames.h is covered separately, by +# metric_constants(), because those identifiers are not `k`-prefixed and are +# referenced at call sites as `label::jobType` — see metric_label_names(). LABEL_CONST_DEF = re.compile( r'(?:inline\s+)?constexpr\s+char\s+(k[A-Za-z0-9_]*)\s*\[\s*\]\s*=\s*"([a-z_][a-z0-9_]*)"' ) @@ -478,6 +517,24 @@ def main() -> None: run_rule_d_dashboards(root, l1_keys, metric_labels, report) run_rule_e_runbook(root, l1_keys, report) + # --- Metric rules I/J/K -------------------------------------------------- + # The metric-name counterpart of the span rules above. Rule I is the mirror + # of Rule F (no literals at an emit site) and, like it, runs + # unconditionally because it is purely syntactic. Rules J and K are + # presence-gated on *MetricNames.h existing. + l1_metric_names, l1_metric_labels, _ = metric_constants(root) + if find_metricname_headers(root): + report.ok( + f"L1-metrics: {len(l1_metric_names)} instrument name(s) and " + f"{len(l1_metric_labels)} label key(s) from " + f"{len(find_metricname_headers(root))} *MetricNames.h header(s)" + ) + else: + report.skip("L1-metrics", "no *MetricNames.h present") + run_rule_i_metric_literals(root, report) + run_rule_j_metric_suffixes(root, report) + run_rule_k_expected_metrics(root, report) + report.render_and_exit() @@ -814,10 +871,23 @@ def metric_label_names(root: Path) -> Set[str]: `counter->Add(1, {{"job_type", value}})` in MetricsRegistry.cpp. These are a valid source of dashboard labels distinct from span attributes (L1). + Collects both the first label of a set and every subsequent one, so a + multi-label instrument such as `{{"from", a}, {"to", b}}` contributes ALL + of its keys. + A label key may be written inline as a string literal or hoisted into a named constant (`constexpr char kHandlerLabel[] = "handler";`, then `{{kHandlerLabel, value}}`). Both forms are collected, and the constants - may be declared in a header, so headers are scanned as well as sources.""" + may be declared in a header, so headers are scanned as well as sources. + + Also includes the keys declared as constants in `*MetricNames.h` + (L1-metrics), because Rule I requires call sites to reference a constant + rather than a literal — so once a subsystem is converted, its label keys no + longer appear as literals anywhere and a literal-only scan would + under-derive the set and make Rule D reject a dashboard querying a label + the code really emits. That covers the `namespace label` header style, + whose identifiers are not `k`-prefixed and so are invisible to + LABEL_CONST_DEF; the two derivations are complementary.""" labels: Set[str] = set() # Constant name -> label string, gathered across every scanned file so a # `{{kFooLabel, ...}}` call site resolves even when the definition lives in @@ -842,9 +912,109 @@ def metric_label_names(root: Path) -> Set[str]: labels |= set(METRIC_LABEL.findall(label_map)) used_constants |= set(METRIC_LABEL_CONST.findall(label_map)) labels |= {constants[name] for name in used_constants if name in constants} + labels |= metric_constants(root)[1] return labels +# --------------------------------------------------------------------------- +# L1-metrics: parse *MetricNames.h into the authoritative metric-name/label set +# --------------------------------------------------------------------------- + +# A metric-name constant definition: `inline constexpr char NAME[] = "wire";`. +# The metric headers use `constexpr char[]` rather than the `makeStr`/StaticStr +# DSL the span headers use, because the OTel C++ API takes nostd::string_view, +# which constructs from `char const*` but NOT from std::string_view. +METRIC_CONST_DEF = re.compile( + r"inline\s+constexpr\s+char\s+(\w+)\s*\[\s*\]\s*=\s*\"([^\"]*)\"\s*;" +) + + +def find_metricname_headers(root: Path) -> List[Path]: + """Every `*MetricNames.h` in the tree (the metric-side counterpart of + `find_spanname_headers`).""" + return sorted( + p + for p in list((root / "src").rglob("*MetricNames.h")) + + list((root / "include").rglob("*MetricNames.h")) + if p.is_file() + ) + + +def metric_constants(root: Path) -> Tuple[Set[str], Set[str], Set[str]]: + """Return (instrument_names, label_keys, label_values) declared across the + `*MetricNames.h` headers, split by which namespace block each constant sits + in: `namespace metric` -> instrument names, `namespace label` -> label keys, + `namespace lval` -> bounded label values. + + Comments are stripped first, so an illustrative `@code` example in a doc + comment cannot seed the authoritative set (same reasoning as + `strip_comments` for L1 spans). + + Two header styles are recognised, because both are in use: + + * Namespaced: constants sit inside `namespace metric` / `label` / `lval`, + and the enclosing namespace decides the bucket. + * Flat `k`-prefixed: a header with no such namespaces names the role in the + identifier instead -- `kLabelFoo` is a label key, `kResultFoo` and + `kReasonFoo` are label values. Used by the per-subsystem headers. + + A constant that neither sits in one of the three namespaces nor carries a + recognised prefix is ignored rather than guessed at, keeping the derivation + conservative in the same direction as L1.""" + names: Set[str] = set() + keys: Set[str] = set() + values: Set[str] = set() + for h in find_metricname_headers(root): + text = strip_comments(read_source(h)) + flat = text + for ns, bucket in ( + ("metric", names), + ("label", keys), + ("lval", values), + ): + for block in namespace_spans(text, ns): + for m in METRIC_CONST_DEF.finditer(block): + bucket.add(m.group(2)) + # Remove the block before the flat pass below, so the + # enclosing namespace stays the only thing that decides a + # namespaced constant's bucket. Without this, a `kLabel`-style + # identifier written inside `namespace metric` lands in both + # `names` and `keys`, and an instrument name silently becomes + # an allowed label key for Rule D. + flat = flat.replace(block, "", 1) + # Flat style: classify by identifier prefix. Only constants outside the + # namespaced blocks reach here, so a namespaced header is unaffected. + for m in METRIC_CONST_DEF.finditer(flat): + ident, value = m.group(1), m.group(2) + if ident.startswith("kLabel"): + keys.add(value) + elif ident.startswith(("kResult", "kReason")): + values.add(value) + return names, keys, values + + +def namespace_spans(text: str, want: str) -> List[str]: + """Return the body text of each `namespace { ... }` block, brace + matched so nested namespaces (e.g. `lval::dns_resolve`) are contained in + their parent's span. Generalizes `attr_namespace_spans`, which is hardcoded + to `attr`.""" + spans: List[str] = [] + for opener in NS_OPEN.finditer(text): + if opener.group(1).split("::")[-1] != want: + continue + i = opener.end() + depth, start = 1, i + while i < len(text) and depth > 0: + c = text[i] + if c == "{": + depth += 1 + elif c == "}": + depth -= 1 + i += 1 + spans.append(text[start : i - 1]) + return spans + + # Identity labels stamped by EXTERNAL infrastructure the OTel pipeline in this # repo does not own: the perf-iac harness attaches these to every metric it # scrapes so dashboards can filter by which build/role produced a series. They @@ -961,5 +1131,449 @@ def run_rule_e_runbook(root: Path, l1_keys: Set[str], report: Report) -> None: report.ok("E: runbook attribute references consistent with L1") +# --------------------------------------------------------------------------- +# Metric rules I / J / K +# --------------------------------------------------------------------------- + +# A metric emit site whose name argument must be a constant. Two families: +# * the XRPL_METRIC_* call-site macros, where arg0 is the app/registry and +# arg1 is the instrument name; +# * the OTel meter factories, where arg0 is the instrument name. +# Matching the macro by name (rather than by receiver, as CALLSITE does) is +# sufficient because the XRPL_METRIC_ prefix is unambiguous in this repo. +METRIC_MACRO_CALL = re.compile(r"\bXRPL_METRIC_([A-Z_]+)\s*\(") +METRIC_FACTORY_CALL = re.compile( + r"\bCreate(?:UInt64|Int64|Double)" + r"(?:Counter|UpDownCounter|Histogram|Gauge|ObservableGauge|" + r"ObservableCounter|ObservableUpDownCounter)\s*\(" +) +# A `{ KEY , ...` label pair opener inside an already-isolated argument list. +# KEY is what Rule I checks; the VALUE after the comma is exempt (runtime data). +METRIC_PAIR_KEY = re.compile(r"\{\s*(\"(?:[^\"\\]|\\.)*\"|[A-Za-z_][\w:]*)\s*,") + +# The unit/kind suffix convention every instrument name must satisfy (Rule J). +# A cumulative counter reads correctly under rate() only if a reader can tell it +# from a gauge, and a duration is ambiguous unless it carries its unit -- the +# OTel `unit` argument is not surfaced on the Prometheus metric name. +METRIC_COUNTER_SUFFIX = "_total" +METRIC_DURATION_SUFFIXES = ("_us", "_ms", "_seconds") + + +def metric_calls(text: str): + """Yield (kind, name_index, arglist, lineno) for every metric emit site. + + `kind` is the macro suffix (e.g. `COUNTER_INC_LABELED`) for a macro call, or + the factory method name for a `meter->Create*` call. `name_index` is which + top-level argument holds the instrument name.""" + for m in METRIC_MACRO_CALL.finditer(text): + arglist = balanced_arglist(text, m.end()) + yield m.group(1), 1, arglist, text.count("\n", 0, m.start()) + 1 + for m in METRIC_FACTORY_CALL.finditer(text): + arglist = balanced_arglist(text, m.end()) + name = m.group(0).rstrip("( \t\n") + yield name, 0, arglist, text.count("\n", 0, m.start()) + 1 + + +def balanced_arglist(text: str, start: int) -> str: + """Return the argument-list text of a call whose '(' ends at `start`-1, + balancing nesting and ignoring parens inside string literals.""" + i, depth, in_str, esc = start, 1, False, False + while i < len(text) and depth > 0: + c = text[i] + if in_str: + if esc: + esc = False + elif c == "\\": + esc = True + elif c == '"': + in_str = False + elif c == '"': + in_str = True + elif c == "(": + depth += 1 + elif c == ")": + depth -= 1 + i += 1 + return text[start : i - 1] + + +def run_rule_i_metric_literals(root: Path, report: Report) -> None: + """Rule I: no string literal as a metric instrument NAME or label KEY. + + The metric-side mirror of Rule F. An instrument name or label key passed to + an `XRPL_METRIC_*` macro or to a `meter->Create*` factory must reference a + `*MetricNames.h` constant, so a rename is one edit instead of six and a typo + is a compile error instead of a metric that silently never appears. + + Exempt, for the same reasons Rule F exempts them: + * label VALUES -- the argument after a label key is runtime data; + * the instrument DESCRIPTION -- prose, not naming surface; + * `*MetricNames.h` itself -- that is where the strings are defined; + * `MetricMacros.h` -- the macro definitions, whose `(name)` is a parameter; + * test code -- tests pass arbitrary literals to exercise the API, and + asserting on a literal is what proves a constant's value is unchanged. + + Scope -- a RATCHET, not a flag day. The rule fires only on a metric whose + FAMILY already has constants declared in a `*MetricNames.h` (matched on the + name's first underscore segment, e.g. `sync_`, `jobq_`, `unl_`). The metric + surface predates this rule by many phases, and failing the build on ~27 + pre-existing instruments at once would force one un-reviewable mega-change. + Declaring a constant for a family is what opts that family in, so: + * a NEW metric in an already-converted family is caught immediately; + * a literal left behind in a family being converted is caught; + * an untouched legacy family stays green until someone converts it, at + which point the whole family is enforced. + A literal whose family has no constants at all is reported as a Rule L + warning instead (non-fatal), so the remaining work stays visible. + + Presence-gated on `*MetricNames.h` existing at all; with no metric header + in the tree the rule has no families to enforce and is skipped.""" + if not find_metricname_headers(root): + report.skip("I", "no *MetricNames.h present") + return + names, keys, _ = metric_constants(root) + owned = metric_prefixes(names) + found = False + deferred: List[Tuple[str, str]] = [] + + def report_literal(rel: object, lineno: int, token: str, wire: str) -> None: + nonlocal found + # A label key is enforced once ANY label-key constant exists, because + # the label vocabulary is shared across families -- `outcome` means the + # same thing everywhere, so there is no per-family ratchet for it. + in_scope = ( + any(wire.startswith(p) for p in owned) if token.startswith("name") else True + ) + if in_scope: + found = True + report.violation( + "I", f"{rel}:{lineno}", token, "use a *MetricNames.h constant" + ) + else: + deferred.append((f"{rel}:{lineno}", token)) + + for path in sorted(iter_sources(root)): + if path.name.endswith("MetricNames.h") or path.name == "MetricMacros.h": + continue + if is_test_path(path): + continue + text = strip_comments(read_source(path)) + rel = path.relative_to(root) + for kind, name_idx, arglist, lineno in metric_calls(text): + args = split_top_level_args(arglist) + if len(args) > name_idx: + lit = STRING_LITERAL.search(args[name_idx]) + if lit: + report_literal( + rel, lineno, f'name "{lit.group(1)}" ({kind})', lit.group(1) + ) + # Label KEYS live in the label-set initializer, e.g. + # `{{"outcome", v}, {"reason", w}}`. Scanned over the WHOLE argument + # list rather than per split argument, because + # `split_top_level_args` balances only parentheses -- a brace- + # enclosed label set contains top-level commas and would be split + # apart mid-pair, hiding every key from the pattern. + # + # Scanning the whole list is safe: METRIC_PAIR_KEY requires an + # opening `{` before the key, which the instrument name and the + # description never have. Only the key position is checked; the + # VALUE after it is runtime data and exempt. + if keys: + for pm in METRIC_PAIR_KEY.finditer(arglist): + key = pm.group(1) + if not key.startswith('"'): + continue + report_literal(rel, lineno, f"label {key} ({kind})", key.strip('"')) + for loc, token in deferred: + report.warning("L", loc, token, "metric family not yet converted") + if not found: + report.ok("I: no string-literal names/label keys in converted metric families") + + +def iter_sources(root: Path) -> List[Path]: + """Every C++ source/header under src/ and include/.""" + return [ + p + for base in ("src", "include") + for ext in ("*.h", "*.cpp") + for p in (root / base).rglob(ext) + if p.is_file() + ] + + +def instrument_kinds(root: Path, wire_by_symbol: Dict[str, str]) -> Dict[str, Set[str]]: + """Map each declared instrument's WIRE name to the set of OTel instrument + kinds its emit sites actually create it with. + + The kind is what decides which suffix is correct, so it must be read from + the emit site rather than guessed from the name -- guessing from words like + "latency" or "us" mislabels a multi-series gauge whose units live in its + label VALUES (e.g. `nodestore_state` observing `write_mean_us`), which is + a legitimate shape, not a violation. + + A SET rather than one kind per name, because one wire name created through + two different factories is itself the defect worth reporting: the SDK would + export two instruments under one name and the collector would see whichever + arrived last. Recording only the last kind visited hid exactly that case. + + Values are drawn from `counter`, `histogram`, `gauge`, `updown`. A name whose + emit site is not found is absent from the result, so Rule J checks only the + shape-independent rules for it.""" + kinds: Dict[str, Set[str]] = {} + for path in iter_sources(root): + if path.name == "MetricMacros.h" or path.name.endswith("MetricNames.h"): + continue + if is_test_path(path): + continue + text = strip_comments(read_source(path)) + for kind, name_idx, arglist, _ in metric_calls(text): + args = split_top_level_args(arglist) + if len(args) <= name_idx: + continue + arg = args[name_idx].strip() + lit = re.fullmatch(r'"((?:[^"\\]|\\.)*)"', arg) + if lit: + wire: Optional[str] = lit.group(1) + else: + ident = IDENTIFIER_ARG.match(arg) + wire = ( + wire_by_symbol.get(ident.group(1).split("::")[-1]) + if ident + else None + ) + if wire is None: + continue + classified = classify_instrument_kind(kind) + # `other` is the sentinel for a macro that is not an instrument + # factory. Adding it would let a future non-instrument + # `XRPL_METRIC_*` macro turn a correctly named metric into + # "created as counter and other", a conflict message that reads as + # nonsense. Dropping it keeps the sentinel doing exactly what it did + # before: matching no shape rule. + if classified != "other": + kinds.setdefault(wire, set()).add(classified) + return kinds + + +def classify_instrument_kind(kind: str) -> str: + """Normalise a macro suffix (`COUNTER_INC_LABELED`) or a factory method + name (`CreateInt64ObservableGauge`) to one of counter/histogram/gauge/ + updown. Order matters: `UpDown` and `Observable` are checked before the + bare `Counter` substring they both contain.""" + if "HISTOGRAM" in kind or "Histogram" in kind: + return "histogram" + if "UPDOWN" in kind or "UpDown" in kind: + return "updown" + if "GAUGE" in kind or "Gauge" in kind: + return "gauge" + if "COUNTER" in kind or "Counter" in kind: + return "counter" + return "other" + + +def symbol_wire_names(root: Path) -> Dict[str, str]: + """Map each `*MetricNames.h` constant's C++ symbol name to its wire string, + so an emit site referencing `metric::dnsResolveTotal` can be resolved back + to `dns_resolve_total`.""" + out: Dict[str, str] = {} + for h in find_metricname_headers(root): + for m in METRIC_CONST_DEF.finditer(strip_comments(read_source(h))): + out[m.group(1)] = m.group(2) + return out + + +def run_rule_j_metric_suffixes(root: Path, report: Report) -> None: + """Rule J: instrument names follow the naming and unit/kind suffix rules. + + Checked against the `namespace metric` constants in `*MetricNames.h`, which + Rule I makes the only place an instrument name can be spelled. Enforced: + + * lower_snake_case (same shape as Rule G for span attributes); + * no `xrpld_`/`xrpl_` prefix -- the Prometheus exporter adds the namespace + itself, so a name carrying it would emit `xrpld_xrpld_*` on the wire; + * a monotonic COUNTER ends in `_total`, so `rate()` over it reads + correctly and a reader can tell it from a gauge; + * a HISTOGRAM ends in `_us`, `_ms` or `_seconds`: histograms in this repo + are all durations, and the OTel `unit` argument is not surfaced on the + Prometheus metric name, so an unlabelled duration is ambiguous; + * a GAUGE does not end in `_total`, which is reserved for counters. + + The instrument KIND comes from the emit site (see `instrument_kinds`), not + from words in the name, so a multi-series gauge that carries its units in + its label values is not mistaken for a mis-suffixed duration. + + Presence-gated: skipped when no `*MetricNames.h` exists.""" + if not find_metricname_headers(root): + report.skip("J", "no *MetricNames.h present") + return + names, _, _ = metric_constants(root) + if not names: + report.skip("J", "no metric instrument-name constants declared") + return + kinds = instrument_kinds(root, symbol_wire_names(root)) + found = False + + def flag(name: str, expected: str) -> None: + nonlocal found + found = True + report.violation("J", "*MetricNames.h", name, expected) + + for name in sorted(names): + if not SNAKE_SEGMENT.match(name): + flag(name, "must be lower_snake_case") + continue + if name.startswith(("xrpld_", "xrpl_")): + flag(name, "drop the prefix; the exporter adds it") + continue + found_kinds = kinds.get(name) or set() + if len(found_kinds) > 1: + # One wire name created through two factories exports two + # instruments under one name; no suffix can be right for both, so + # report the conflict itself rather than picking one arbitrarily. + flag(name, f"created as {' and '.join(sorted(found_kinds))}; pick one kind") + continue + kind = next(iter(found_kinds), None) + if kind == "counter" and not name.endswith(METRIC_COUNTER_SUFFIX): + flag(name, "counter must end in _total") + elif kind == "histogram" and not name.endswith(METRIC_DURATION_SUFFIXES): + flag(name, "histogram needs _us/_ms/_seconds suffix") + elif kind == "gauge" and name.endswith(METRIC_COUNTER_SUFFIX): + flag(name, "_total is reserved for counters") + if not found: + report.ok(f"J: {len(names)} metric instrument name(s) follow the naming rules") + + +def run_rule_k_expected_metrics(root: Path, report: Report) -> None: + """Rule K: every metric named in the workload's `expected_metrics.json` + exists as a `namespace metric` constant. + + The layer that cannot reference a C++ constant, so it is validated against + one instead. This is the check that catches the failure mode this rule set + exists for: a metric renamed in code while the workload validator still + asserts on the old name, which fails as "metric never appeared" at runtime + rather than at review time. + + Scope. Only groups whose metrics come from the OTel instrument API are + checked. The file also inventories `beast::insight` statsd gauges/counters + (`statsd_gauges`, `statsd_counters`) and collector-derived spanmetrics, whose + names are minted by a different emit path -- `formatName()` lowercasing an + insight metric, or the spanmetrics connector -- and so have no + `*MetricNames.h` constant to resolve to by design. Checking them would fail + the build on metrics this convention does not govern. Within the OTel groups + the family ratchet still applies, so an unconverted family is out of scope + too. Presence-gated on both layers.""" + path = root / "docker" / "telemetry" / "workload" / "expected_metrics.json" + if not path.is_file(): + report.skip("K", "expected_metrics.json not present") + return + if not find_metricname_headers(root): + report.skip("K", "no *MetricNames.h to validate against") + return + names, _, _ = metric_constants(root) + try: + doc = json.loads(read_source(path)) + except json.JSONDecodeError as exc: + report.violation("K", str(path.relative_to(root)), str(exc), "valid JSON") + return + declared = { + base_metric_name(e) for e in expected_metric_names(doc, NON_OTEL_METRIC_GROUPS) + } + # Only entries inside a metric FAMILY the constants already own are checked. + # Anything else in the file predates *MetricNames.h, so demanding a constant + # for it would fail the build on unrelated pre-existing metrics rather than + # on the drift this rule targets. + owned = metric_prefixes(names) + unknown = sorted( + n for n in declared if n not in names and any(n.startswith(p) for p in owned) + ) + for n in unknown: + report.violation( + "K", + str(path.relative_to(root)), + n, + "no matching *MetricNames.h constant", + ) + if not unknown: + checked = sum(1 for n in declared if n in names) + report.ok(f"K: {checked} expected-metric name(s) resolve to constants") + + +# Suffixes the Prometheus exporter appends to a histogram instrument. The +# instrument name declared in code is the stem, so they are stripped before the +# constant lookup. +HISTOGRAM_SUFFIXES = ("_bucket", "_count", "_sum") + + +def base_metric_name(entry: str) -> str: + """Reduce an `expected_metrics.json` entry to the bare instrument name. + + Entries are written the way an operator would query them, not the way the + code declares them, so two forms have to be normalized away: + * a PromQL label selector -- `sync_state{metric="ledgers_behind"}`; + * an exporter-appended histogram suffix -- `..._ms_bucket`/`_count`/`_sum`, + which the SDK adds and the code never names. + Without this the rule would reject every entry in the file, which is a + checker bug rather than a naming violation.""" + name = entry.split("{", 1)[0].strip() + for suffix in HISTOGRAM_SUFFIXES: + if name.endswith(suffix): + return name[: -len(suffix)] + return name + + +def metric_prefixes(names: Set[str]) -> Set[str]: + """The first underscore-delimited segment of each declared instrument name, + e.g. {`sync_`, `jobq_`, `unl_`}. Used by Rule K to decide whether an entry + in `expected_metrics.json` belongs to a family this repo's constants own -- + so the rule flags a stale name inside a converted family without demanding a + constant for every unrelated metric in the file.""" + return {n.split("_", 1)[0] + "_" for n in names if "_" in n} + + +# Groups in expected_metrics.json whose names are NOT minted by the OTel +# instrument API, and therefore have no *MetricNames.h constant by design: +# statsd_gauges / statsd_counters -- beast::insight metrics, whose wire names +# come from formatName() lowercasing an insight metric path; +# spanmetrics -- synthesised by the collector's spanmetrics connector from +# span names, not declared in C++ at all. +NON_OTEL_METRIC_GROUPS = frozenset({"statsd_gauges", "statsd_counters", "spanmetrics"}) + + +def expected_metric_names( + doc: object, skip_groups: frozenset = frozenset() +) -> Set[str]: + """Collect metric names from `expected_metrics.json`. + + The file groups metrics by source (`spanmetrics`, `statsd_gauges`, + `phase9_nodestore`, ...), each group holding a `metrics` LIST OF STRINGS + alongside a prose `description`. Both that shape and a + `[{"metric": "..."}]` shape are accepted, and the walk is generic, so a + layout change degrades to "checks fewer names" rather than silently + checking none. `description` is skipped explicitly -- it is prose, and + letting it through would feed whole sentences to the family match. + + `skip_groups` drops whole top-level groups (see NON_OTEL_METRIC_GROUPS).""" + out: Set[str] = set() + + def walk(node: object, in_metrics: bool = False) -> None: + if isinstance(node, dict): + for key, val in node.items(): + if key == "description" or key in skip_groups: + continue + if key in ("metric", "name") and isinstance(val, str): + out.add(val) + else: + walk(val, in_metrics or key == "metrics") + elif isinstance(node, list): + for item in node: + if in_metrics and isinstance(item, str): + out.add(item) + else: + walk(item, in_metrics) + + walk(doc) + return out + + if __name__ == "__main__": main() diff --git a/.github/scripts/otel-naming/test_check_otel_naming.py b/.github/scripts/otel-naming/test_check_otel_naming.py index 05500768ebc..7c1ed72e756 100644 --- a/.github/scripts/otel-naming/test_check_otel_naming.py +++ b/.github/scripts/otel-naming/test_check_otel_naming.py @@ -1,5 +1,9 @@ #!/usr/bin/env python3 +# cspell:ignore ISTOGRAM +# The all-caps macro name XRPL_METRIC_HISTOGRAM_RECORD trips cspell's +# compound-word splitter, which emits the subword "ISTOGRAM"; ignore it here. + """Unit tests for check_otel_naming.py. Stdlib-only (unittest), matching the dependency-free policy of the check itself. @@ -960,5 +964,534 @@ def test_clean_runbook_records_ok(self): shutil.rmtree(d) +# A minimal *MetricNames.h body: the three namespaces the metric rules read. +def _metric_header( + metric_body: str = "", label_body: str = "", lval_body: str = "" +) -> str: + return ( + "#pragma once\n" + "namespace xrpl::telemetry {\n" + f"namespace metric {{\n{metric_body}\n}}\n" + f"namespace label {{\n{label_body}\n}}\n" + f"namespace lval {{\n{lval_body}\n}}\n" + "}\n" + ) + + +def _mc(symbol: str, wire: str) -> str: + """One `inline constexpr char SYM[] = "wire";` declaration.""" + return f'inline constexpr char {symbol}[] = "{wire}";' + + +class MetricConstantExtraction(unittest.TestCase): + """L1-metrics: instrument names / label keys / label values are read from + the right namespace, and comments never seed the set.""" + + def _run(self, header_text): + d = Path(tempfile.mkdtemp()) + try: + _write(d / "src" / "xrpld" / "telemetry" / "MetricNames.h", header_text) + return chk.metric_constants(d) + finally: + shutil.rmtree(d) + + def test_splits_by_namespace(self): + names, keys, vals = self._run( + _metric_header( + _mc("dnsResolveTotal", "dns_resolve_total"), + _mc("outcome", "outcome"), + _mc("resolved", "resolved"), + ) + ) + self.assertEqual(names, {"dns_resolve_total"}) + self.assertEqual(keys, {"outcome"}) + self.assertEqual(vals, {"resolved"}) + + def test_nested_lval_namespace_included(self): + # `namespace lval { namespace dns_resolve { ... } }` — the inner block is + # inside the outer's brace-matched span, so its values must be collected. + _, _, vals = self._run( + _metric_header( + lval_body="namespace dns_resolve {\n" + + _mc("resolved", "resolved") + + "\n}\n" + ) + ) + self.assertEqual(vals, {"resolved"}) + + def test_commented_constant_not_collected(self): + names, _, _ = self._run( + _metric_header( + "// " + _mc("ghost", "ghost_total") + "\n" + _mc("real", "real_total") + ) + ) + self.assertEqual(names, {"real_total"}) + + def test_block_commented_constant_not_collected(self): + names, _, _ = self._run( + _metric_header("/* " + _mc("ghost", "ghost_total") + " */\n") + ) + self.assertEqual(names, set()) + + def test_no_header_empty_sets(self): + d = Path(tempfile.mkdtemp()) + try: + (d / "src").mkdir() + self.assertEqual(chk.metric_constants(d), (set(), set(), set())) + finally: + shutil.rmtree(d) + + def test_namespaced_constant_ignores_its_flat_prefix(self): + # A `kLabel`-prefixed identifier written inside `namespace metric` must + # be an instrument name only. The flat prefix pass exists for headers + # that have no such namespace, so it must not also claim this one: + # letting it through would make an instrument name a valid label key, + # and Rule D would then accept `label_peer_total` as a label. + names, keys, vals = self._run( + _metric_header( + metric_body=_mc("kLabelPeerTotal", "label_peer_total"), + label_body=_mc("outcome", "outcome"), + ) + ) + self.assertEqual(names, {"label_peer_total"}) + self.assertEqual(keys, {"outcome"}) + self.assertEqual(vals, set()) + + def test_flat_prefix_still_read_when_namespaces_absent(self): + # The counterpart to the test above: with no `namespace metric`/`label` + # block to excise, the flat pass must still classify by prefix. + names, keys, vals = self._run( + "#pragma once\n" + "namespace xrpl::telemetry {\n" + + _mc("kLabelOutcome", "outcome") + + "\n" + + _mc("kResultResolved", "resolved") + + "\n}\n" + ) + self.assertEqual(names, set()) + self.assertEqual(keys, {"outcome"}) + self.assertEqual(vals, {"resolved"}) + + +class RuleIMetricLiterals(unittest.TestCase): + """Rule I: literal instrument names / label keys at a metric emit site are + flagged, but only inside a metric FAMILY that already has constants.""" + + def _run(self, rel_path, source, header=None): + d = Path(tempfile.mkdtemp()) + try: + _write( + d / "src" / "xrpld" / "telemetry" / "MetricNames.h", + ( + header + if header is not None + else _metric_header( + _mc("syncState", "sync_state"), _mc("outcome", "outcome") + ) + ), + ) + _write(d / rel_path, source) + report = chk.Report() + chk.run_rule_i_metric_literals(d, report) + return ( + sorted(v[2] for v in report.violations), + sorted(w[2] for w in report.warnings), + ) + finally: + shutil.rmtree(d) + + # ----- positive: a literal in a converted family must FAIL ----- + def test_literal_macro_name_flagged(self): + v, _ = self._run( + "src/xrpld/Foo.cpp", + 'XRPL_METRIC_COUNTER_INC(app, "sync_acquire_total", "d");\n', + ) + self.assertEqual(v, ['name "sync_acquire_total" (COUNTER_INC)']) + + def test_literal_factory_name_flagged(self): + v, _ = self._run( + "src/xrpld/Foo.cpp", + 'meter_->CreateInt64ObservableGauge("sync_thing", "d");\n', + ) + self.assertEqual(v, ['name "sync_thing" (CreateInt64ObservableGauge)']) + + def test_literal_label_key_flagged(self): + v, _ = self._run( + "src/xrpld/Foo.cpp", + 'XRPL_METRIC_COUNTER_INC_LABELED(app, metric::syncState, "d", ' + '{{"outcome", std::string(x)}});\n', + ) + self.assertEqual(v, ['label "outcome" (COUNTER_INC_LABELED)']) + + def test_multiline_call_flagged(self): + v, _ = self._run( + "src/xrpld/Foo.cpp", + 'XRPL_METRIC_COUNTER_INC(\n app,\n "sync_x_total",\n "d");\n', + ) + self.assertEqual(v, ['name "sync_x_total" (COUNTER_INC)']) + + # ----- negative: things Rule I must NOT flag ----- + def test_constant_name_accepted(self): + v, _ = self._run( + "src/xrpld/Foo.cpp", + 'XRPL_METRIC_COUNTER_INC(app, metric::syncState, "d");\n', + ) + self.assertEqual(v, []) + + def test_label_value_exempt(self): + # The VALUE after a constant key is runtime data and must not be flagged. + v, _ = self._run( + "src/xrpld/Foo.cpp", + 'XRPL_METRIC_COUNTER_INC_LABELED(app, metric::syncState, "d", ' + '{{label::outcome, std::string("resolved")}});\n', + ) + self.assertEqual(v, []) + + def test_description_exempt(self): + # arg2 is prose, not naming surface. + v, _ = self._run( + "src/xrpld/Foo.cpp", + 'XRPL_METRIC_COUNTER_INC(app, metric::syncState, "Some description");\n', + ) + self.assertEqual(v, []) + + def test_test_path_exempt(self): + v, _ = self._run( + "src/tests/libxrpl/telemetry/MetricMacros.cpp", + 'XRPL_METRIC_COUNTER_INC(app, "sync_lit_total", "d");\n', + ) + self.assertEqual(v, []) + + def test_metricnames_header_exempt(self): + v, _ = self._run( + "src/xrpld/telemetry/OtherMetricNames.h", + 'XRPL_METRIC_COUNTER_INC(app, "sync_lit_total", "d");\n', + ) + self.assertEqual(v, []) + + def test_macro_definition_header_exempt(self): + v, _ = self._run( + "src/xrpld/telemetry/MetricMacros.h", + 'XRPL_METRIC_COUNTER_INC(app, "sync_lit_total", "d");\n', + ) + self.assertEqual(v, []) + + # ----- the ratchet: an unconverted family warns (L) instead of failing ----- + def test_unconverted_family_warns_not_fails(self): + v, w = self._run( + "src/xrpld/Foo.cpp", + 'meter_->CreateDoubleObservableGauge("txq_metrics", "d");\n', + ) + self.assertEqual(v, []) + self.assertEqual(w, ['name "txq_metrics" (CreateDoubleObservableGauge)']) + + def test_skip_when_no_metric_header(self): + d = Path(tempfile.mkdtemp()) + try: + _write( + d / "src" / "xrpld" / "Foo.cpp", + 'XRPL_METRIC_COUNTER_INC(app, "x_total", "d");\n', + ) + report = chk.Report() + chk.run_rule_i_metric_literals(d, report) + self.assertEqual(report.violations, []) + self.assertTrue(any("I" in s for s in report.skips)) + finally: + shutil.rmtree(d) + + +class RuleJMetricSuffixes(unittest.TestCase): + """Rule J: instrument names follow the kind-appropriate suffix rules, with + the KIND read from the emit site rather than guessed from the name.""" + + def _run(self, metric_body, emit_source=""): + d = Path(tempfile.mkdtemp()) + try: + _write( + d / "src" / "xrpld" / "telemetry" / "MetricNames.h", + _metric_header(metric_body), + ) + if emit_source: + _write(d / "src" / "xrpld" / "Foo.cpp", emit_source) + report = chk.Report() + chk.run_rule_j_metric_suffixes(d, report) + return sorted((v[2], v[3]) for v in report.violations) + finally: + shutil.rmtree(d) + + # ----- positive ----- + def test_counter_without_total_flagged(self): + self.assertEqual( + self._run( + _mc("dnsResolve", "dns_resolve"), + 'XRPL_METRIC_COUNTER_INC(app, metric::dnsResolve, "d");\n', + ), + [("dns_resolve", "counter must end in _total")], + ) + + def test_histogram_without_unit_flagged(self): + self.assertEqual( + self._run( + _mc("dialLatency", "overlay_dial_latency"), + 'XRPL_METRIC_HISTOGRAM_RECORD(app, metric::dialLatency, "d", v);\n', + ), + [("overlay_dial_latency", "histogram needs _us/_ms/_seconds suffix")], + ) + + def test_gauge_with_total_flagged(self): + self.assertEqual( + self._run( + _mc("syncTotal", "sync_total"), + 'meter_->CreateInt64ObservableGauge(metric::syncTotal, "d");\n', + ), + [("sync_total", "_total is reserved for counters")], + ) + + def test_prefixed_name_flagged(self): + self.assertEqual( + self._run(_mc("prefixed", "xrpld_sync_state")), + [("xrpld_sync_state", "drop the prefix; the exporter adds it")], + ) + + def test_non_snake_case_flagged(self): + self.assertEqual( + self._run(_mc("camel", "syncState")), + [("syncState", "must be lower_snake_case")], + ) + + # ----- negative ----- + def test_conforming_counter_accepted(self): + self.assertEqual( + self._run( + _mc("dnsResolveTotal", "dns_resolve_total"), + 'XRPL_METRIC_COUNTER_INC(app, metric::dnsResolveTotal, "d");\n', + ), + [], + ) + + def test_conforming_histogram_accepted(self): + self.assertEqual( + self._run( + _mc("dialMs", "overlay_dial_latency_ms"), + 'XRPL_METRIC_HISTOGRAM_RECORD(app, metric::dialMs, "d", v);\n', + ), + [], + ) + + def test_gauge_with_unit_bearing_label_values_is_not_flagged(self): + # The regression this rule's kind-awareness exists for: a multi-series + # GAUGE whose units live in its label VALUES (nodestore_state + # observing write_mean_us) must not be read as a mis-suffixed duration. + self.assertEqual( + self._run( + _mc("nodestoreState", "nodestore_state"), + 'meter_->CreateInt64ObservableGauge(metric::nodestoreState, "d");\n', + ), + [], + ) + + def test_unknown_kind_only_shape_checked(self): + # With no emit site found, the kind is unknown, so only the + # shape-independent rules apply -- a bare gauge-ish name is fine. + self.assertEqual(self._run(_mc("syncState", "sync_state")), []) + + def test_one_name_created_as_two_kinds_is_flagged(self): + # One wire name built through two different factories exports two + # instruments under a single name, and no suffix can satisfy both. The + # kind map therefore records a SET per name: keeping only the last kind + # visited silently hid this, because whichever emit site the walk + # reached last decided the verdict. + violations = self._run( + _mc("dualKind", "dual_kind_total"), + 'meter_->CreateUInt64Counter(metric::dualKind, "d");\n' + 'meter_->CreateInt64ObservableGauge(metric::dualKind, "d");\n', + ) + self.assertEqual(len(violations), 1, violations) + # The message names both kinds, so the reader sees the conflict rather + # than a suffix complaint that would contradict one of the two sites. + self.assertIn("counter", violations[0][-1]) + self.assertIn("gauge", violations[0][-1]) + + def test_two_kinds_where_the_last_one_alone_looks_clean(self): + # The sharper case for the same bug. Here the LAST emit site visited is a + # histogram and the name carries a duration suffix, so under last-wins + # semantics Rule J saw a well-formed histogram and reported nothing at + # all -- the conflict was not merely mislabelled, it was invisible. With + # a set per name the mismatch surfaces regardless of walk order. + violations = self._run( + _mc("dualShape", "dual_shape_us"), + 'meter_->CreateInt64ObservableGauge(metric::dualShape, "d");\n' + 'meter_->CreateUInt64Histogram(metric::dualShape, "d");\n', + ) + self.assertEqual(len(violations), 1, violations) + self.assertIn("gauge", violations[0][-1]) + self.assertIn("histogram", violations[0][-1]) + + def test_skip_when_no_header(self): + d = Path(tempfile.mkdtemp()) + try: + (d / "src").mkdir() + report = chk.Report() + chk.run_rule_j_metric_suffixes(d, report) + self.assertEqual(report.violations, []) + self.assertTrue(any("J" in s for s in report.skips)) + finally: + shutil.rmtree(d) + + +class RuleKExpectedMetrics(unittest.TestCase): + """Rule K: a name in expected_metrics.json inside a converted family must + resolve to a *MetricNames.h constant.""" + + def _run(self, json_text, metric_body): + d = Path(tempfile.mkdtemp()) + try: + _write( + d / "src" / "xrpld" / "telemetry" / "MetricNames.h", + _metric_header(metric_body), + ) + _write( + d / "docker" / "telemetry" / "workload" / "expected_metrics.json", + json_text, + ) + report = chk.Report() + chk.run_rule_k_expected_metrics(d, report) + return sorted(v[2] for v in report.violations), report.skips + finally: + shutil.rmtree(d) + + def test_stale_name_in_converted_family_flagged(self): + # `sync_` is owned (sync_state is declared), so a sibling that resolves + # to nothing is the rename-drift this rule exists to catch. + v, _ = self._run( + '{"metrics": [{"metric": "sync_stale_total"}]}', + _mc("syncState", "sync_state"), + ) + self.assertEqual(v, ["sync_stale_total"]) + + def test_declared_name_accepted(self): + v, _ = self._run( + '{"metrics": [{"metric": "sync_state"}]}', _mc("syncState", "sync_state") + ) + self.assertEqual(v, []) + + def test_unowned_family_not_flagged(self): + # `txq_` has no constants, so its entries are out of scope rather than + # a failure -- the ratchet again. + v, _ = self._run( + '{"metrics": [{"metric": "txq_metrics"}]}', _mc("syncState", "sync_state") + ) + self.assertEqual(v, []) + + def test_name_key_also_read(self): + v, _ = self._run( + '{"metrics": [{"name": "sync_stale_total"}]}', + _mc("syncState", "sync_state"), + ) + self.assertEqual(v, ["sync_stale_total"]) + + def test_nested_structure_walked(self): + v, _ = self._run( + '{"groups": {"a": {"items": [{"metric": "sync_stale_total"}]}}}', + _mc("syncState", "sync_state"), + ) + self.assertEqual(v, ["sync_stale_total"]) + + def test_promql_selector_stripped(self): + # Entries are written as an operator would query them; the selector must + # be stripped before the constant lookup or every entry would fail. + v, _ = self._run( + '{"g": {"metrics": ["sync_state{metric=\\"ledgers_behind\\"}"]}}', + _mc("syncState", "sync_state"), + ) + self.assertEqual(v, []) + + def test_histogram_suffix_stripped(self): + # _bucket/_count/_sum are appended by the exporter, never named in code. + v, _ = self._run( + '{"g": {"metrics": ["overlay_dial_latency_ms_bucket"]}}', + _mc("dialMs", "overlay_dial_latency_ms"), + ) + self.assertEqual(v, []) + + def test_statsd_group_skipped(self): + # beast::insight names come from formatName(), not the OTel API, so a + # statsd group must not be held to a *MetricNames.h constant. + v, _ = self._run( + '{"statsd_gauges": {"metrics": ["sync_not_a_constant"]}}', + _mc("syncState", "sync_state"), + ) + self.assertEqual(v, []) + + def test_spanmetrics_group_skipped(self): + v, _ = self._run( + '{"spanmetrics": {"metrics": ["sync_span_calls_total"]}}', + _mc("syncState", "sync_state"), + ) + self.assertEqual(v, []) + + def test_description_prose_not_treated_as_metric(self): + v, _ = self._run( + '{"g": {"description": "sync_ stuff about metrics", "metrics": []}}', + _mc("syncState", "sync_state"), + ) + self.assertEqual(v, []) + + def test_malformed_json_reported(self): + v, _ = self._run("{not json", _mc("syncState", "sync_state")) + self.assertEqual(len(v), 1) + + def test_skip_when_file_absent(self): + d = Path(tempfile.mkdtemp()) + try: + _write( + d / "src" / "xrpld" / "telemetry" / "MetricNames.h", + _metric_header(_mc("syncState", "sync_state")), + ) + report = chk.Report() + chk.run_rule_k_expected_metrics(d, report) + self.assertEqual(report.violations, []) + self.assertTrue(any("K" in s for s in report.skips)) + finally: + shutil.rmtree(d) + + +class InstrumentKindClassification(unittest.TestCase): + """classify_instrument_kind: `UpDown`/`Observable` are checked before the + bare `Counter` substring they both contain.""" + + def test_macro_kinds(self): + for kind, want in ( + ("COUNTER_INC", "counter"), + ("COUNTER_ADD_LABELED", "counter"), + ("HISTOGRAM_RECORD", "histogram"), + ("UPDOWN_ADD", "updown"), + ("OBSERVABLE_GAUGE_REGISTER", "gauge"), + ): + self.assertEqual(chk.classify_instrument_kind(kind), want, kind) + + def test_factory_kinds(self): + for kind, want in ( + ("CreateUInt64Counter", "counter"), + ("CreateDoubleHistogram", "histogram"), + ("CreateInt64UpDownCounter", "updown"), + ("CreateInt64ObservableGauge", "gauge"), + ("CreateInt64ObservableCounter", "counter"), + ("CreateInt64ObservableUpDownCounter", "updown"), + ): + self.assertEqual(chk.classify_instrument_kind(kind), want, kind) + + +class MetricPrefixFamilies(unittest.TestCase): + def test_first_segment_is_the_family(self): + self.assertEqual( + chk.metric_prefixes({"sync_state", "jobq_saturation", "unl_quorum"}), + {"sync_", "jobq_", "unl_"}, + ) + + def test_name_without_underscore_has_no_family(self): + self.assertEqual(chk.metric_prefixes({"uptime"}), set()) + + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2642b8a377d..81f78e11db9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -416,13 +416,67 @@ python .github/scripts/otel-naming/check_otel_naming.py See [.github/scripts/otel-naming/README.md](.github/scripts/otel-naming/README.md) for the full rule list. +## Telemetry metric naming + +The metric-side counterpart of the span rules above. Metric instrument names and +metric label keys are duplicated across the emit site, the instrument +registration, the unit test, `expected_metrics.json`, the dashboard PromQL and +the runbook, so a rename touches six places and a typo in any one of them fails +silently at runtime — a metric that never appears, or a label that never joins. +The constants in the `*MetricNames.h` headers are the single source of truth for +the C++ layers; a CI check validates the layers that cannot reference a constant. + +1. Instrument names are bare `lower_snake_case` with **no `xrpld_` prefix**. The + Prometheus exporter adds the namespace itself, so a name carrying it emits + `xrpld_xrpld_*` on the wire. +2. A monotonic counter ends in `_total`, so `rate()` over it reads correctly and + a reader can tell it from a gauge at a glance. +3. A duration carries its unit as the suffix — `_us`, `_ms` or `_seconds`. The + unit belongs in the name because the OTel `unit` argument is not surfaced on + the Prometheus metric name. +4. A gauge that snapshots current state takes no suffix (`jobq_saturation`, + `sync_state`), and never `_total`. +5. Label keys are `lower_snake_case` and must have **bounded** cardinality. A + multi-series gauge discriminates its readings with the `metric` label rather + than minting one instrument per reading. +6. Label **values** are declared as constants only when the code picks them from + a fixed set (`namespace lval`). A value derived from runtime data — a peer + address, a ledger hash — must never become a label on a metric. + +Always reference the `*MetricNames.h` constants for instrument names and label +keys — never pass a string literal. (Label _values_ may be runtime data.) Note +that these headers use `constexpr char[]`, not the `makeStr`/`StaticStr` DSL the +`*SpanNames.h` headers use: the OTel C++ API takes `nostd::string_view`, which +constructs from `char const*` but has no constructor from `std::string_view`, so +`StaticStr` does not compile in an instrument-name or label-key position. + +Enforcement is by the same script as the span rules, whose metric rules are: + +- **I** — no string literal as an instrument name or label key at an emit site + (the mirror of Rule F). Scoped by metric _family_ (the first underscore + segment) so conversion can proceed subsystem by subsystem: declaring a + constant opts that family in. An unconverted family is reported as a + non-fatal **L** warning, keeping the remaining work visible. +- **J** — the suffix conventions above. The instrument _kind_ is read from the + emit site, not guessed from the name, so a multi-series gauge whose units live + in its label values is not mistaken for a mis-suffixed duration. A name created + through two different factories is reported as a kind conflict rather than a + suffix complaint, because no suffix can be correct for both. +- **K** — every metric named in `docker/telemetry/workload/expected_metrics.json` + resolves to a declared constant. This is the check that catches a metric + renamed in code while the workload validator still asserts the old name. + Groups whose names come from a different emit path (`statsd_gauges`, + `statsd_counters` from `beast::insight`, and collector-derived `spanmetrics`) + are out of scope by design. + ## Adding a new OTel metric See `src/xrpld/telemetry/MetricMacros.h` for the call-site macros covering every OTel instrument kind (Counter, UpDownCounter, Histogram, Gauge, and their -Observable/async counterparts) and the "Adding a New Metric" section in -[docs/telemetry-runbook.md](docs/telemetry-runbook.md) for the walkthrough and a -need-to-macro lookup table. +Observable/async counterparts), `src/xrpld/telemetry/MetricNames.h` for the name +and label constants to reference (and the rules above), and the "Adding a New +Metric" section in [docs/telemetry-runbook.md](docs/telemetry-runbook.md) for the +walkthrough and a need-to-macro lookup table. ## Contracts and instrumentation diff --git a/OpenTelemetryPlan/09-data-collection-reference.md b/OpenTelemetryPlan/09-data-collection-reference.md index 29d11f680a1..31e457001de 100644 --- a/OpenTelemetryPlan/09-data-collection-reference.md +++ b/OpenTelemetryPlan/09-data-collection-reference.md @@ -33,7 +33,7 @@ graph LR end subgraph viz["Visualization"] - F["Grafana :3000
13 dashboards"] + F["Grafana :3000
15 dashboards"] end A -->|"OTLP/HTTP :4318
(traces + attributes)"| R1 @@ -710,6 +710,23 @@ sampled at the same instant. > **See also**: [05-configuration-reference.md](./05-configuration-reference.md) §5.8 for Grafana data source provisioning (Tempo, Prometheus) and TraceQL query examples. +Fifteen dashboards are provisioned in total. §3.1 and §3.2 below cover the +original ten; the remaining five were added by later phases and are catalogued +where they were introduced, so this section is not the full inventory: + +| Dashboard | UID | Catalogued in | +| ------------------ | -------------------- | ---------------------------------------------------- | +| Fee Market & TxQ | `fee-market` | §5b "New Grafana Dashboards (Phase 9)" | +| Job Queue Analysis | `job-queue` | §5b "New Grafana Dashboards (Phase 9)" | +| Validator Health | `validator-health` | §5d "New Grafana Dashboards (Phase 9)" | +| Peer Quality | `peer-quality` | §5d "New Grafana Dashboards (Phase 9)" | +| Ledger Sync Health | `ledger-sync-health` | "Fresh-node sync diagnostics" (end of this document) | + +The authoritative count is whatever +`docker/telemetry/grafana/dashboards/*.json` holds; +`validate_dashboards.py` prints it and the workload harness asserts every +board renders. + ### 3.1 Span-Derived Dashboards (5) | Dashboard | UID | Data Source | Key Panels | @@ -758,7 +775,7 @@ for how the tier attributes are set and reach metrics. 1. Open Grafana at **http://localhost:3000** 2. Navigate to **Dashboards → xrpld** folder -3. All 10 dashboards are auto-provisioned from `docker/telemetry/grafana/dashboards/` +3. All 15 dashboards are auto-provisioned from `docker/telemetry/grafana/dashboards/` --- @@ -1389,8 +1406,8 @@ information the batch totals do not already carry. **All three histograms need an explicit bucket view.** The SDK's default histogram boundaries top out at 10000. Every one of these three exceeds that, so -without a view their top quantiles would all read as a flat 10000. Six views are -registered in `src/xrpld/telemetry/MetricsRegistry.cpp`, and three of the six are +without a view their top quantiles would all read as a flat 10000. Ten views are +registered in `src/xrpld/telemetry/MetricsRegistry.cpp`, and three of the ten are for this family: | Instrument | View helper | Boundaries | @@ -1399,9 +1416,13 @@ for this family: | `getobject_request_objects` | `addHistogramView()`, own set | `1, 2, 4, 8, 16, 64, 256, 1024, 4096, 12288` | | `getobject_charge` | `addHistogramView()`, own set | `0, 100, 500, 1000, 5000, 10000, 25000, 50000, 100000` | -The other three views are `addMicrosecondHistogramView()` on `job_queued_us`, -`job_running_us`, and `rpc_method_us` — four µs-ladder views plus these two -custom sets. +The other seven views are `addMicrosecondHistogramView()` on `job_queued_us`, +`job_running_us`, `rpc_method_us` and `sweep_malloc_trim_us`; +`addRoundDurationHistogramView()` on `consensus_round_duration_ms`; and +`addHistogramView()` with its own set on `dns_resolve_latency_ms` and +`overlay_dial_latency_ms`. That is five µs-ladder views, one round-duration +ladder, and four caller-supplied sets — the two above plus those two millisecond +latencies. **Why the latter two do not use the µs ladder.** They are not durations. The µs ladder's buckets are chosen for time (sub-millisecond jobs through multi-second @@ -1751,3 +1772,122 @@ prefix=xrpld | `trace_consensus` | `1` | `consensus.*` spans | | `trace_ledger` | `1` | `ledger.*` spans | | `trace_peer` | `1` | `peer.*` spans (high volume) | + +--- + +## Fresh-node sync diagnostics + +Signals that explain why a freshly-started node is slow to reach, or never +reaches, a validated ledger (`server_state=full`). Two groups: pre-quorum +bootstrap (DNS, dial, handshake, UNL/quorum, clock skew) and the post-peering +ledger/tx-set acquire pipeline. + +Rendered by the **Ledger Sync Health** dashboard (uid `ledger-sync-health`), +whose nine rows follow the order a fresh node progresses — so reading the board +top-to-bottom walks the same path a sync does: + +1. `Bootstrap (Domain 0)` — can it reach peers and form a quorum at all? +2. `Peer supply` — does any peer hold what this node needs? +3. `Sync state` — is the node advancing through the mode machine? +4. `Ledger acquire & SHAMap fetch` — is ledger data arriving and being applied? +5. `Job queue` — does arrived work ever get a worker thread? +6. `Quorum & publish` — does a held ledger ever validate, and reach clients? +7. `Terminal blockers & serving` — will the node stop validating for good? +8. `Back-fill & persistence` — is an existing database the bottleneck? +9. `Spans & traces` — _which_ fetch, peer or object, not how many? + +Rows 8 and 9 answer conditional questions: row 8 only applies to a node with +existing history, and row 9 is span-derived, so it inherits trace sampling and +the `trace_ledger` / `trace_peer` flags. They are still expanded by default -- +a row an operator has to remember to open is a row they read too late. + +Operator flow: [telemetry-runbook.md](../docs/telemetry-runbook.md) +"Diagnosing slow/stuck fresh sync". Terms: +[telemetry-glossary.md](../docs/telemetry-glossary.md) +"Fresh-node sync diagnostics". + +The table below is the single index for these signals; one row is added per +signal as it lands. `Type` is the instrument kind (counter / gauge / histogram / +span / span attr), `Emit site` the owning source file, and `Panel` the dashboard +panel that renders it. Panel names are verbatim `ledger-sync-health` panel +titles unless another board is named explicitly, and `n/a` means the signal has +no panel (it is read in Tempo instead). + + + + +| Signal | Type | Emit site | Panel | Meaning | +| -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------ | -------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `dns_resolve_total` (`outcome` = `resolved` \| `empty`) | counter | `OverlayImpl.cpp` — `OverlayImpl::reportDnsResolve` | DNS Resolve Outcome Rate | Peer hostname resolutions. `empty` means a configured bootstrap or `[ips_fixed]` name returned no address, so that peer is never dialled. | +| `dns_resolve_latency_ms` | histogram | `OverlayImpl.cpp` — `OverlayImpl::reportDnsResolve` | DNS Resolve Latency (p95) | Time to resolve a configured peer hostname. Seconds-scale values mean the resolver is timing out ahead of every dial. | +| `overlay_connect_total` (`outcome` = `connected` \| `tcp_fail` \| `tls_fail` \| `self_connection` \| `upgrade_fail` \| `timeout`) | counter | `ConnectAttempt.cpp` — `ConnectAttempt::reportOutcome` | Outbound Dial Outcome Rate | Outbound peer connection attempts by terminal outcome. The outcome names the stage that broke: TCP, TLS, HTTP upgrade, or no terminal state in time. | +| `overlay_dial_latency_ms` | histogram | `ConnectAttempt.cpp` — `ConnectAttempt::reportOutcome` | Outbound Dial Latency (p95) | Time from starting an outbound dial to its terminal outcome, successes and failures together. A p95 near the dial timeout means peers accept TCP but never finish the handshake. | +| `handshake_negotiation_fail_total` (`reason`, 14 values incl. `wrong_network`, `invalid_network_id`, `clock_skew`, `self_connection`, `session_verify_failed`) | counter | `Handshake.cpp` — `throwNegotiationFailure` (from `verifyHandshake`) | Handshake Negotiation Failures by Reason | Peer handshakes rejected after TLS while checking network id, clock, keys and addresses. `reason` names the failing check. | +| `unl_fetch_total` (`site` = UNL site as scheme://host/path, userinfo and query stripped; `outcome` = the 9 `ListDisposition` strings `accepted` \| `expired` \| `same_sequence` \| `pending` \| `known_sequence` \| `unsupported_version` \| `untrusted` \| `stale` \| `invalid`, plus `fetch_error` \| `bad_status` \| `parse_error`) | counter | `ValidatorSite.cpp` — `ValidatorSite::reportFetchOutcome` | UNL Fetch Rate by Site & Outcome | Validator-list fetches per site. `accepted` is the only success; `same_sequence` and `known_sequence` are normal no-op refreshes; the three literals are transport or content faults. | +| `unl_quorum` (`metric` = `trusted_keys` \| `quorum` \| `quorum_disabled`) | observable gauge | `MetricsRegistry.cpp` — `registerUnlQuorumGauge` | UNL Trusted Keys vs Quorum; UNL Quorum Headroom | Trusted UNL key count against the validations a ledger needs. `trusted_keys` at or below `quorum` means the node can never declare a ledger validated. | +| `clock_close_offset_seconds` (`metric` = `offset`) | observable gauge | `MetricsRegistry.cpp` — `registerClockSkewGauge` | Clock Close Offset | Network close time offset from the local clock. Negative means the local clock runs ahead. `server_info` only surfaces `close_time_offset` at 60 s or more, so this gauge sees skew far earlier. | +| `state_changes_total` (`from`, `to` = `disconnected` \| `connected` \| `syncing` \| `tracking` \| `full`) | counter | `NetworkOPs.cpp` — `NetworkOPsImp::setMode` | Mode Transitions by Edge | Operating-mode transitions keyed on the (`from`, `to`) edge. The edge is what separates a clean `disconnected`→`connected`→`syncing`→`tracking`→`full` climb from `full`→`connected` flapping; an unlabelled total cannot tell them apart. | +| `sync_state` (`metric` = `initial_full_duration_us`) | observable gauge | `MetricsRegistry.cpp` — `registerSyncStateGauge` | Time to First FULL | Microseconds from process start to the first `full` transition, sourced from `NetworkOPs::getInitialSyncDurationUs()`. Stays 0 until `full` is reached, so a flat 0 is itself the "never synced" signal; once set it never changes. | +| `sync_state` (`metric` = `network_ledger_gate`) | observable gauge | `MetricsRegistry.cpp` — `registerSyncStateGauge` | Network Ledger Gate | 1 while the node is still waiting to see a full network ledger (`NetworkOPs::isNeedNetworkLedger()`), else 0. A persistent 1 blocks transaction submission and `full`, whatever the rest of the pipeline shows. | +| `sync_state` (`metric` = `server_stall_seconds`) | observable gauge | `MetricsRegistry.cpp` — `registerSyncStateGauge` | Server Stall | Current main-loop stall duration from `LoadManager::getCurrentStallSeconds()`, 0 when healthy. Same duration the load monitor logs as "Server stalled for N seconds", which previously existed only in that log line. | +| `sync_state` (`metric` = `ledgers_behind`) | observable gauge | `MetricsRegistry.cpp` — `registerSyncStateGauge` | Ledgers Behind Network | Peer-reported network tip minus our validated sequence, floored at 0 (`NetworkOPs::getLedgersBehindNetwork()`). Reads each peer's already cached ledger range, so no new network round trip. | +| `server_stall_events_total` | observable counter | `MetricsRegistry.cpp` — `registerStallEventsCounter` | Server Stall Event Rate | Distinct stall episodes since process start, counted once per episode rather than per stalled second. A rising rate is repeated fresh stalls; a flat rate with a large `server_stall_seconds` is one long stall. | +| `sync_acquire` (`metric` = `missing_state_nodes_max` \| `missing_tx_nodes_max`) | observable gauge | `MetricsRegistry.cpp` — `registerSyncAcquireGauge` | Missing SHAMap Nodes per Acquire (state/tx) | Largest outstanding SHAMap node count across in-flight acquires, split by tree, from the count `getMissingNodes()` already produces during its sweep (`InboundLedger.cpp` — `InboundLedger::trigger`). **The headline stuck-sync signal:** flat and non-zero across ticks means the acquire will never finish; shrinking means slow but alive. Aggregated as a max rather than labelled per ledger, because a `ledger_seq` label would mint one series per ledger acquired — per-ledger identity stays on the `ledger.acquire` span. | +| `sync_acquire` (`metric` = `received_data_depth`) | observable gauge | `MetricsRegistry.cpp` — `registerSyncAcquireGauge` | Received-Data Stash Depth & In-Flight Acquires | Peer packets stashed across all in-flight acquires waiting to be applied, summed because it measures one shared processing backlog. A growing depth means arriving node data outpaces processing, so the limit is the job queue or disk rather than peer supply. | +| `sync_acquire` (`metric` = `in_flight`) | observable gauge | `MetricsRegistry.cpp` — `registerSyncAcquireGauge` | Received-Data Stash Depth & In-Flight Acquires | Number of ledger acquires currently running. Exported so the three values above can be read in context: all zero with `in_flight` zero is an idle node, not a healthy one. | +| `shamap_cache_hit_rate` (`metric` = `treenode`) | observable gauge | `MetricsRegistry.cpp` — `registerCacheHitRateDetailGauge` | SHAMap TreeNode Cache Hit Rate | Share of SHAMap tree-node lookups served from memory, from the previously-uncalled `TaggedCache::getHitRate()`, normalized from 0-100 to 0.0-1.0. Distinct from `nodestore_state`-derived NuDB Cache Hit Ratio on the Ledger Data Sync dashboard: this is the in-memory layer **above** the node store, so a miss here is what causes a read there. The full-below cache is not reported — it is a `KeyCache` whose only lookup path increments `stats_.hits`/`stats_.misses` while `getHitRate()` reads the separate `hits_`/`misses_` members, so its rate is hard-wired to 0 until that accounting is fixed. | +| `sync_acquire_no_progress_total` | counter | `InboundLedger.cpp` — `InboundLedger::onTimer` | Acquire Stall Rate (no progress) | Acquire timeouts where not one new node arrived since the previous timeout, from the `progress_` flag that was previously log-only. Fires on the 3 s acquire timer, never per node. A sustained rate together with a flat missing-node count is the definitive "stuck, not slow" signature. | +| `sync_addnode_total` (`outcome` = `good` \| `duplicate` \| `invalid`) | counter | `InboundLedger.cpp` — `InboundLedger::recordBatchOutcome` | Add-Node Outcomes | SHAMap nodes received during acquire, split by result. Emitted once per received packet from the aggregated batch tally the trace log already printed — never inside the per-node `receiveNode()` loop. Separates real progress (`good`) from wasted bandwidth (`duplicate`) and a misbehaving peer (`invalid`), all three of which look like healthy throughput in traffic metrics. | +| `sync_acquire_source_total` (`source` = `local` \| `network`) | counter | `InboundLedger.cpp` — `InboundLedger::init` | Acquire Source (local vs network) | Whether an acquire was satisfied entirely from the local node store or needed peers, emitted once per new acquire after the first local lookup. Sustained `network` on a node that should already hold the range means sync is disk-bound rather than peer-bound. | +| `jobq_saturation` (`metric` = `running_tasks` \| `worker_threads` \| `total_waiting`) | observable gauge | `MetricsRegistry.cpp` — `registerJobQueueSaturationGauge` | Worker Pool Saturation; Worker Pool Capacity & Total Backlog | Global worker-pool saturation from `JobQueue::getWorkerSaturation()`: tasks in flight, threads the pool is configured to run, and jobs queued across all types, all from one reading so the ratio and the backlog describe the same instant. `worker_threads` is exported rather than hardcoded in the dashboard because it is derived at startup from `[workers]`, node size and hardware concurrency. Exists separately from the per-job-type gauges `JobQueue::collect()` publishes (`jobq__waiting` / `_running` / `_deferred`) because a pool-wide slowdown otherwise appears as an independent fault in every subsystem queued behind it; a `running_tasks / worker_threads` ratio at 1.0 **with** a non-zero `total_waiting` attributes it to pool exhaustion once. | +| `peer_ledger_supply` (`metric` = `peers_reporting` \| `peers_serving_validated` \| `peers_serving_next` \| `supply_min_seq` \| `supply_max_seq`) | observable gauge | `MetricsRegistry.cpp` — `registerPeerLedgerSupplyGauge` (aggregating `OverlayImpl::getPeerLedgerSupply`) | Peers Able to Serve Needed Sequence; Peer Supply Window Margin (history headroom vs tip gap) | How much of the sequence range this node needs its connected peer set can actually serve, from one pass over the active peers reading the range each already advertised in `mtSTATUS_CHANGE`. **`peers_serving_next` is the signal this exists for:** zero there with a non-zero `peers_reporting` means no connected peer holds validated + 1, so the peer set must change and waiting cannot finish the sync. `peers_reporting` is the denominator that makes the rest readable — peers advertising `[0, 0]` have not reported yet and are excluded from every field, so they cannot make a healthy peer set appear to serve from genesis; when nothing has reported, both window fields read 0 meaning **unknown**, not genesis. `supply_min_seq` / `supply_max_seq` separate "asking for history nobody kept" from "asking for a tip nobody reached". The _Peer Supply Window Margin_ panel renders both as distances from `server_info{metric="validated_ledger_seq"}` rather than as absolute sequences, because the raw values sit around 1.05e8 and roughly 3e5 apart, so one linear axis flattens the tip movement that shows whether sync is progressing; the subtraction also makes zero the boundary in both directions. Both operands are gated `> 0` in PromQL so the unknown-window sentinel cannot turn into a whole-sequence-space spike when subtracted. Distinct from `server_info{metric="peers"}`, a bare connection count with no notion of what those peers hold; from `sync_state{metric="ledgers_behind"}`, which uses the same per-peer maxima but collapses them to a single distance-to-tip number that cannot say how many peers can serve that distance or whether the range has a hole; and from `peer_quality{metric="peers_insane_count"}`, which counts peers on a different chain and is therefore a correctness signal, not an availability one. | +| `peer_disconnect_total` (`reason` = `graceful` \| `shutdown` \| `stopping` \| `read_error` \| `write_error` \| `timer_error` \| `ping_timeout` \| `not_useful` \| `large_sendq` \| `charge_resources` \| `malformed_handshake` \| `shared_value` \| `unknown`; `direction` = `inbound` \| `outbound`) | counter | `PeerImp.cpp` — `PeerImp::close` | Peer Disconnects by Reason | Peer teardowns split by cause and by which side opened the connection. Emitted once per teardown at `close()`, the single funnel every disconnect path passes through, and `close()` already self-guards on the socket being open, so a repeated close cannot double-count and the total matches the existing unlabelled tally. `reason` is set by whichever site decided to disconnect, first writer wins, so a later generic reason never masks the real one; the value is always one of a fixed set of literals in `PeerImp.cpp`, never peer-supplied data, so cardinality is bounded by the code. The split is the whole point: it separates our-fault backpressure (`large_sendq`, `charge_resources`) from topology and network faults (`not_useful`, `ping_timeout`, `read_error`), and normal churn (`graceful`) from either. Distinct from the existing `server_info{metric="peer_disconnects_resources"}`, which counts only the resource-charge subset and carries no labels, and from the StatsD `overlay_peer_disconnects`, which is the unlabelled grand total in which every reason above collapses into one number. | +| `peer_accept_total` (`outcome` = `accepted` \| `local_endpoint_fail` \| `resource_limit` \| `no_slot` \| `not_peer_request` \| `protocol_mismatch` \| `bad_cookie` \| `slot_refused` \| `handshake_error`) | counter | `OverlayImpl.cpp` — `OverlayImpl::onHandoff` via `reportAcceptOutcome` | Inbound Peer Accept Outcomes | Terminal outcome of every inbound connection this node is offered, one emit per handoff. `accepted` is reported only after `run()`, so anything that threw on the way lands on `handshake_error` instead; the two early returns that are not peer attempts at all (a handled HTTP request, and a request that never asked to upgrade) are deliberately not counted. The `outcome` names the stage that refused: no local endpoint, the resource manager, PeerFinder having no slot or seeing a duplicate, a non-peer upgrade request, protocol version disagreement, a bad security cookie, or activation being refused. This is the **inbound twin** of the existing `overlay_connect_total{outcome}`, which covers outbound dials only; without it a node refusing every inbound connection is indistinguishable from one nobody dials, and reading the two together gives the full in/out split. | +| `peerfinder_slot_census` (`metric` = `out_active` \| `out_max` \| `in_active` \| `in_max` \| `connecting` \| `fixed_configured` \| `fixed_active` \| `bootcache` \| `livecache`) | observable gauge | `MetricsRegistry.cpp` — `registerSlotCensusGauge` (from `Logic::getSlotCensus`) | PeerFinder Slot Census; PeerFinder Address Caches & Fixed Peers | Slot occupancy against capacity, outbound dials in flight, configured-versus-connected fixed peers, and the depth of both address caches. All nine come from a single acquire of the PeerFinder lock, so they are mutually consistent, share one label set and can be compared against each other. That is what makes the three most common bootstrap failures visible: `connecting` non-zero while `out_active` stays below `out_max` (dials starting and never completing), `bootcache` and `livecache` both at 0 (nothing to dial at all), and `fixed_active` below `fixed_configured` (a peer named in the configuration is unreachable). `fixed_configured` is the count of peers named in the config, so the pair an operator reads is "how many did I ask for" against "how many do I have" — the same comparison `autoconnect()` makes. All nine values already existed inside PeerFinder; only two of them were exported, as the legacy beast::insight gauges `peer_finder_active_inbound_peers` and `peer_finder_active_outbound_peers`. Those two carry no capacity, attempt or cache term, are read at unrelated instants, and so cannot be joined with each other let alone with a capacity term — leaving all three failures above indistinguishable from a node that is simply not dialling. | +| `serve_refused_total` (`request` = `ledger` \| `txset` \| `object` \| `fetchpack`; `reason` = `sendq_full` \| `load_shed` \| `not_found` \| `no_map` \| `bad_type` \| `empty_reply`) | counter | `PeerImp.cpp` — `processLedgerRequest`, `onMessage(TMGetObjectByHash)`, `doFetchPack` | Ledger/Object Serve Refusals | Peer data requests this node declined to answer, split by what was asked for and why. This is the **supply side** of the sync exchange — what this node refuses to serve OTHERS — and nothing equivalent existed before, so a node shedding every ledger request looked identical to one being asked for nothing. `sendq_full` and `load_shed` are self-inflicted backpressure (the send queue at `Tuning::kDropSendQueue`, or the local fee track loaded, or too many pack jobs queued), while `not_found` is a genuine history gap and `no_map` / `bad_type` / `empty_reply` mean the request was answerable in principle but produced nothing to send. `fetchpack` is counted apart from `ledger` because a fetch pack is how a syncing peer catches up in bulk and its shed threshold is a different one. Emitted at most once per request — `empty_reply` is reported after the node loop, never inside it — and both labels are code literals, so cardinality is bounded at compile time. | +| `amendment_block` (`metric` = `warned` \| `seconds_to_block`) | observable gauge | `MetricsRegistry.cpp` — `registerAmendmentBlockGauge` | Amendment Block Countdown; Amendment Warned | `warned` is 1 once an unsupported amendment has reached majority (`NetworkOPs::isAmendmentWarned()`, previously only an admin-only `server_info` warning). **`seconds_to_block` is the leading indicator:** seconds until that amendment activates, from `AmendmentTable::firstUnsupportedExpected()` against the network close time. It reads `-1` when nothing is pending — a distinct healthy value rather than a missing series, matching the sentinel `validator_health{metric="unl_expiry_days"}` already uses — and is clamped at 0 rather than going negative, because past-due means the block is imminent, not overdue by some amount worth charting; the subtraction is done in `std::int64_t` so a past-due activation cannot wrap. Amendment-blocked is a terminal sync blocker: the node stops validating and never resumes without a software upgrade. The existing `validator_health{metric="amendment_blocked"}` reports that state after the fact, when nothing can be done about it; this gauge is the window before it, which is the only actionable part. The blocking amendment's identity is deliberately **not** a label — the network can vote on an arbitrary 256-bit amendment id, not drawn from this build's known features, so an id label would be unbounded cardinality and would mint a permanent new series per amendment. The id is available in logs from `AmendmentTableImpl::doValidatedLedger` ("Unsupported amendment \ reached majority at ..."), correlated to this series by node and time. | +| `ledger_jump_total` | counter | `NetworkOPs.cpp` — `NetworkOPsImp::switchLastClosedLedger` | Byzantine Ledger Jumps | Forced jumps of the last closed ledger onto a divergent chain: the node was told the network's LCL is not the one it built on and discarded its own chain tip to follow. Nothing equivalent existed — this was log-only ("JUMP last closed ledger to ..."), so a node repeatedly thrashing between chains left no time series to correlate against the rest of the sync pipeline. Any non-zero rate is abnormal by construction; repeated jumps are wrong-chain thrash, which points at the peer set and the configured network id rather than anywhere in the acquire pipeline. Deliberately unlabelled: the ledger hash and sequence would both be unbounded as label values, and the log line beside the emit already carries them. | +| `nodestore_state` (`metric` = `write_mean_us` \| `read_mean_us` \| `node_writes` \| `node_reads_total` \| `node_writes_duration_us` \| `node_reads_duration_us`) | observable gauge | `MetricsRegistry.cpp` — `registerNodeStoreGauge` (`observeNodeStoreTotals`) | NodeStore Write vs Read Latency (us/op); NodeStore Operation Rate (writes vs reads) | The store/fetch latency half of the pre-existing `nodestore_state` gauge (its I/O counters and queue depth are tabled under NodeStore I/O above). Mean microseconds per node-store store and per fetch, with both operation counts and both cumulative duration totals so a panel can divide the two _rates_ and read interval latency instead of the since-boot average. **The write side is the signal.** `storeDurationUs_` was declared in `Database.h` and never written, and no accessor existed, so no write-path latency was observable anywhere; the read total was already exposed as `nodestore_state{metric="node_reads_duration_us"}`. This is the fingerprint of the "a node with a large existing DB syncs slower than a fresh one" symptom, which is write-bound and therefore invisible in every read-side metric. All three concrete store paths now time themselves through `Database::recordStoreDuration` (`DatabaseNodeImp::store`, `DatabaseRotatingImp::store` and `Database::importInternal`), so `write_mean_us` is live on an ordinary node rather than only on the `[import_db]` admin path. Chosen as a gauge over a histogram deliberately: a histogram gives true percentiles but costs one `Record()` per node object on the store/fetch path, and a single ledger write walks thousands of SHAMap nodes — this gauge instead reads existing atomics once per ~10 s tick and adds nothing to the hot path. Consequence: **p99 is not obtainable from this signal**, and a histogram added later would also need an explicit-bucket View (`addMicrosecondHistogramView`) because the SDK default buckets top out at 10,000. Related to the Ledger Data Sync dashboard's NuDB Read Latency panel, which divides the two read-side fields in PromQL; that panel predates the write numerator and has no write-side equivalent. Each mean is computed by `MetricsRegistry::scaledMean`, which saturates at `INT64_MAX` rather than wrapping and returns no value when its count is 0 — so the series is omitted instead of reporting a misleading 0 µs, while the four totals are always observed. A short-lived duplicate `nodestore_latency` gauge published the same six values from the same accessors and has been retired. | +| `sweep_malloc_trim_us` | histogram | `Application.cpp` — `ApplicationImp::trimHeapAndRecord` | Sweep Heap-Trim Duration (p50/p95) | Wall-clock duration of the `malloc_trim` call that ends every cache sweep. **This is the leading explanation for "a node with a large existing DB syncs slower than a fresh one" on glibc:** the trim runs after EVERY sweep, its cost scales with the resident heap, and the pages it hands back must be re-faulted as the caches refill. The numbers all already existed on `MallocTrimReport`, but were unreachable twice over — the whole measurement block sat inside `if (journal.debug())` in `MallocTrim.cpp`, so an ordinary node at default log level measured nothing, and the return value was then discarded at the call site. The gate now covers only the `JLOG`; measuring costs about 6 µs (two `/proc/self/statm` reads at ~2.8 µs and two `getrusage` calls at ~0.17 µs) against a trim that costs milliseconds on a large heap, at a cadence of `SizedItem::SweepInterval` (10-120 s by node size) — a duty cycle below 1e-6 %, so keeping the RSS read debug-only would only have preserved the blind spot. Needs an explicit-bucket View (`addMicrosecondHistogramView`) because a trim on a multi-gigabyte heap runs well past the SDK default ceiling of 10,000, which would collapse exactly the slow trims this signal exists to catch into one saturated bucket. | +| `sweep_malloc_trim_minor_faults_total` | counter | `Application.cpp` — `ApplicationImp::trimHeapAndRecord` | Sweep Heap-Trim Faults & Reclaim Rate | Minor page faults taken _inside_ the trim call, from the `getrusage(RUSAGE_THREAD)` delta the report already carried. **Honest limitation, and it must not be over-claimed:** the delta is scoped to the trim call only, so it proves the trim itself faults — it does NOT prove the trim causes the faults taken later, as the caches refill and touch the pages the trim returned. That later re-fault cost is the actual mechanism the hypothesis proposes and it is NOT measured by this counter. Read the duration against sweep-job queueing rather than treating this number as the total cost of trimming. Emitted only when the delta is above zero: a trim that faulted nothing publishes no series, because a zero would read as "measured, and free" when the honest statement is that there was nothing to fault on. | +| `sweep_malloc_trim_reclaimed_kb_total` | counter | `Application.cpp` — `ApplicationImp::trimHeapAndRecord` | Sweep Heap-Trim Faults & Reclaim Rate | Resident kilobytes the trim actually returned to the kernel, so the cost above can be judged against what it bought. Cumulative and sign-corrected: `MallocTrimReport::deltaKB()` is after-minus-before, so a successful trim is NEGATIVE and the emit site publishes its magnitude. A sweep across which RSS GREW — another thread allocating faster than the trim released — is dropped rather than negated, because a counter cannot decrease and there is no reclaim of a negative size. Zero reclaim beside a non-zero duration is the worst reading: the trim is walking the heap and freeing nothing, which is pure cost. | +| `rotation_state` (`metric` = `in_flight` \| `copy_forward`) | observable gauge | `MetricsRegistry.cpp` — `registerRotationStateGauge` | Online-Delete Rotation Window & Copy-Forward Writes | The online-delete rotation window, and the running total of the extra writes it forces. A rotation rewrites into the new backend any node body the doomed archive serves, which is I/O an ordinary fetch would never perform and which scales with the archive — so it appears only on a populated, already-rotated database, which is precisely why it never shows on a fresh node. `copy_forward` comes from `DatabaseRotatingImp::copyForwardCount_`, which existed but was log-only AND reset by `rotate()` on every swap; a series that drops to zero per rotation cannot be rated, so a second never-reset total was added beside it and this gauge reads that. `in_flight` is exposed because the extra writes only happen inside that window, so a panel needs to know when to expect the total to move; the same total climbing while the flag reads 0 would mean the flag leaked, not that rotation is cheap. Polled rather than pushed because `DatabaseRotatingImp` is libxrpl and cannot include `xrpld/telemetry` — the two readings are taken through new `DatabaseRotating` accessors from the same collection tick pattern `registerNodeStoreGauge` already uses. **Publishes NO series when `online_delete` is not configured** (the `dynamic_cast` to `DatabaseRotating` fails and the callback returns early), deliberately: an absent series means "rotation is not configured", which a zero would misreport as "rotation is free". Rotation _duration_ is deliberately not recorded — see the note below the table. | +| `rotation_copy_node_restore_total` | counter | `SHAMapStoreImp.cpp` — `SHAMapStoreImp::copyNode` | Rotation Node Re-Store Rate | Nodes the rotation had to rescue because they were present in NEITHER backend, re-stored from the in-memory state map. The genuinely unmeasured rotation write: each is an extra store on top of the whole-state-map walk the rotation already performs, and the branch was warn-log-only, so the volume was invisible unless someone was reading logs. A non-zero rate says more than cost — it says an earlier rotation removed the only on-disk copy of a clean node (`cowid == 0`, so `flushDirty` skips it) that the current validated state map still reaches, and without the rescue it would later surface as an unresolvable `SHAMapMissingNode`. The node hash is deliberately not a label: it is unbounded runtime data that would mint one series per rescued node. Correlate a spike with the `copyNode` warning line in Loki, by node and time. | +| `ledger_replay_fallback_total` (`stage` = `skiplist` \| `delta`) | counter | `SkipListAcquire.cpp` / `LedgerDeltaAcquire.cpp` — `trigger` | Replay Fallback to Full Acquire (by stage) | A ledger-replay sub-task abandoning its shortcut and acquiring the whole ledger through `InboundLedger` instead, because too few connected peers support the `LedgerReplay` protocol feature. Both branches were debug-log-only, so a silently defeated replay optimisation left no metric at all — back-fill simply ran on the slower path with nothing to show why. Emitted once, on the transition into fallback, not at the acquire call, which re-runs on every later trigger. The `stage` label separates the skip-list acquire (which fetches the list of historical ledger hashes) from the per-ledger delta acquire, because they fail independently. | +| `ledger_replay_outcome_total` (`outcome` = `success` \| `timeout` \| `build_failed` \| `parameter_failed`) | counter | `LedgerReplayTask.cpp` — `LedgerReplayTask::recordOutcome` | Replay Outcomes (by terminal state) | Terminal state of every ledger-replay task, one emit per task. Every terminal path previously only set an internal `complete_`/`failed_` flag and wrote a log line, so a replay that never succeeded was indistinguishable from one that was never attempted. The outcome names the layer at fault: `timeout` means the deltas never arrived (a peer-supply problem), `build_failed` means a delta would not apply to its parent, and `parameter_failed` means a peer served a skip list inconsistent with what the task asked for — the latter two are data faults, not slowness. Read with `ledger_replay_fallback_total`: fallbacks rising while successes stay flat is replay-based catch-up degrading to full-ledger acquisition. | +| `ledger_quorum_publish` (`metric` = `trusted_validation_tally` \| `quorum_target`) | observable gauge | `MetricsRegistry.cpp` — `registerLedgerQuorumPublishGauge` | Trusted Validations vs Quorum Target | Trusted validations counted at the most recent pre-accept gate, beside the number that gate required. Snapshotted in `LedgerMaster::checkAccept` before the shortfall check, so a node that keeps failing the gate still reports both numbers — which is the whole point: the tally alone cannot say whether validations are accumulating toward quorum (slow, will finish) or plateaued below it (stuck). Read the sustained floor of the tally, not a single sample: each series is a snapshot of the last evaluation, and the first evaluation of each round runs before peer validations arrive, so a healthy node sawtooths. `quorum_target` is what the gate actually demanded, as opposed to `unl_quorum{metric="quorum"}` which is what the trusted list configures. When the trusted list disables quorum entirely (`getNeededValidations` returns `SIZE_MAX`) the target is reported as int64 max rather than wrapping to -1, so it reads far above any tally instead of inverting the comparison — the same sentinel handling as the `unl_quorum` gauge. | +| `ledger_quorum_publish` (`metric` = `publish_lag`) | observable gauge | `MetricsRegistry.cpp` — `registerLedgerQuorumPublishGauge` | Publish Lag (validated minus published) | Ledgers fully validated but not yet published to clients and subscribers: the validated sequence minus the published sequence, floored at zero. `pubLedgerSeq_` was never exported, so this gap was not derivable from any other series. Publishing trails validation by design and a small lag drains each round; a lag that stays positive or grows means validation is healthy and the publish pipeline is not, which is a distinct fault from anything the quorum or acquire signals can show. The two sequences are read as independent relaxed loads, so a sample taken mid-update may be off by one ledger for one poll — immaterial for a lag trend, and the price of not taking the LedgerMaster mutex on the metrics poll thread. | +| `ledger_quorum_publish` (`metric` = `time_to_first_validated_us`) | observable gauge | `MetricsRegistry.cpp` — `registerLedgerQuorumPublishGauge` | Time to First Validated Ledger | Microseconds from process start until the first ledger passed the pre-accept quorum gate. A one-shot measurement like `sync_state{metric="initial_full_duration_us"}`: written once under `mutex_` and never changed, so it has no trend. Exactly two readings are meaningful — a duration, meaning the node reached its first fully-validated ledger and this is how long that took, or 0, meaning it never has. Clamped to a minimum of 1 so a genuine sub-microsecond reading can never be confused with the never-reached zero. A value here alongside a zero on time-to-first-FULL, or the reverse, separates "reached the full server state" from "fully validated a ledger". | +| `ledger_quorum_shortfall_total` (`stage` = `pre_accept`) | counter | `LedgerMaster.cpp` — `LedgerMaster::checkAccept` | Pre-Accept Quorum Shortfall Rate | One increment per pre-accept gate evaluation rejected because the trusted validation tally was below quorum. Previously trace-log-only, which made a node that peers and receives validations yet never validates indistinguishable from an idle one. A non-zero rate is **not** by itself a fault: `doAccept` issues this node's own validation and calls `consensusBuilt` → `checkAccept` immediately, before peer validations for that ledger arrive, so the first evaluation of every round tallies short and is retried as validations come in — a healthy cluster emits this counter every round. The fault signature is the rate climbing well above the ledger-close rate while the tally stays flat below its target and `time_to_first_validated_us` stays at 0. Emitted while `mutex_` is held, which is safe against the metrics poll because every accessor the sync gauges read is a lock-free atomic load, so no OTel callback ever acquires `mutex_`. | +| `consensus_round_duration_ms` | histogram | `RCLConsensus.cpp` — `RCLConsensus::Adaptor::makeAcceptSpan` | Consensus Round Duration Distribution; Consensus Round Duration (p50/p95) | Wall-clock duration of a completed consensus round, in milliseconds. Promotes the long-standing `round_time_ms` span attribute into a native instrument: the attribute answers "how long did THIS round take" inside a trace, next to the proposers and disputes that explain it, while the histogram gives the distribution over time, which is what an alert or SLO panel needs and what a raw trace query cannot cheaply produce fleet-wide. Being native it is also **never sampled**, so it stays complete when tracing is head-sampled down. Recorded at exactly one site — `makeAcceptSpan` is the single function both the synchronous (`onForceAccept`) and asynchronous (`onAccept`) accept paths call once per round — so it can neither double-count nor be skipped, and it adds no per-peer, per-proposal or per-transaction work. **Explicit buckets** are registered for it in `MetricsRegistry::initExporterAndProvider` (`addRoundDurationHistogramView`, boundaries 500 ms → 120 s): the SDK default tops out at 10,000 ms, which would collapse every slow round into one saturated bucket and read every quantile as 10 s, and the consensus parameters themselves allow a round up to `ledgerAbandonConsensus` = 120 s. Needs **no collector change** — a native metric rides the existing OTLP → Prometheus path. | +| `consensus.validation.accept` (`validation_status`, `accept_gated`, `ledger_hash`, `ledger_seq`, `full_validation`) | span + span attr | `RCLValidations.cpp` — `handleNewValidation` | Trusted Validation Accept Rate by Status | One span per **trusted** validation as it reaches the ledger-acceptance gate, so its rate is bounded by the UNL size per ledger close (untrusted validations cannot move acceptance and get no span). Its trace id is derived from the **validated ledger's** hash, so it joins that ledger's trace rather than the round trace — see the per-ledger trace join below. `validation_status` is one of the six `ValStatus` values and only `current` continues to the gate, which is the difference between a node whose arriving validations are counting and one whose validations are all rejected; from the outside both look like a node that receives validations and never validates. `accept_gated` is true when another thread was already accepting the same ledger, which is why a trace can show a validation with no `ledger.validate` after it. Both are spanmetrics dimensions in **both** collector configs (6 and 2 values, bounded); `ledger_hash` / `ledger_seq` stay span-only and Tempo-indexed, since a per-ledger metric dimension would mint one series per ledger. | +| Per-ledger trace join (`ledger_hash` as trace-id seed) | trace scheme | `LedgerMaster.h/.cpp` — `LedgerMaster::makeLedgerTraceSpan` | n/a — read in Tempo, `{span.ledger_hash="LEDGER_HASH"}` | Makes one slow ledger readable as **one connected trace** instead of a set of orphan spans on different threads. `ledger.validate` (`LedgerMaster::checkAccept`), `ledger.store` (`LedgerMaster::storeLedger`) and `consensus.validation.accept` (`handleNewValidation`) each derive their trace id from the **same 32-byte ledger hash** via `SpanGuard::hashSpan`, which seeds the trace id from `hash[0:16]`. Nothing is propagated between the threads: every one of those sites already holds the ledger hash, which is the whole reason the key was chosen — the same pattern the apply pipeline uses to join `tx.preflight` / `tx.preclaim` / `tx.transactor` on the transaction id (`libxrpl/tx/applySteps.cpp`). Each span is a **true root** (deterministic trace id, empty parent), so the ledger's spans are siblings in one trace rather than a parent/child chain, which is the honest shape: none causes another directly and their order varies with the sync path (`checkAccept` is entered from a peer thread via `handleNewValidation`, from the acquire-completion job, and from the consensus thread via `switchLCL`). The full hash is also recorded as the `ledger_hash` attribute — it is what an operator searches by, and it is how a reader confirms two spans are genuinely the same ledger rather than a trace-id coincidence, since the trace id is only the leading 16 bytes. Asserted end-to-end by the `trace_join_groups` block in `expected_spans.json` (`assert_trace_join_groups` in `validate_telemetry.py`), which fails CI if the members stop sharing a trace. | +| `ledger.acquire` span (`outcome` = `complete` \| `failed` \| `abandoned`; `acquire_reason`, `timeouts`, `peer_count`, `ledger_hash`, `ledger_seq`) | span | `InboundLedger.cpp` — `InboundLedger::init` / `InboundLedger::finalizeAcquireSpan` | Ledger Acquire Phase Duration (p95 by phase) (its three phase children); Ledger Acquire Duration (Inbound Fetch) and Ledger Acquire Rate by Outcome, both on the **node-health** board | Parent of the three phase spans: one whole fetch of one missing ledger, from the first request to the terminal state. Pre-existing since Phase 6, extended here with `ledger_hash` (set at `init()`, so a fetch that never finishes is still findable in a trace search, and it is the trace-id seed that joins this span to the `ledger.validate`, `ledger.store` and `consensus.validation.accept` spans for the same ledger) and with the fourth `outcome` value `abandoned`, recorded when the acquire is destroyed by a sweep or shutdown before reaching a result. Without `abandoned` a stuck-then-swept fetch left the span with no `outcome` at all, so it vanished from every outcome rate — the exact failure a stalled fresh sync produces. `ledger_seq` is re-stamped at the end because a by-hash acquire starts with `seq_ == 0` and learns the sequence only when the header arrives. | +| `ledger.acquire.header` span (`outcome`, `timed_out`, `ledger_hash`, `ledger_seq`) | span | `InboundLedger.cpp` — `InboundLedger::syncPhaseSpans` / `endPhaseSpan` | Ledger Acquire Phase Duration (p95 by phase); Ledger Acquire Phase Outcomes (by phase & timeout) | Child of `ledger.acquire` covering the wait for the ledger header, which gates both tree phases — until it arrives the account-state and transaction root hashes are unknown, so nothing else can even be requested. The parent span is flat and its duration is dominated by the state tree, so a node stuck waiting to be TOLD what to fetch was indistinguishable from one stuck fetching it. No `missing_nodes`: a header is a single object, not a tree. Opened and closed by one idempotent state sync over the `have*_` flags rather than by open/close calls scattered through the fetch code, so the span boundary cannot drift out of step with the real phase boundary. | +| `ledger.acquire.astree` span (`outcome`, `timed_out`, `missing_nodes`, `ledger_hash`, `ledger_seq`) | span | `InboundLedger.cpp` — `InboundLedger::syncPhaseSpans` / `endPhaseSpan` | Ledger Acquire Phase Duration (p95 by phase); Ledger Acquire Phase Outcomes (by phase & timeout) | Child of `ledger.acquire` covering the account-state SHAMap fetch — **nearly all of the work in a real fresh sync**, and the reason the phase split exists: the flat parent span could not separate it from the small transaction tree. `missing_nodes` is read from the count `getMissingNodes()` already produced during its sweep, never recomputed, so no second tree walk is added. `outcome=timeout` together with a non-zero `missing_nodes` is the "peers are not serving this tree" signature; `timed_out` is a separate dimension from `outcome` because a phase can time out and still be retried by its parent acquire. | +| `ledger.acquire.txtree` span (`outcome`, `timed_out`, `missing_nodes`, `ledger_hash`, `ledger_seq`) | span | `InboundLedger.cpp` — `InboundLedger::syncPhaseSpans` / `endPhaseSpan` | Ledger Acquire Phase Duration (p95 by phase); Ledger Acquire Phase Outcomes (by phase & timeout) | Child of `ledger.acquire` covering the transaction SHAMap fetch. Usually completes long before the account-state phase, and that asymmetry is the point of separating them: the parent span's duration is the state tree's, not this one's, so a transaction tree that is genuinely slow is invisible inside it. Closed the moment its own tree completes (from `receiveNode`, `trigger` or `takeHeader`), so its duration is the real fetch time rather than stretching to the next trigger. | +| `txset.acquire` span (`outcome`, `txset_hash`, `duration_ms`, `timeouts`, `peer_count`) | span | `TransactionAcquire.cpp` — `TransactionAcquire::finalizeAcquireSpan` | Tx-Set Acquire Outcomes; Tx-Set Acquire Duration (p95) | One attempt to fetch the transaction set a consensus proposal referenced but this node did not hold. `TransactionAcquire` had **zero** telemetry of any kind before this, so a consensus round stalled waiting on a set was indistinguishable from an idle one. The sibling of `ledger.acquire`: same `TimeoutCounter` base, same trigger/onTimer/takeNodes shape, and the same `trace_ledger` flag so the two halves of a stuck sync cannot be enabled apart. `outcome` is `complete` \| `failed` \| `timeout` \| `abandoned`, stamped on both exits (`done()`, and the destructor when the round sweep in `InboundTransactions::newRound` drops a set that never arrived) by one idempotent finalizer. `timeout` is distinct from `failed` because the exhausted-budget path sets the terminal `failed_` flag too — that flag is how the timer loop stops — so the outcome rule checks the timeout first or every timeout would read as a data fault. `txset_hash` identifies WHICH set stalled and stays span-only: one metric series per consensus round would be unbounded. | +| `ledger.serve` span (`object_type`, `outcome`, `served_nodes`, `peer_id`, `ledger_seq`) | span | `PeerImp.cpp` — `PeerImp::processLedgerRequest` (the `JtLedgerReq` worker) | Ledger Serve Rate by Object Type | This node answering a peer's `TMGetLedger` request — the **supply side** of the sync exchange, and the trace-level companion to the existing `serve_refused_total` counter. The whole serve path had no span, so how long this node takes to answer, and whether it answered at all, was unobservable. A fresh trace root, because the request arrives from the wire on a shared worker whose ambient span is unrelated. `object_type` (`header` \| `tx` \| `as` \| `txset`) and `outcome` (`complete` \| `partial` \| `refused`) are both derived by shared rules in `LedgerSpanNames.h` rather than named per branch, which is what stops the eight exits of `processLedgerRequest` disagreeing about one request. `outcome` is derived from the reply itself — `served_nodes` is the reply's own node count and is 0 on all seven refusal paths — so nothing is accumulated and no work is added to the per-node assembly loop. `partial` means the reply hit `Tuning::kSoftMaxReplyNodes`, so the peer must make another round trip. | +| `peer.dial` span (`outcome`, `remote_endpoint`, `duration_ms`) | span | `ConnectAttempt.cpp` — `ConnectAttempt::reportOutcome` | Outbound Dial Outcomes (span-derived, per attempt) | One outbound connect attempt, as a per-attempt timeline rather than a rate. The trace-level companion to `overlay_connect_total` / `overlay_dial_latency_ms`: it carries the same six `outcome` values, set from the same `reportOutcome` funnel, so span and counter cannot disagree, and the funnel's existing first-call-wins guard makes the span exactly-once for free. What it adds is `remote_endpoint` — WHICH peer — which the counter deliberately cannot carry, because one series per peer address would be unbounded cardinality; it is a dedicated Tempo span column instead. A fresh trace root: a dial is the first thing a starting node does, so there is nothing to parent it to. An attempt torn down mid-dial by shutdown ends its span in the destructor with no `outcome`, which is the honest record of "never concluded" rather than a dropped span. | + +### Why rotation duration is not recorded + +An obvious fifth rotation signal would be how long a rotation takes, and it is +deliberately absent. `SHAMapStoreImp::run` calls `healthWait()` at eight points +inside the rotation sequence, and `healthWait()` blocks in +`std::this_thread::sleep_for(recoveryWaitTime_)` for as long as the node is not +`FULL` or its validated ledger is older than the age threshold. A wall-clock +duration spanning the rotation would therefore add a deliberate throttle to real +work and report the sum as one number — and the throttle dominates precisely +when the node is unhealthy, which is when the number would be read. + +Subtracting the sleep is not clean either: the waits are interleaved with the +work at eight sites, and instrumenting each interval separately would mean eight +new emit points inside a sequence whose control flow already has several early +returns. The two signals in the table answer the question rotation duration was +wanted for — how much extra I/O did rotation cause — directly and without that +ambiguity, so the duration is left unmeasured rather than published as a number +that conflates work with throttling. diff --git a/docker/telemetry/.env.grafanacloud-alloy.example b/docker/telemetry/.env.grafanacloud-alloy.example index a43e09e6d92..692acc58ec1 100644 --- a/docker/telemetry/.env.grafanacloud-alloy.example +++ b/docker/telemetry/.env.grafanacloud-alloy.example @@ -25,3 +25,11 @@ GRAFANACLOUD_OTLP_KEY= # --- Per-node label --- # host label applied to this node's scraped metrics (e.g. the node's hostname). XRPLD_HOST_LABEL= + +# --- Not settable here --- +# deployment.environment and xrpl.network.type are NOT env vars. They are +# literals inside `otelcol.processor.transform "tier"` in alloy/config.alloy, +# because OTTL statements do not expand sys.env(). Edit them there to match this +# node, or every dashboard's environment and network filters will read prod and +# mainnet. Dashboards filter on service_instance_id, which passes through from +# the node's own [telemetry] config and needs nothing here. diff --git a/docker/telemetry/alloy/config.alloy b/docker/telemetry/alloy/config.alloy index 87cb52ca160..e1c7967ffd6 100644 --- a/docker/telemetry/alloy/config.alloy +++ b/docker/telemetry/alloy/config.alloy @@ -234,6 +234,18 @@ otelcol.connector.spanmetrics "xrpld" { dimension { name = "outcome" } dimension { name = "acquire_reason" } + // Sync-diagnostic span dimensions. `timed_out` separates "slow but arriving" + // from "the retry budget ran out" on the acquire phase spans; `object_type` + // splits ledger.serve by what was asked for. Both are small closed sets. + dimension { name = "timed_out" } + dimension { name = "object_type" } + + // Trace-causality dimensions on consensus.validation.accept. The ledger_hash + // join key stays span-only -- one metric series per ledger would be unbounded + // -- and is indexed in Tempo instead. + dimension { name = "validation_status" } + dimension { name = "accept_gated" } + output { // Derived span metrics rejoin the metric stream at the batch processor. metrics = [otelcol.processor.batch.xrpld.input] diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index d33f2234fbb..1ab1d90a2a1 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -1502,7 +1502,7 @@ }, { "title": "Job Queue Backlog and Deferred by Type", - "description": "###### What this is:\n*Per-job-type queue depth, two series per type. Waiting is the whole backlog: every job enqueued for that type that has not started yet. Deferred is the subset of that backlog that is blocked specifically because the type is already running at its concurrency limit. Deferred is the leading indicator of backpressure, because JobQueue::addJob never rejects a job for queue pressure -- it returns success and defers instead, so a capped type under pressure produces no error and no dropped work. Without these the only evidence is latency, which appears after the harm is already done.*\n\n###### How it's computed:\n*Two targets, each the top 10 gauges by current value: jobq__waiting and jobq__deferred. JobQueue::collect snapshots both counters under the one lock that guards them, so the pair is read at the same instant and is directly comparable, then publishes them on the 1-second export cycle. Gauges exist only for non-special job types, so the 11 special types -- the ones declared with a limit of 0, which bypass the limit logic entirely and therefore never defer -- do not appear on either series.*\n\n###### Reading it:\n*Read the two together; the ratio is the diagnostic, not either value alone. Deferred is always a subset of waiting, because addRefCountedJob increments waiting for every job and deferred only for the ones that arrive while the type is at its limit. Waiting high with deferred at zero means the type has spare slots and the backlog is just arrival burstiness -- it will drain without intervention. Waiting high with deferred also high means the concurrency limit is the binding constraint, not the work. Both near zero is the normal state. These are depths, not rates: the value is how many jobs are queued right now. finishJob drains deferred one per completion, so a deferred line that stays elevated means arrivals are outpacing completions rather than one isolated burst. Only the 10 highest series per state are drawn, which on an idle node is arbitrary among the zeros and under load is exactly the types under pressure.*\n\n###### Healthy range:\n*Deferred zero on all types. Waiting near zero, with brief spikes during ledger close.*\n\n###### Watch for:\n*ledgerrequest deferred above zero: the 3-slot ledgerRequest queue is full, so TMGetLedger service to syncing peers is being delayed. Use LedgerReq Wait by Handler next to see which of its two producers is responsible. ledgerdata or fetchtxndata deferred: inbound ledger data cannot be absorbed fast enough, which is what makes validated ledger age grow. A waiting line that climbs steadily while deferred stays flat points at the worker pool or at slow jobs rather than at the limit. Note both are sampled once per export cycle, so a sub-second spike can be missed; a reading of zero is not proof that nothing was ever queued or deferred.*\n\n###### Keywords:\n- **Deferred job** *(per node)* — a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n- **Concurrency limit** *(per node)* — the cap on how many jobs of one type may run at once; a type at its cap cannot start more work.\n- **Job queue / job type** *(per node)* — xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[core/JobQueue.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)\n\n###### Function:\n`JobQueue::addRefCountedJob / JobQueue::collect`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#deferred-job)", + "description": "###### What this is:\n*Per-job-type queue depth, two series per type. Waiting is the whole backlog: every job enqueued for that type that has not started yet. Deferred is the subset of that backlog that is blocked specifically because the type is already running at its concurrency limit. Deferred is the leading indicator of backpressure, because JobQueue::addJob never rejects a job for queue pressure -- it returns success and defers instead, so a capped type under pressure produces no error and no dropped work. Without these the only evidence is latency, which appears after the harm is already done.*\n\n###### How it's computed:\n*Two targets, each the top 10 gauges by current value: jobq__waiting and jobq__deferred. JobQueue::collect snapshots both counters under the one lock that guards them, so the pair is read at the same instant and is directly comparable, then publishes them on the 10-second export cycle. Gauges exist only for non-special job types, so the 11 special types -- the ones declared with a limit of 0, which bypass the limit logic entirely and therefore never defer -- do not appear on either series.*\n\n###### Reading it:\n*Read the two together; the ratio is the diagnostic, not either value alone. Deferred is always a subset of waiting, because addRefCountedJob increments waiting for every job and deferred only for the ones that arrive while the type is at its limit. Waiting high with deferred at zero means the type has spare slots and the backlog is just arrival burstiness -- it will drain without intervention. Waiting high with deferred also high means the concurrency limit is the binding constraint, not the work. Both near zero is the normal state. These are depths, not rates: the value is how many jobs are queued right now. finishJob drains deferred one per completion, so a deferred line that stays elevated means arrivals are outpacing completions rather than one isolated burst. Only the 10 highest series per state are drawn, which on an idle node is arbitrary among the zeros and under load is exactly the types under pressure.*\n\n###### Healthy range:\n*Deferred zero on all types. Waiting near zero, with brief spikes during ledger close.*\n\n###### Watch for:\n*ledgerrequest deferred above zero: the 3-slot ledgerRequest queue is full, so TMGetLedger service to syncing peers is being delayed. Use LedgerReq Wait by Handler next to see which of its two producers is responsible. ledgerdata or fetchtxndata deferred: inbound ledger data cannot be absorbed fast enough, which is what makes validated ledger age grow. A waiting line that climbs steadily while deferred stays flat points at the worker pool or at slow jobs rather than at the limit. Note both are sampled once per export cycle, so a sub-second spike can be missed; a reading of zero is not proof that nothing was ever queued or deferred.*\n\n###### Keywords:\n- **Deferred job** *(per node)* — a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n- **Concurrency limit** *(per node)* — the cap on how many jobs of one type may run at once; a type at its cap cannot start more work.\n- **Job queue / job type** *(per node)* — xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[core/JobQueue.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)\n\n###### Function:\n`JobQueue::addRefCountedJob / JobQueue::collect`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#deferred-job)", "type": "timeseries", "gridPos": { "h": 8, diff --git a/docker/telemetry/grafana/dashboards/ledger-sync-health.json b/docker/telemetry/grafana/dashboards/ledger-sync-health.json new file mode 100644 index 00000000000..530a3373594 --- /dev/null +++ b/docker/telemetry/grafana/dashboards/ledger-sync-health.json @@ -0,0 +1,5607 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Annotate perf-iac runs", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac"], + "type": "tags" + }, + "type": "tags" + } + ] + }, + "description": "What this shows: Fresh-node ledger-sync diagnostics: pre-quorum bootstrap (Domain 0) and the ledger/tx-set acquire pipeline. \u2014 Use it to: Find out why a freshly started node is slow to reach, or never reaches, server_state full.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 55, + "panels": [], + "title": "Bootstrap (Domain 0)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Count of peer-hostname DNS resolutions, split by outcome (resolved or empty).*\n\n###### How it's computed:\n*Count over the selected range of completed resolutions grouped by outcome, per node.*\n\n###### Reading it:\n*Resolved should account for every attempt; the empty line should stay flat at zero.*\n\n###### Healthy range:\n*A short burst of resolved at startup, then flat. Non-zero empty is always a defect.*\n\n###### Watch for:\n*Any non-zero empty count means a configured bootstrap or [ips_fixed] hostname returned no address, so the node never even tries to dial that peer.*\n\n###### Keywords:\n- **DNS resolve** *(per node)* \u2014 turning a configured peer hostname into IP addresses before any dial is attempted; `outcome=empty` means the name resolved to nothing.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::reportDnsResolve`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#dns-resolve)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "DNS Resolve [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(dns_resolve_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "DNS Resolve Outcomes (Count By Outcome)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Time taken to resolve a configured peer hostname, at the 95th percentile.*\n\n###### How it's computed:\n*Resolution duration samples aggregated to their 95th percentile per node.*\n\n###### Reading it:\n*Lower is better; it is the delay before the node can start dialling peers.*\n\n###### Healthy range:\n*Tens of milliseconds against a healthy resolver.*\n\n###### Watch for:\n*Seconds-scale latency means the resolver is timing out and every bootstrap attempt is paying that delay before the first dial.*\n\n###### Keywords:\n- **DNS resolve** *(per node)* \u2014 turning a configured peer hostname into IP addresses before any dial is attempted; slow resolution delays the whole bootstrap.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::reportDnsResolve`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#dns-resolve)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Latency (ms)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_work_item) (dns_resolve_latency_ms_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"})), \"series\", \"P95 DNS Resolve\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "DNS Resolve Latency (p95)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Count of outbound peer connection attempts, split by terminal outcome.*\n\n###### How it's computed:\n*Count over the selected range of finished dials grouped by outcome, per node. Filter the outcome set with the Dial Outcome variable.*\n\n###### Reading it:\n*Connected should dominate. The failure lines name the stage that broke: tcp_fail (no route or refused), tls_fail (TLS handshake or shared-value exchange), self_connection (we dialled our own address -- a local misconfiguration), upgrade_fail (HTTP upgrade or protocol negotiation), timeout (no terminal state in time).*\n\n###### Healthy range:\n*Connected rising to the configured peer count, then flat with failures near zero.*\n\n###### Watch for:\n*All attempts landing on one failure outcome and no connected line \u2014 the node has no outbound peers and can never sync.*\n\n###### Keywords:\n- **Outbound dial latency** *(per node)* \u2014 an outbound peer connection attempt from TCP connect through TLS to protocol upgrade; each attempt ends in exactly one outcome.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ConnectAttempt.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/ConnectAttempt.cpp)\n\n###### Function:\n`ConnectAttempt::reportOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Dial Outcome [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 5, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(overlay_connect_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$dial_outcome\"}[$__range])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Outbound Dial Outcomes (Count By Outcome)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Time from starting an outbound peer dial to its terminal outcome, at the 95th percentile.*\n\n###### How it's computed:\n*Dial duration samples aggregated to their 95th percentile per node.*\n\n###### Reading it:\n*Lower is better. The series covers successes and failures together, so a rising p95 usually means attempts are ending in timeout rather than being refused fast.*\n\n###### Healthy range:\n*Tens to low hundreds of milliseconds on a local or same-region peer.*\n\n###### Watch for:\n*A p95 pinned near the dial timeout, which means peers accept the TCP connection but never complete the handshake.*\n\n###### Keywords:\n- **Outbound dial latency** *(per node)* \u2014 elapsed time of an outbound peer connection attempt, measured to whichever outcome ends it.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ConnectAttempt.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/ConnectAttempt.cpp)\n\n###### Function:\n`ConnectAttempt::reportOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Latency (ms)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 6, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_work_item) (overlay_dial_latency_ms_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"})), \"series\", \"P95 Outbound Dial\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Outbound Dial Latency (p95)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Count of peer handshakes rejected during protocol negotiation, split by reason.*\n\n###### How it's computed:\n*Count over the selected range of rejected handshakes grouped by reason, per node. Filter the reason set with the Handshake Reason variable.*\n\n###### Reading it:\n*Flat at zero is healthy. The reason names the exact check that rejected the peer, so one dominant reason is the fault to fix.*\n\n###### Healthy range:\n*Zero, or a low background count of self_connection and remote_ip_mismatch on a NAT'd host.*\n\n###### Watch for:\n*wrong_network or invalid_network_id \u2014 the node is configured for a different network than its peers and will never reach a quorum. clock_skew points at the local clock; session_verify_failed and bad_public_key at a misbehaving peer.*\n\n###### Keywords:\n- **Handshake negotiation failure** *(per node)* \u2014 a peer connection rejected after TLS while checking network id, clock, keys and addresses; the reason label names the failing check.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Handshake.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/Handshake.cpp)\n\n###### Function:\n`throwNegotiationFailure`\n\n###### References:\n[Peer protocol on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#handshake-negotiation-failure)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Handshake Fail [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 7, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (reason, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(handshake_negotiation_fail_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", reason=~\"$handshake_reason\"}[$__range])), \"series\", \"$1\", \"reason\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Handshake Negotiation Failures (Count By Reason)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Count of validator-list fetches from each configured UNL site, split by outcome.*\n\n###### How it's computed:\n*Count over the selected range of fetch attempts grouped by site and outcome, per node. Filter with the UNL Site and UNL Fetch Outcome variables.*\n\n###### Reading it:\n*accepted is the only success value. same_sequence and known_sequence are normal no-op refreshes of a list the node already holds. fetch_error, bad_status and parse_error are transport or content faults; expired, stale, untrusted, invalid and unsupported_version mean the list was retrieved but rejected.*\n\n###### Healthy range:\n*A first accepted per site at startup, then a steady low count of same_sequence refreshes.*\n\n###### Watch for:\n*A site with only fetch_error or bad_status is unreachable. Only expired or invalid means the site is reachable but its list is unusable, so no trusted keys are loaded from it.*\n\n###### Keywords:\n- **UNL fetch outcome** *(per node)* \u2014 the result of retrieving and applying a validator list from a configured site; `accepted` is the only success.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ValidatorSite.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/ValidatorSite.cpp)\n\n###### Function:\n`ValidatorSite::reportFetchOutcome`\n\n###### References:\n[UNL (Unique Node List)](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-fetch-outcome)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "UNL Fetch [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 8, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_join(sum by (outcome, site, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(unl_fetch_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", site=~\"$unl_site\", outcome=~\"$unl_outcome\"}[$__range])), \"series\", \" \", \"outcome\", \"site\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "UNL Fetch Outcomes (Count By Site & Outcome)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Trusted validator keys currently in effect, plotted against the number of agreeing validations a ledger needs.*\n\n###### How it's computed:\n*Two series read from the same gauge: trusted_keys (usable UNL size) and quorum (validations required to declare a ledger validated).*\n\n###### Reading it:\n*Trusted Keys must sit above Quorum. Where the lines cross, or where Trusted Keys is zero, the node cannot reach a quorum and will never validate a ledger no matter how healthy the rest of the pipeline looks.*\n\n###### Healthy range:\n*Trusted Keys comfortably above Quorum and both flat.*\n\n###### Watch for:\n*Trusted Keys at zero (no usable UNL loaded) or below Quorum. Steps in Quorum track validator-list changes; steps down in Trusted Keys mean keys were dropped.*\n\n###### Keywords:\n- **UNL quorum headroom** *(per node)* \u2014 trusted UNL key count minus the required quorum; at or below zero the node can never declare a ledger validated.\n- **UNL (Unique Node List)** *(per node)* \u2014 the list of validators a node trusts not to collude; the basis for its consensus and quorum.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerUnlQuorumGauge`\n\n###### References:\n[UNL (Unique Node List)](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-quorum-headroom)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Validator Keys", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 9, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(unl_quorum{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"trusted_keys\"}, \"series\", \"Trusted Keys\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(unl_quorum{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"quorum\"}, \"series\", \"Quorum\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "UNL Trusted Keys vs Quorum", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Spare trusted validator keys above the required quorum \u2014 the single number that says whether this node can ever validate.*\n\n###### How it's computed:\n*Trusted key count minus the required quorum, matched per node.*\n\n###### Reading it:\n*Positive is healthy. Zero or negative (red) means the trusted UNL is too small to ever satisfy quorum, so the node will stay short of a validated ledger.*\n\n###### Healthy range:\n*Positive; the exact figure depends on UNL size and the configured quorum.*\n\n###### Watch for:\n*Zero or below. Pair it with UNL Fetch Outcomes (Count By Site & Outcome): a site stuck on fetch_error or expired is the usual cause of a UNL too small to meet quorum.*\n\n###### Keywords:\n- **UNL quorum headroom** *(per node)* \u2014 trusted UNL key count minus the required quorum; at or below zero the node can never declare a ledger validated.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerUnlQuorumGauge`\n\n###### References:\n[Validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-quorum-headroom)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "green", + "value": 2 + } + ] + }, + "unit": "short", + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(unl_quorum{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"trusted_keys\"} - ignoring(metric) unl_quorum{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"quorum\"}, \"series\", \"Quorum Headroom\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "UNL Quorum Headroom", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How far the network's agreed close time sits from this node's own clock.*\n\n###### How it's computed:\n*Signed offset in seconds, plus its magnitude so a threshold band applies in either direction.*\n\n###### Reading it:\n*Both series should hug zero. A negative signed value means the local clock runs ahead of the network, positive means it lags. The magnitude is what matters: the threshold lines sit at 1s (suspicious) and 60s.*\n\n###### Healthy range:\n*Magnitude under 1 second.*\n\n###### Watch for:\n*A magnitude above 1 second that does not decay, which delays consensus participation. Note that server_info only surfaces close_time_offset once the magnitude reaches 60 seconds, so this panel sees skew long before the API does; a persistent offset is a local NTP fault, not a network one.*\n\n###### Keywords:\n- **Clock close offset** *(per node)* \u2014 the difference between the network's agreed close time and this node's clock; a persistent offset delays consensus participation.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerClockSkewGauge`\n\n###### References:\n[Ledger close times on xrpl.org](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#clock-close-offset)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "Close Offset (s)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 60 + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 11, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(clock_close_offset_seconds{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"offset\"}, \"series\", \"Close Offset\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(abs(clock_close_offset_seconds{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"offset\"}), \"series\", \"Offset Magnitude\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "Clock Close Offset", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 56, + "panels": [], + "title": "Peer supply", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How many connected peers can actually serve the ledger this node needs next, next to how many advertise a ledger range at all.*\n\n###### How it's computed:\n*peer_ledger_supply series peers_reporting, peers_serving_validated and peers_serving_next, counted from the ledger ranges connected peers advertise and compared against this node's own validated sequence.*\n\n###### Reading it:\n*peers_serving_next is the line that matters: it counts peers holding validated+1, the one ledger this node must acquire before it can advance. peers_serving_validated counts peers that can re-serve the ledger already held, and peers_reporting is the denominator \u2014 peers that advertised any range at all.*\n\n###### Healthy range:\n*peers_serving_next at or near peers_reporting.*\n\n###### Watch for:\n*peers_serving_next at 0 while peers_reporting is above 0. No peer this node is connected to holds the ledger it needs, so this is a supply problem, not a slow-peer problem \u2014 retrying harder or waiting longer cannot fix it, the node needs different peers. Distinct from the acquire panels above, which measure requests already sent to peers that did hold the data.*\n\n###### Keywords:\n- **Peer ledger supply** *(per node)* \u2014 the count of connected peers whose advertised ledger range covers a sequence this node needs.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerLedgerSupplyGauge`\n\n###### References:\n[Complete ledger ranges](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#complete-ledger-ranges) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#fresh-node-sync-diagnostics)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Peers", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 62 + }, + "id": 28, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peer_ledger_supply{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=~\"peers_reporting|peers_serving_validated|peers_serving_next\"}, \"series\", \"$1\", \"metric\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Peers Able to Serve Needed Sequence", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Two distances that say whether the sequence this node needs is inside the window its peers can serve: how much history is still reachable behind it, and how far the network tip is ahead of it.*\n\n###### How it's computed:\n*History Headroom = this node's validated_ledger_seq minus peer_ledger_supply supply_min_seq. Tip Gap = supply_max_seq minus validated_ledger_seq. Both operands are gated on being above zero, so neither line is drawn until the node has a validated ledger and at least one peer has advertised a range.*\n\n###### Reading it:\n*Plotted as distances rather than absolute sequences on purpose. The raw sequences sit around 105,890,000 and roughly 300,000 apart, so on one linear axis the tip movement that shows whether sync is progressing is a fraction of a percent of the scale and reads as a flat line. Differencing puts the meaning on the axis: zero is the boundary in both cases. History Headroom uses the left axis, Tip Gap the right, because the two differ by orders of magnitude.*\n\n###### Healthy range:\n*History Headroom comfortably positive and roughly steady. Tip Gap at or near zero.*\n\n###### Watch for:\n*History Headroom crossing below zero \u2014 every connected peer has pruned the history this node still needs, so the gap can never be closed from this peer set and the node needs different peers. Tip Gap growing steadily means the network is closing ledgers faster than this node validates them. Pair with Peers Able to Serve Needed Sequence: that panel counts how many peers can serve the next ledger, this one says how far outside the served window the node has drifted.*\n\n###### Keywords:\n- **History headroom** *(per node)* \u2014 validated sequence minus the lowest sequence any connected peer offers; how much of the history behind this node is still reachable.\n- **Tip gap** *(per node)* \u2014 the highest sequence any connected peer offers minus this node's validated sequence; how far behind the peer-reported tip this node is.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query subtracts the two series and aggregates them.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerLedgerSupplyGauge`\n\n###### References:\n[Complete ledger ranges](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#complete-ledger-ranges) \u00b7 [Back-fill / catch-up](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#back-fill-catch-up)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Ledgers", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + }, + "axisSoftMin": 0 + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "Tip Gap.*" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.axisLabel", + "value": "Ledgers behind tip" + }, + { + "id": "custom.axisSoftMin", + "value": -5 + }, + { + "id": "custom.axisSoftMax", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 62 + }, + "id": 29, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace((server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"validated_ledger_seq\"} > 0) - ignoring(metric) (peer_ledger_supply{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"supply_min_seq\"} > 0), \"series\", \"History Headroom\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace((peer_ledger_supply{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"supply_max_seq\"} > 0) - ignoring(metric) (server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"validated_ledger_seq\"} > 0), \"series\", \"Tip Gap\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "Peer Supply Window Margin (history headroom vs tip gap)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*PeerFinder's slot accounting in one snapshot: outbound and inbound slots in use against their configured maxima, plus outbound attempts still in flight.*\n\n###### How it's computed:\n*peerfinder_slot_census series out_active, out_max, in_active, in_max and connecting, all read from ONE PeerFinder snapshot taken under a single lock so the five values describe the same instant.*\n\n###### Reading it:\n*out_active should climb to out_max and stay there. connecting counts outbound attempts started but not yet resolved either way, so it is the in-flight term the active counts cannot show.*\n\n###### Healthy range:\n*out_active at out_max; connecting low and transient.*\n\n###### Watch for:\n*out_active pinned below out_max while connecting stays non-zero \u2014 dials are being started and never completing, so the node is trying and failing rather than sitting idle. Note out_active and in_active are also exported separately as the legacy beast::insight gauges peer_finder_active_outbound_peers and peer_finder_active_inbound_peers. Those two are unrelated single series read at different instants with no capacity, attempt or cache terms, so no reading of them can distinguish this case. The census exists to report all nine fields from one snapshot under one lock, so they are mutually consistent and joinable on one labelset.*\n\n###### Keywords:\n- **PeerFinder slot census** *(per node)* \u2014 one consistent snapshot of PeerFinder's outbound and inbound slot use, capacities, in-flight attempts, fixed peers and address caches.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerSlotCensusGauge`\n\n###### References:\n[Overlay](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay) \u00b7 [Outbound dial latency](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Slots", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 74 + }, + "id": 30, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peerfinder_slot_census{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=~\"out_active|out_max|in_active|in_max|connecting\"}, \"series\", \"$1\", \"metric\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "PeerFinder Slot Census", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The address supply PeerFinder dials from \u2014 its boot cache and live cache \u2014 next to configured versus currently connected fixed peers.*\n\n###### How it's computed:\n*peerfinder_slot_census series bootcache, livecache, fixed_configured and fixed_active, read from the same single snapshot as the slot census.*\n\n###### Reading it:\n*bootcache holds seed addresses persisted across restarts; livecache holds addresses learned from peers while running. fixed_active is how many of the fixed peers named in the config are connected right now.*\n\n###### Healthy range:\n*bootcache and livecache non-zero; fixed_active equal to fixed_configured.*\n\n###### Watch for:\n*bootcache at 0 on a fresh node means there are no seed addresses to dial at all, so no outbound connection is ever attempted and every downstream sync signal on this dashboard stays empty for a reason that has nothing to do with sync. fixed_active below fixed_configured means a configured fixed peer is unreachable.*\n\n###### Keywords:\n- **PeerFinder address cache** *(per node)* \u2014 the boot cache (seed addresses persisted across restarts) and live cache (addresses learned from peers) that supply outbound dial candidates.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerSlotCensusGauge`\n\n###### References:\n[Overlay](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay) \u00b7 [DNS resolve](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#dns-resolve)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Count", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 74 + }, + "id": 31, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(peerfinder_slot_census{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=~\"bootcache|livecache|fixed_configured|fixed_active\"}, \"series\", \"$1\", \"metric\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "PeerFinder Address Caches & Fixed Peers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Count of inbound peer connection handoffs, split by terminal outcome.*\n\n###### How it's computed:\n*Count over the selected range of finished inbound handoffs grouped by outcome, per node. Filter the outcome set with the Accept Outcome variable.*\n\n###### Reading it:\n*accepted should dominate. The failure outcomes name what rejected the connection: resource_limit and no_slot are this node's own capacity, protocol_mismatch and bad_cookie are the peer or the network identity, handshake_error and local_endpoint_fail are the transport.*\n\n###### Healthy range:\n*accepted dominant, failures near zero.*\n\n###### Watch for:\n*Read this together with Outbound Dial Outcomes (Count By Outcome) above, which is the outbound twin on overlay_connect_total{outcome}. The two share the same one-outcome-per-attempt shape, so together they give the full in/out split: a node that accepts nothing but dials successfully has a very different fault from one that can neither dial nor accept. A steady no_slot or resource_limit is this node refusing peers it has no room for, which is capacity rather than a fault.*\n\n###### Keywords:\n- **Inbound peer accept outcome** *(per node)* \u2014 the terminal result of an inbound peer connection handoff; each handoff ends in exactly one outcome.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::onHandoff`\n\n###### References:\n[Overlay](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay) \u00b7 [Outbound dial latency](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Peer Accept [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 86 + }, + "id": 32, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(peer_accept_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$accept_outcome\"}[$__range])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Inbound Peer Accept Outcomes (Count By Outcome)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Number of times peer connections are closed, split by why they closed and by which side opened them.*\n\n###### How it's computed:\n*Count over the selected range of peer closes grouped by reason and direction (inbound or outbound), per node. Filter with the Disconnect Reason and Disconnect Direction variables.*\n\n###### Reading it:\n*The split separates our-fault backpressure from topology and network faults. large_sendq and charge_resources are this node shedding a peer it cannot keep up with. not_useful and ping_timeout are topology and liveness. read_error and write_error are the transport. graceful, stopping and shutdown are ordinary lifecycle, not faults.*\n\n###### Healthy range:\n*Mostly graceful; the fault reasons near zero.*\n\n###### Watch for:\n*A sustained large_sendq or charge_resources means this node is the bottleneck and is dropping peers \u2014 which removes the very peers it needs to sync from, so a slow node makes itself slower. A rising not_useful or ping_timeout on outbound points at the peer set instead. Distinct from the single unlabelled peer-disconnect total on the Node Health dashboard, which cannot say which of these is happening.*\n\n###### Keywords:\n- **Peer disconnect reason** *(per node)* \u2014 the cause recorded when a peer connection is closed, paired with whether that connection was inbound or outbound.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::close`\n\n###### References:\n[Resource disconnect](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-disconnect) \u00b7 [Insane / diverged peers](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#insane-diverged-peers)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Peer Disconnect [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 86 + }, + "id": 33, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_join(sum by (reason, direction, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(peer_disconnect_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", reason=~\"$disconnect_reason\", direction=~\"$disconnect_direction\"}[$__range])), \"series\", \" \", \"reason\", \"direction\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Peer Disconnects (Count By Reason & Direction)", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 98 + }, + "id": 57, + "panels": [], + "title": "Sync state", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How long the node took, from process start, to reach the full server state for the first time.*\n\n###### How it's computed:\n*sync_state series initial_full_duration_us, converted from microseconds to seconds.*\n\n###### Reading it:\n*A value appears only once the node has actually synced. Zero (red) means it has never reached full \u2014 that is the signal, not missing data.*\n\n###### Healthy range:\n*Seconds to a few minutes on a warm node; longer on a fresh one that must acquire history.*\n\n###### Watch for:\n*A flat zero. The value never changes after the first full transition, so it either fills in or the node never synced.*\n\n###### Keywords:\n- **Time to first FULL** *(per node)* \u2014 elapsed time from process start until the node first reached the full server state.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerSyncStateGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#time-to-first-full)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 600 + }, + { + "color": "red", + "value": 1800 + } + ] + }, + "unit": "s", + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}" + } + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 0, + "y": 99 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sync_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"initial_full_duration_us\"} / 1e6, \"series\", \"Time to FULL\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Time to First FULL", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Whether the node is still waiting to see a full network ledger before it will participate.*\n\n###### How it's computed:\n*sync_state series network_ledger_gate: 1 while the gate is closed, 0 once it opens.*\n\n###### Reading it:\n*0 (green) is healthy. A persistent 1 (red) means the node has never seen a complete network ledger, so it refuses transactions and can never reach full no matter how healthy the rest of the pipeline looks.*\n\n###### Healthy range:\n*0 within the first few minutes of startup.*\n\n###### Watch for:\n*A 1 that never clears. Pair it with the Bootstrap row \u2014 no peers or no quorum is the usual cause.*\n\n###### Keywords:\n- **Network ledger gate** *(per node)* \u2014 the startup guard that holds a node back until it has seen a full ledger from the network.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerSyncStateGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#network-ledger-gate)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short", + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}" + } + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 8, + "y": 99 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sync_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"network_ledger_gate\"}, \"series\", \"Gate Closed\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Network Ledger Gate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How many seconds the server's main loop has currently been unresponsive.*\n\n###### How it's computed:\n*sync_state series server_stall_seconds, the same duration the load monitor logs as \"Server stalled for N seconds\".*\n\n###### Reading it:\n*0 (green) is healthy. Any non-zero value means the main loop missed its heartbeat for at least the 10 second reporting threshold.*\n\n###### Healthy range:\n*0.*\n\n###### Watch for:\n*Any sustained non-zero value. A stall points at main-loop overload rather than sync data starvation, so check job-queue depth and disk latency next, not peer supply.*\n\n###### Keywords:\n- **Server stall** *(per node)* \u2014 the main loop failing to check in with the load monitor, measured in seconds of unresponsiveness.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LoadManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/LoadManager.cpp)\n\n###### Function:\n`LoadManager::updateStallState`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#server-stall)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "s", + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}" + } + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 16, + "y": 99 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sync_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"server_stall_seconds\"}, \"series\", \"Stalled\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Server Stall", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How far this node's validated ledger trails the highest ledger any connected peer reports holding.*\n\n###### How it's computed:\n*sync_state series ledgers_behind: the peer-reported network tip minus this node's validated sequence, floored at zero.*\n\n###### Reading it:\n*Trending to zero is healthy convergence. Flat or rising means the node is not catching up. Zero also covers \"no peer has reported a newer ledger\", which on a node with no peers is the same thing.*\n\n###### Healthy range:\n*0 to 1 on a synced node.*\n\n###### Watch for:\n*A plateau or a climb during initial sync: the node is acquiring slower than the network advances, so it will never converge. Correlate with the acquire and job-queue panels.*\n\n###### Keywords:\n- **Ledgers behind network** *(per node)* \u2014 the gap between the peer-reported network tip and this node's own validated ledger sequence.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerSyncStateGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledgers-behind-network)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Ledgers", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 50 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 111 + }, + "id": 15, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sync_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"ledgers_behind\"}, \"series\", \"Ledgers Behind\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Ledgers Behind Network", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How often the server enters a NEW stall episode, as opposed to how long one stall lasts.*\n\n###### How it's computed:\n*Count over the selected range of server_stall_events_total, which counts once per stall episode rather than once per stalled second.*\n\n###### Reading it:\n*Flat at zero is healthy. Read it beside the Server Stall stat: a rising count means repeated fresh stalls, while a flat count with a large stall value means one long unresolved stall.*\n\n###### Healthy range:\n*0.*\n\n###### Watch for:\n*Any repeating count. Recurring short stalls and one long stall have different causes, and this panel is what separates them.*\n\n###### Keywords:\n- **Server stall** *(per node)* \u2014 the main loop failing to check in with the load monitor, measured in seconds of unresponsiveness.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LoadManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/LoadManager.cpp)\n\n###### Function:\n`LoadManager::updateStallState`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#server-stall)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 111 + }, + "id": 16, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(last_over_time(server_stall_events_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range]), \"series\", \"Stall Episodes\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Server Stall Episodes (Count)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Which edges of the sync state machine the node actually traversed over the dashboard window, counted per from-to pair.*\n\n###### How it's computed:\n*increase(state_changes_total) over the selected range, summed by the from and to labels.*\n\n###### Reading it:\n*A clean fresh sync shows single traversals along disconnected to connected to syncing to tracking to full. Repeated counts on the full-to-connected edge paired with connected-to-full is flapping.*\n\n###### Healthy range:\n*One traversal per climb edge and nothing on the reverse edges.*\n\n###### Watch for:\n*High counts on a reverse edge such as full to connected: the node reaches full and keeps losing it, which an unlabelled state-change total cannot distinguish from a clean climb.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 how fully the node is participating, in ascending order: disconnected, connected, syncing, tracking, full.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::setMode`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "${__field.labels.series}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 123 + }, + "id": 17, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_join(sum by (from, to, service_instance_id, xrpl_branch, xrpl_work_item) (increase(state_changes_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", from=~\"$mode_from\", to=~\"$mode_to\"}[$__range])), \"series\", \" \u2192 \", \"from\", \"to\")", + "refId": "A" + } + ], + "title": "Mode Transitions by Edge", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 135 + }, + "id": 58, + "panels": [], + "title": "Ledger acquire & SHAMap fetch", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Outstanding SHAMap nodes the busiest in-flight ledger acquire still needs, split by tree. This is the signal that separates a sync that is merely slow from one that will never finish.*\n\n###### How it's computed:\n*sync_acquire series missing_state_nodes_max and missing_tx_nodes_max: the largest outstanding node count across all in-flight acquires, refreshed after each getMissingNodes sweep. The maximum, not the sum, so one stuck acquire stays visible instead of being averaged away.*\n\n###### Reading it:\n*Falling toward zero means the acquire is progressing. A value pinned at 256 is the sweep cap, meaning there are at least that many nodes outstanding. Zero on one tree with a value on the other means that tree is already complete.*\n\n###### Healthy range:\n*Falling to 0 within seconds per ledger.*\n\n###### Watch for:\n*A flat, non-zero value across several minutes: no peer is serving that tree, so this acquire will never complete. Pair with Acquire Stalls \u2014 No Progress (Count) \u2014 both flat and climbing together is the definitive stuck-sync signature.*\n\n###### Keywords:\n- **Missing SHAMap node** *(per node)* \u2014 a tree node this node needs to complete a ledger but does not yet hold.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerSyncAcquireGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#missing-shamap-node)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Nodes", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 256 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 136 + }, + "id": 18, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sync_acquire{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=~\"missing_(state|tx)_nodes_max\"}, \"series\", \"$1 tree\", \"metric\", \"missing_(.*)_nodes_max\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Missing SHAMap Nodes per Acquire (state/tx)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Count of ledger-acquire timeouts where not a single new node arrived since the previous timeout.*\n\n###### How it's computed:\n*count of sync_acquire_no_progress_total, incremented on each acquire timeout whose progress flag was false. The acquire timer fires every 3 seconds at most.*\n\n###### Reading it:\n*Zero means every timeout window saw at least some new data. Any sustained count means acquires are repeatedly timing out with nothing received.*\n\n###### Healthy range:\n*0 on a synced node; brief non-zero bursts during initial sync are normal.*\n\n###### Watch for:\n*A sustained count together with a flat Missing SHAMap Nodes panel: the node is asking and no peer is answering. Check peer count and whether any peer holds the ledger range being requested.*\n\n###### Keywords:\n- **Acquire stall** *(per node)* \u2014 an acquire timeout in which no new SHAMap node was received, so the acquire made no progress at all.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::onTimer`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#acquire-stall)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Acquire Stalls ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 136 + }, + "id": 19, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(last_over_time(sync_acquire_no_progress_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range]), \"series\", \"Stalled Timeouts\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Acquire Stalls \u2014 No Progress (Count)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate of SHAMap nodes received during ledger acquire, split by whether each node was useful, already held, or rejected as invalid.*\n\n###### How it's computed:\n*rate of sync_addnode_total by outcome (good / duplicate / invalid), emitted once per received packet from the batch tally that the acquire trace log already computed.*\n\n###### Reading it:\n*good is real progress. duplicate is bandwidth spent on nodes already held. invalid is a peer sending data that failed validation. Traffic-level metrics show all three as healthy throughput, which is why the split matters.*\n\n###### Healthy range:\n*good dominant during sync; a small duplicate share is normal.*\n\n###### Watch for:\n*A rising invalid share points at a specific misbehaving peer. A duplicate share that swamps good means peers keep re-sending known data, so the acquire burns bandwidth without progressing.*\n\n###### Keywords:\n- **Add-node outcome** *(per node)* \u2014 the result of applying one received SHAMap node: good (new and valid), duplicate (already held), or invalid (rejected).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::recordBatchOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#add-node-outcome)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Nodes / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 148 + }, + "id": 20, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (rate(sync_addnode_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$addnode_outcome\"}[$__rate_interval])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Add-Node Outcomes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Peer packets stashed across all in-flight acquires waiting to be applied, alongside how many acquires are running.*\n\n###### How it's computed:\n*sync_acquire series received_data_depth (summed across acquires) and in_flight (the acquire count). The depth mirrors the receive stash size; in_flight gives the context that makes an all-zero reading legible.*\n\n###### Reading it:\n*A depth near zero means node data is applied as fast as it arrives. in_flight at zero means the node is idle, which is why an all-zero Missing Nodes panel is not by itself a healthy reading.*\n\n###### Healthy range:\n*Depth 0 to a few; in_flight low single digits during sync.*\n\n###### Watch for:\n*A growing depth means arriving data outpaces processing, which is a job-queue or disk problem rather than a peer-supply one. Check the job-queue backlog next.*\n\n###### Keywords:\n- **Received-data stash** *(per node)* \u2014 peer packets held for later processing because the acquire cannot apply them as fast as they arrive.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerSyncAcquireGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#received-data-stash)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Count", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*received_data_depth.*" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 16 + } + ] + } + }, + { + "id": "custom.thresholdsStyle", + "value": { + "mode": "line" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 148 + }, + "id": 23, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sync_acquire{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=~\"received_data_depth|in_flight\"}, \"series\", \"$1\", \"metric\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Received-Data Stash Depth & In-Flight Acquires", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Count of ledger acquires split by whether the local node store already held the whole ledger or the data had to come from peers.*\n\n###### How it's computed:\n*count of sync_acquire_source_total by source, emitted once per new acquire right after the first local-store lookup.*\n\n###### Reading it:\n*network dominant during initial sync is expected \u2014 nothing is local yet. local dominant on a warm node means the store is serving requests without peer traffic.*\n\n###### Healthy range:\n*Mostly local on a warm node with complete history.*\n\n###### Watch for:\n*Sustained network on a node that should already hold the range: the local store is not retaining data, so sync is disk-bound rather than peer-bound. Read with the SHAMap cache hit-rate panel.*\n\n###### Keywords:\n- **Acquire source** *(per node)* \u2014 whether a ledger acquire was satisfied entirely from the local node store or required fetching from peers.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::init`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#acquire-source)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Acquire Source [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 160 + }, + "id": 21, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (source, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(sync_acquire_source_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", source=~\"$acquire_source\"}[$__range])), \"series\", \"$1\", \"source\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Acquire Source (Count: Local vs Network)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Fraction of SHAMap tree-node lookups served from memory instead of the node store. The layer above the NuDB cache: a miss here is what causes a node-store read.*\n\n###### How it's computed:\n*shamap_cache_hit_rate series treenode, from TaggedCache::getHitRate() on the node family's tree-node cache, normalized from 0-100 to 0.0-1.0.*\n\n###### Reading it:\n*Near 1.0 on a warm node. Low during a fresh sync while the cache fills. Distinct from the NuDB Cache Hit Ratio panel on the Ledger Data Sync dashboard, which measures the node-store layer beneath this one.*\n\n###### Healthy range:\n*> 0.9 on a warm node.*\n\n###### Watch for:\n*A persistently low rate on a node that should be warm: the working set does not fit the cache, or continuous re-acquisition is churning it, so every tree walk pays disk latency. Read with Acquire Source.*\n\n###### Keywords:\n- **SHAMap cache hit rate** *(per node)* \u2014 the share of SHAMap tree-node lookups answered from the in-memory cache rather than the node store.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerCacheHitRateDetailGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#shamap-cache-hit-rate)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Hit Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 160 + }, + "id": 22, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(shamap_cache_hit_rate{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"TreeNode Cache\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "SHAMap TreeNode Cache Hit Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 172 + }, + "id": 59, + "panels": [], + "title": "Job queue", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Share of the worker-thread pool currently executing a job. This is the pool-wide view: when the pool itself is exhausted, every subsystem queued behind it looks independently slow, and this panel attributes the whole slowdown once.*\n\n###### How it's computed:\n*jobq_saturation running_tasks divided by worker_threads (the denominator is clamped to at least 1). The thread count is exported rather than hardcoded because it is derived at startup from [workers], node size and hardware concurrency.*\n\n###### Reading it:\n*Below 80% (green) means the pool has spare capacity, so a slow stage is that stage's own fault. At 100% every worker is busy \u2014 read Worker Pool Capacity & Total Backlog next: 100% with a queue is an exhausted pool, 100% with an empty queue is merely busy.*\n\n###### Healthy range:\n*< 80%.*\n\n###### Watch for:\n*A sustained 100% together with a non-zero backlog. Every job type is then starved by the pool, so fix pool capacity or the long-running jobs holding it, not the individual victim subsystems.*\n\n###### Keywords:\n- **Worker-pool saturation** *(per node)* \u2014 worker threads executing a job as a share of the threads the pool is configured to run.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerJobQueueSaturationGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#worker-pool-saturation)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "percentunit", + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 173 + }, + "id": 26, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(jobq_saturation{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"running_tasks\"} / ignoring(metric) clamp_min(jobq_saturation{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"worker_threads\"}, 1), \"series\", \"Worker saturation\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Worker Pool Saturation", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The three raw worker-pool numbers behind the saturation ratio: tasks in flight, threads configured, and total jobs queued across every job type.*\n\n###### How it's computed:\n*jobq_saturation series running_tasks, worker_threads and total_waiting, all read from one JobQueue sample so the ratio and the backlog describe the same instant.*\n\n###### Reading it:\n*running_tasks tracking worker_threads means the pool is fully committed. total_waiting is what makes that legible: queued work behind a fully committed pool is exhaustion, no queued work is just a busy moment.*\n\n###### Healthy range:\n*running_tasks below worker_threads; total_waiting near 0.*\n\n###### Watch for:\n*total_waiting climbing while running_tasks is pinned at worker_threads. Distinct from the jobq_job_count depth panel on the Ledger Data Sync dashboard, which has no capacity term at all, so a depth reading there cannot say whether the pool is the cause.*\n\n###### Keywords:\n- **Worker thread pool** *(per node)* \u2014 the fixed set of threads that execute all job-queue work, sized at startup.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerJobQueueSaturationGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#worker-pool-saturation)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Count", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 173 + }, + "id": 27, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(jobq_saturation{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=~\"$saturation_metric\"}, \"series\", \"$1\", \"metric\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Worker Pool Capacity & Total Backlog", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 185 + }, + "id": 60, + "panels": [], + "title": "Quorum & publish", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Trusted validations counted at the most recent pre-accept gate, plotted against the number that gate required.*\n\n###### How it's computed:\n*Two series from the same gauge: trusted_validation_tally (agreeing trusted validations seen for the candidate ledger, after the negative-UNL filter) and quorum_target (what the gate demanded). Both are snapshotted on every gate evaluation, whether it passed or failed, so a node that keeps failing still reports both numbers.*\n\n###### Reading it:\n*Read the shape of the tally, not any single value. A tally climbing toward the target is a slow sync that will finish, so keep waiting. A tally flat below the target is stuck: it will never reach quorum on its own, and nothing in the acquire pipeline can fix it. Expect a sawtooth on a healthy node: each series is a snapshot of the most recent gate evaluation, and the first evaluation of every round runs before peer validations arrive, so a sampled low reading between higher ones is normal. Judge it over minutes, and only the sustained floor of the tally against the target carries the signal.*\n\n###### Healthy range:\n*Tally at or above Target, both flat, on a node that is validating.*\n\n###### Watch for:\n*A tally pinned below the target \u2014 too few trusted validators are reachable, or the UNL / negative-UNL configuration excludes the ones that are. Also watch the target jumping to about 9.2e18 (signed 64-bit maximum): that is the explicit quorum-disabled sentinel, meaning too many publishers are unavailable and the trusted list switched quorum off entirely, so the node can never validate however far the tally climbs. It is reported as that maximum rather than wrapping negative precisely so it cannot be misread as a tally that already exceeds its target. Both series flat at 0 means the gate has never been evaluated \u2014 nothing has been offered for validation yet, which sends you back to the Bootstrap row.*\n\n###### Keywords:\n- **Quorum shortfall** *(per node)* \u2014 trusted validations for a candidate ledger falling short of the quorum needed to declare it validated, so the node holds the ledger and still cannot call it validated.\n- **Validation quorum** *(per node)* \u2014 the number of agreeing trusted validations a ledger needs before this node treats it as validated.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerQuorumPublishGauge`\n\n###### References:\n[Negative UNL and validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#quorum-shortfall)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Validations", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 186 + }, + "id": 42, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledger_quorum_publish{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"trusted_validation_tally\"}, \"series\", \"Trusted Tally\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledger_quorum_publish{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"quorum_target\"}, \"series\", \"Quorum Target\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "Trusted Validations vs Quorum Target", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Number of times the pre-accept gate refused to declare a candidate ledger validated because its trusted validations were below quorum.*\n\n###### How it's computed:\n*Count of ledger_quorum_shortfall_total by stage. One increment per rejected gate evaluation, emitted from the early return that was trace-log-only before, so a node that peers and receives validations yet never validates is no longer indistinguishable from an idle one. The gate is re-entered on every fresh trusted validation for the candidate ledger, so one ledger that eventually validates can contribute several increments on its way there.*\n\n###### Reading it:\n*A steady low count is NORMAL and is not a fault. The gate is evaluated the instant this node finishes building a ledger, before its peers' validations for that ledger have arrived, so the first evaluation of each round routinely tallies short and is retried as validations come in. Read this against the count of ledger closes and against Trusted Validations vs Quorum Target \u2014 it is the ratio and the accompanying tally that carry the signal, never the bare presence of a count.*\n\n###### Healthy range:\n*A low steady count, on the order of one per ledger close or less, on a node that is validating.*\n\n###### Watch for:\n*A count that climbs well above one per ledger close while Publish Lag grows and Time to First Validated Ledger stays at zero \u2014 that combination is the retry loop never converging, so the tally is not merely early, it never reaches the target. Confirm on Trusted Validations vs Quorum Target: a climbing tally is slow and will finish, a flat tally below the target is stuck (too few trusted validators reachable, or a UNL / negative-UNL misconfiguration). Check UNL Quorum Headroom in the Bootstrap row before anything else in this row, because a trusted list that cannot satisfy quorum makes every panel below it look starved.*\n\n###### Keywords:\n- **Quorum shortfall** *(per node)* \u2014 trusted validations for a candidate ledger falling short of the quorum needed to declare it validated, so the node holds the ledger and still cannot call it validated.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerMaster.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerMaster.cpp)\n\n###### Function:\n`LedgerMaster::checkAccept`\n\n###### References:\n[Negative UNL and validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#quorum-shortfall)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Quorum Shortfall [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 186 + }, + "id": 44, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (stage, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(ledger_quorum_shortfall_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", stage=~\"$shortfall_stage\"}[$__range])), \"series\", \"$1\", \"stage\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Pre-Accept Quorum Shortfalls (Count By Stage)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How long the node took, from process start, to pass the pre-accept quorum gate for the first time.*\n\n###### How it's computed:\n*ledger_quorum_publish series time_to_first_validated_us, converted from microseconds to seconds.*\n\n###### Reading it:\n*A one-shot measurement: it fills in the moment the node first fully validates a ledger and never changes again, so it has no trend to read. Exactly two readings matter \u2014 a duration, meaning the node got there and this is how long it took, or zero (red), meaning it never has.*\n\n###### Healthy range:\n*Seconds to a few minutes; longer on a fresh node that must acquire history first.*\n\n###### Watch for:\n*A flat zero while Time to First FULL shows a value: the node reached the full server state but has still never fully validated a ledger, which points at the quorum gate rather than at acquire. The measurement is clamped to a minimum of 1 microsecond so a genuine reading can never be confused with the never-reached zero.*\n\n###### Keywords:\n- **Time to first validated ledger** *(per node)* \u2014 elapsed time from process start until the node first declared a ledger fully validated.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerQuorumPublishGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#time-to-first-validated-ledger)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 600 + }, + { + "color": "red", + "value": 1800 + } + ] + }, + "unit": "s", + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 198 + }, + "id": 45, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledger_quorum_publish{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"time_to_first_validated_us\"} / 1e6, \"series\", \"Time to First Validated\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Time to First Validated Ledger", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How many ledgers this node has fully validated but not yet published to its clients and subscribers.*\n\n###### How it's computed:\n*ledger_quorum_publish series publish_lag: the validated ledger sequence minus the published ledger sequence, floored at zero. The published sequence was never exported before, so this gap was not derivable from any other series.*\n\n###### Reading it:\n*Publishing trails validation by design, so a small lag that drains each round is normal. A lag that stays positive, or grows, means validation is healthy and the publish pipeline is not \u2014 a different fault from anything the quorum or acquire panels can show.*\n\n###### Healthy range:\n*0 to 1 ledger.*\n\n###### Watch for:\n*A monotonic climb: the publish loop is falling behind a chain tip the node already holds, so clients and subscriptions see stale data while the node itself is current. Read it with Worker Pool Saturation and the per-job-type jobq__deferred gauges \u2014 a starved job queue is the usual cause. A flat 0 is only healthy on a node that is validating: on one that never has, the 0 means nothing has been validated to publish, so read Trusted Validations vs Quorum Target first.*\n\n###### Keywords:\n- **Publish lag** *(per node)* \u2014 validated ledgers not yet published to clients and subscribers, i.e. the gap between the validated and the published sequence.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerQuorumPublishGauge`\n\n###### References:\n[Ledger close and publication on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#publish-lag)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Ledgers", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 198 + }, + "id": 43, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledger_quorum_publish{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"publish_lag\"}, \"series\", \"Publish Lag\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Publish Lag (validated minus published)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Distribution of consensus round wall-clock duration over time, from the native round-duration histogram.*\n\n###### How it's computed:\n*Rate of each consensus_round_duration_ms bucket over 5-minute windows, drawn as a heatmap. Recorded once per round in xrpld at the single point both accept paths pass through, with explicit buckets spanning 500 ms to 120 s \u2014 the SDK default tops out at 10 s, which would collapse every slow round into one saturated bucket.*\n\n###### Reading it:\n*A tight band around 3-4 s is a healthy network. Read the band's position and width, not single cells: the width is the spread across rounds and the position is the typical round.*\n\n###### Healthy range:\n*Most rounds in the 2-5 s bands.*\n\n###### Watch for:\n*The band drifting upward, or a second band appearing high up: rounds are taking longer, which on a syncing node usually means transaction sets or validations are arriving late. Check Tx-Set Acquire Duration and Trusted Validations vs Quorum Target next. This histogram is a native metric, so unlike the span-derived panels it is never sampled \u2014 it stays complete when tracing is sampled down.*\n\n###### Keywords:\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::makeAcceptSpan`\n\n###### References:\n[Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus-round-duration)", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "unit": "ms", + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 210 + }, + "id": 52, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "mode": "scheme", + "scheme": "Turbo", + "steps": 64 + }, + "legend": { + "show": true + }, + "tooltip": { + "mode": "multi", + "showColorScale": true, + "sort": "desc" + }, + "yAxis": { + "axisLabel": "Round duration (ms)", + "unit": "ms" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (le) (rate(consensus_round_duration_ms_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))", + "format": "heatmap", + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Consensus Round Duration Distribution", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Median and 95th-percentile consensus round duration, from the same native histogram as the heatmap beside it.*\n\n###### How it's computed:\n*Quantiles over the consensus_round_duration_ms buckets. The heatmap shows the whole shape; these two lines are the trend an alert can be written against.*\n\n###### Reading it:\n*P50 is the typical round and should sit near the network's close interval. The gap between P50 and P95 is the tail: a small gap means rounds are uniform, a wide one means some rounds are much slower than the rest.*\n\n###### Healthy range:\n*P50 around 3-4 s, P95 within a couple of seconds of it.*\n\n###### Watch for:\n*P95 climbing while P50 stays flat \u2014 a minority of rounds are stalling, which is the early form of the problem the heatmap shows later as a second band. Both rising together is the whole network slowing rather than this node.*\n\n###### Keywords:\n- **Consensus round duration** *(per node)* \u2014 wall-clock time from the start of a consensus round to its accepted ledger, as this node measured it.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::makeAcceptSpan`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus-round-duration)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Round duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 210 + }, + "id": 53, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.5, sum by (le, service_instance_id, xrpl_branch, xrpl_work_item) (rate(consensus_round_duration_ms_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"P50 round duration\", \"__name__\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_work_item) (rate(consensus_round_duration_ms_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"P95 round duration\", \"__name__\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "Consensus Round Duration (p50/p95)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 222 + }, + "id": 61, + "panels": [], + "title": "Terminal blockers & serving", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Seconds until an unsupported amendment activates and this node stops validating for good. This is the LEADING indicator \u2014 the countdown before the block, while there is still time to upgrade.*\n\n###### How it's computed:\n*amendment_block series seconds_to_block. A value of -1 is the sentinel for \"no unsupported amendment is pending\", the same convention validator_health{metric=\"unl_expiry_days\"} already uses.*\n\n###### Reading it:\n*Because -1 is the healthy sentinel and a small positive number is the emergency, the colour scale is not monotonic \u2014 read the value, not only the colour. -1 is green and means nothing is pending. A large positive value is green above 7 days and yellow under 7 days: an unsupported amendment holds majority but there is still time to upgrade. A small positive value under 1 day is red: at 0 this node stops validating and does not resume without a software upgrade.*\n\n###### Healthy range:\n*-1.*\n\n###### Watch for:\n*Any value at or above 0. This is distinct from the Amendment Blocked stat on the Validator Health dashboard (validator_health{metric=\"amendment_blocked\"}), which reports the TERMINAL state \u2014 already blocked, too late to act. This panel is the window before that happens, so the two are read together: countdown first, terminal state as confirmation. The identity of the blocking amendment is deliberately NOT a metric label, because the network can vote on an arbitrary 256-bit amendment id and a label would be unbounded cardinality. The hash is logged instead by AmendmentTableImpl::doValidatedLedger (\"Unsupported amendment reached majority at ...\") and is available via Loki.*\n\n###### Keywords:\n- **Amendment block countdown** *(per node)* \u2014 seconds until an unsupported amendment that has reached majority activates and blocks this node; -1 when none is pending.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerAmendmentBlockGauge`\n\n###### References:\n[Amendments on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/amendments) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#amendment-blocked)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": 86400 + }, + { + "color": "green", + "value": 604800 + } + ] + }, + "unit": "s", + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "None Pending", + "color": "green", + "index": 0 + } + } + } + ] + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 223 + }, + "id": 36, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(amendment_block{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"seconds_to_block\"}, \"series\", \"Seconds to Block\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Amendment Block Countdown", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Whether an amendment this build does not support has reached majority on the network.*\n\n###### How it's computed:\n*amendment_block series warned: 1 while an unsupported amendment holds majority, 0 otherwise.*\n\n###### Reading it:\n*0 is healthy. A 1 is the first warning that an upgrade is required, and it is raised before the amendment activates rather than after.*\n\n###### Healthy range:\n*0.*\n\n###### Watch for:\n*The transition from 0 to 1 \u2014 that is the moment the upgrade clock starts. Read Amendment Block Countdown next for how long is left, and the Amendment Blocked stat on the Validator Health dashboard for whether the block has already happened; that one is the terminal state, this one is the warning. Which amendment is blocking is not a label (an arbitrary 256-bit amendment id would be unbounded cardinality); the hash is logged by AmendmentTableImpl::doValidatedLedger and is available via Loki.*\n\n###### Keywords:\n- **Amendment warning** *(per node)* \u2014 an unsupported amendment has reached majority; the node still validates, but will stop when that amendment activates.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerAmendmentBlockGauge`\n\n###### References:\n[Amendments on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/amendments) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#amendment-blocked)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Warned", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 223 + }, + "id": 37, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(amendment_block{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"warned\"}, \"series\", \"Warned\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Amendment Warned", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Peer requests that this node declined to serve, split by why it declined and by what was asked for.*\n\n###### How it's computed:\n*Count over the selected range of refused serve attempts grouped by reason and request type (ledger, tx-set, object or fetchpack), per node. Filter with the Serve Reason and Serve Request variables, which list the emitted label values verbatim (the tx-set one is a single unhyphenated word).*\n\n###### Reading it:\n*This is the other side of the sync exchange: every other panel here measures what this node fetches, this one measures what it refuses to give back. sendq_full and load_shed are self-inflicted backpressure \u2014 the node holds the data but will not send it. not_found and no_map mean the requester asked for history this node does not hold.*\n\n###### Healthy range:\n*Near zero.*\n\n###### Watch for:\n*sendq_full or load_shed climbing. This node is starving its peers, and on a network of similarly loaded nodes that is exactly the condition that makes everybody's sync slow, so a refusal here can be the cause of another operator's stall. A high not_found is usually benign on a node configured with short history, but it tells peers to look elsewhere.*\n\n###### Keywords:\n- **Serve refusal** *(per node)* \u2014 a peer request for ledger, transaction-set, object or fetch-pack data that this node declined, together with the reason it declined.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::processLedgerRequest` \u00b7 `PeerImp::onMessage(TMGetObjectByHash)` \u00b7 `PeerImp::doFetchPack`\n\n###### References:\n[Fetch-pack](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#fetch-pack) \u00b7 [GetObject / object fetch](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#getobject-object-fetch)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Serve Refused [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 235 + }, + "id": 34, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_join(sum by (reason, request, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(serve_refused_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", reason=~\"$serve_reason\", request=~\"$serve_request\"}[$__range])), \"series\", \" \", \"reason\", \"request\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Ledger/Object Serve Refusals (Count By Reason & Request)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Number of times this node abandoned the chain tip it had built and switched to a different last-closed ledger reported by the network.*\n\n###### How it's computed:\n*Count over the selected range of last-closed-ledger switches where the ledger the network reported was not the one this node built on, per node. The counter carries no labels, so the sum is over one series per node.*\n\n###### Reading it:\n*Any non-zero value means this node was told the network's last-closed ledger is not the one it built on, and it discarded its own chain tip in response. A single jump around startup or a restart is ordinary recovery.*\n\n###### Healthy range:\n*0.*\n\n###### Watch for:\n*Repeated jumps \u2014 that is wrong-chain thrash, not one-off recovery. Check the peer set and the configured network id: a node peered to the wrong network, or into a minority partition, keeps being overruled and keeps throwing away work. Pair with the Bootstrap row and with Ledgers Behind Network.*\n\n###### Keywords:\n- **Byzantine ledger jump** *(per node)* \u2014 the node replaced its own last-closed ledger with a different one reported by the network, discarding the chain tip it had built.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::switchLastClosedLedger`\n\n###### References:\n[Fork](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#fork) \u00b7 [Ledger history mismatch](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-history-mismatch)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 235 + }, + "id": 35, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(ledger_jump_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range])), \"series\", \"Ledger Jumps\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Byzantine Ledger Jumps (Count)", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 247 + }, + "id": 62, + "panels": [], + "title": "Back-fill & persistence", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Node-store write latency next to read latency, in microseconds per operation. The write side is the signal: a node with a large existing database back-fills slower than a fresh one, and back-fill is write-bound, so no read-side metric can show it.*\n\n###### How it's computed:\n*nodestore_state series node_writes_duration_us and node_reads_duration_us, each divided by its own count series (node_writes, node_reads_total) so the reading is the latency during the selected interval rather than the average since boot. All three concrete store paths time themselves, so the write side is live on an ordinary node.*\n\n###### Reading it:\n*Compare the two lines. Reads far above writes points at the read path or a cold cache; writes far above reads points at backend write pressure, which is the large-existing-database case.*\n\n###### Healthy range:\n*Both well under a few hundred microseconds on healthy local storage.*\n\n###### Watch for:\n*A rising write line during history back-fill: the backend cannot absorb writes fast enough and sync will stay slow no matter how many peers are available. Read with Peers Able to Serve Needed Sequence to tell a data-supply problem from a disk problem. This is a mean, not a percentile \u2014 a tail that matters will move it, but p99 is not available from this signal.*\n\n###### Keywords:\n- **Node-store write latency** *(per node)* \u2014 how long the node store takes to persist one object.\n- **Node-store read latency** *(per node)* \u2014 how long the node store takes to retrieve one object.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#node-store-write-latency)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Microseconds / Op", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "\u00b5s" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 248 + }, + "id": 38, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_writes_duration_us\"}[$__rate_interval])) / (sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_writes\"}[$__rate_interval])) > 0), \"series\", \"Write us/op (interval)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_duration_us\"}[$__rate_interval])) / (sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])) > 0), \"series\", \"Read us/op (interval)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "NodeStore Write vs Read Latency (us/op)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Node-store write and read operation rates \u2014 the denominators behind the latency panel.*\n\n###### How it's computed:\n*Rate of the nodestore_state node_writes and node_reads_total series.*\n\n###### Reading it:\n*Writes climb while a node is back-filling history and fall to near the ledger-close rate once it is caught up.*\n\n###### Healthy range:\n*Non-zero writes whenever the node is ingesting ledgers.*\n\n###### Watch for:\n*Write rate at zero while the node is still behind the network: nothing is being persisted, so the stall is upstream of the node store \u2014 check peer supply and the acquire panels rather than storage. A flat latency with a collapsing operation rate also means the latency figure above has gone stale rather than good.*\n\n###### Keywords:\n- **Node-store operation rate** *(per node)* \u2014 stores and fetches per second against the node store.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#node-store-operation-rate)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Ops / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 248 + }, + "id": 39, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_writes\"}[$__rate_interval])), \"series\", \"Writes/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])), \"series\", \"Reads/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "NodeStore Operation Rate (writes vs reads)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Number of times ledger-replay sub-tasks give up and fall back to acquiring a whole ledger, split by which sub-task gave up.*\n\n###### How it's computed:\n*Count of ledger_replay_fallback_total by stage. The skip-list stage and the delta stage each emit once, on the transition into fallback, from the branch that was debug-log-only before.*\n\n###### Reading it:\n*Zero when replay-capable peers are available. Any sustained count means the replay optimisation is being defeated and back-fill has reverted to the slower full-acquire path.*\n\n###### Healthy range:\n*Zero, or brief spikes while the peer set changes.*\n\n###### Watch for:\n*A persistent count on either stage: too few connected peers support the ledger-replay feature, so every historical ledger is fetched in full instead of as a delta. Read with Replay Outcomes.*\n\n###### Keywords:\n- **Replay fallback** *(per node)* \u2014 a replay sub-task abandoning the delta shortcut and acquiring the entire ledger instead.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[SkipListAcquire.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/SkipListAcquire.cpp)\n\n###### Function:\n`SkipListAcquire::trigger` / `LedgerDeltaAcquire::trigger`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#replay-fallback)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Replay Fallback [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 260 + }, + "id": 40, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (stage, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(ledger_replay_fallback_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", stage=~\"$replay_stage\"}[$__range])), \"series\", \"$1\", \"stage\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Replay Fallback To Full Acquire (Count By Stage)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Count of ledger-replay tasks reaching each terminal state: success, timeout, build failure or parameter failure.*\n\n###### How it's computed:\n*Count of ledger_replay_outcome_total by outcome. Every terminal path in the replay task emits exactly once; before this, all four only set an internal flag and wrote a log line.*\n\n###### Reading it:\n*Successes only is healthy. Timeouts mean deltas never arrived; build failures mean a delta would not apply to its parent; parameter failures mean a peer served an inconsistent skip list.*\n\n###### Healthy range:\n*Successes non-zero while replaying, all failure outcomes at zero.*\n\n###### Watch for:\n*Any failure outcome climbing while successes stay flat: replay is running but never completing, so history back-fill is silently falling back to the slower path. The outcome value tells you which layer to look at \u2014 timeouts point at peers, build and parameter failures point at the data those peers served.*\n\n###### Keywords:\n- **Ledger replay** *(per node)* \u2014 rebuilding a range of historical ledgers from a start ledger plus per-ledger deltas instead of downloading each one whole.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerReplayTask.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerReplayTask.cpp)\n\n###### Function:\n`LedgerReplayTask::recordOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-replay)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Replay Outcome [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 260 + }, + "id": 41, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(ledger_replay_outcome_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$replay_outcome\"}[$__range])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Replay Outcomes (Count By Terminal State)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How long the heap trim that ends every cache sweep takes. The trim returns free heap pages to the kernel, and its cost scales with the resident heap \u2014 so this is the panel that shows a node with a large existing database paying a per-sweep penalty a fresh node never pays.*\n\n###### How it's computed:\n*p50 and p95 of the sweep_malloc_trim_us histogram. Recorded once per sweep, so the sample rate is one per sweep interval (10 s on a tiny node through 120 s on a huge one). The measurement used to run only at debug log level, so on an ordinary node there was no value at all.*\n\n###### Reading it:\n*Read the two lines together against the sweep interval. Sub-millisecond is free. Tens of milliseconds against a 10 s interval is still a small duty cycle but means the trim is walking a large heap, and it runs on the sweep job \u2014 so the cost lands on the job queue, not in the background.*\n\n###### Healthy range:\n*Under a millisecond on a warm node with a modest heap.*\n\n###### Watch for:\n*p95 climbing as the database grows, especially alongside a rising sweep-job queue wait. IMPORTANT LIMITATION: the fault counter beside this panel covers the trim call only. It proves the trim itself faults; it does NOT prove the trim causes the later faults taken as the caches refill. Correlate this duration against sweep-job queueing rather than concluding the trim caused a slow sync.*\n\n###### Keywords:\n- **Heap trim** *(per node)* \u2014 returning free heap pages from the allocator's arenas back to the kernel.\n- **Sweep interval** *(per node)* \u2014 how often the periodic cache sweep runs, set by node size.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Application.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/Application.cpp)\n\n###### Function:\n`ApplicationImp::trimHeapAndRecord`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#heap-trim)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Trim Duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "\u00b5s" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 272 + }, + "id": 64, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.5, sum by (le, service_instance_id, xrpl_branch, xrpl_work_item) (rate(sweep_malloc_trim_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"P50 Trim\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_work_item) (rate(sweep_malloc_trim_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"P95 Trim\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "Sweep Heap-Trim Duration (p50/p95)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Two rates side by side: minor page faults taken inside the heap trim, and the resident memory the trim actually returned to the kernel. Together they say whether the trim is buying anything for what it costs.*\n\n###### How it's computed:\n*Count of sweep_malloc_trim_minor_faults_total and of sweep_malloc_trim_reclaimed_kb_total. Both are cumulative counters exported per sweep, so the panel rates them rather than plotting the totals. Reclaim is published only when resident memory actually fell; a sweep during which another thread grew the heap contributes nothing rather than a negative amount.*\n\n###### Reading it:\n*Reclaimed volume with a near-zero fault count is a cheap, useful trim. A fault count that moves with the reclaimed volume means the pages are being handed back and immediately taken again, which is churn rather than savings.*\n\n###### Healthy range:\n*A reclaimed volume that tracks cache turnover, with faults near zero.*\n\n###### Watch for:\n*Reclaim near zero while the trim duration panel shows real time being spent: the trim is walking the heap and freeing nothing, which is pure cost. IMPORTANT LIMITATION: the fault delta is scoped to the trim call, so it cannot show the re-fault cost paid later as the caches refill and touch the returned pages. That later cost is real but is NOT measured here; do not read a low fault count as proof the trim was free.*\n\n###### Keywords:\n- **Minor page fault** *(per node)* \u2014 a memory access satisfied without disk I/O, by mapping a page the kernel already holds.\n- **Reclaimed resident memory** *(per node)* \u2014 resident kilobytes the allocator handed back to the kernel.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Application.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/Application.cpp)\n\n###### Function:\n`ApplicationImp::trimHeapAndRecord`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#minor-page-fault)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 272 + }, + "id": 65, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(sweep_malloc_trim_minor_faults_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range])), \"series\", \"Minor Faults\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(sweep_malloc_trim_reclaimed_kb_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range])), \"series\", \"Reclaimed KB\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B", + "instant": true + } + ], + "title": "Sweep Heap-Trim Faults & Reclaim (Count)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*When an online-delete rotation is running, and the rate of the extra writes it forces. A rotation rewrites into the new backend any node body the doomed archive serves, which is I/O an ordinary fetch would never perform \u2014 and it exists only on a populated, already-rotated database, which is why it never appears on a fresh node.*\n\n###### How it's computed:\n*rotation_state with metric=in_flight plotted raw (it is a 0/1 state flag), and metric=copy_forward \u2014 a cumulative write total \u2014 plotted as a rate. Both are read from the node store on each collection tick. The copy-forward count existed before as a log-only per-rotation tally that reset on every swap; the total behind this panel never resets, so it can be rated.*\n\n###### Reading it:\n*The two must move together: copy-forward writes should only appear while the window flag is 1. Read the write rate against the node-store write latency panel above \u2014 that is what tells extra rotation writes from a slow backend.*\n\n###### Healthy range:\n*Flag at 0 most of the time, rising to 1 briefly once per delete interval, with the write rate non-zero only inside those windows.*\n\n###### Watch for:\n*A copy-forward rate that is large enough to move node-store write latency: rotation is competing with sync I/O, which is the whole hypothesis this panel tests. Copy-forward writes while the flag reads 0 would mean the window flag leaked, not that rotation is cheap. NO SERIES AT ALL on either query means online_delete is not configured on this node, which is different from a rotation that costs nothing.*\n\n###### Keywords:\n- **Rotation window** *(per node)* \u2014 the interval during which an online-delete backend swap is in progress.\n- **Copy-forward write** *(per node)* \u2014 rewriting a node body from the backend about to be deleted into the one replacing it.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerRotationStateGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rotation-window)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "State & Writes / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*Rotation In Flight.*" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "left" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + }, + { + "id": "custom.axisLabel", + "value": "In Flight (0/1)" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Copy-Forward.*" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "cps" + }, + { + "id": "custom.axisLabel", + "value": "Writes / Sec" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 284 + }, + "id": 66, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rotation_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"in_flight\"}), \"series\", \"Rotation In Flight (0/1)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(rotation_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"copy_forward\"}[$__rate_interval])), \"series\", \"Copy-Forward Writes / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "Online-Delete Rotation Window & Copy-Forward Writes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Number of times a rotation has to rescue a tree node that was missing from BOTH backends and rewrite it from memory. Each one is an extra write on top of the whole-state-map walk the rotation already performs.*\n\n###### How it's computed:\n*Count of rotation_copy_node_restore_total. Incremented once per rescued node in the rotation's state-map walk, from a branch that was warn-log-only before, so the volume was invisible unless someone was reading logs.*\n\n###### Reading it:\n*Zero on a healthy node. Any sustained count means clean nodes reachable from the validated state map have no on-disk copy left, because the backend holding them was removed by an earlier rotation and they were never rewritten.*\n\n###### Healthy range:\n*Flat at zero.*\n\n###### Watch for:\n*A non-zero count is the signal that earlier rotations dropped data the current state map still needs: each rescue is a write that competes with sync I/O, and without the rescue the node would later surface as an unresolvable missing-node error. Read with the copy-forward panel \u2014 both are rotation-time writes, but this one also indicates prior data loss rather than merely cost. The node hash is deliberately not a label (unbounded cardinality); get it from the copyNode warning in Loki, correlated by node and time.*\n\n###### Keywords:\n- **Node re-store** *(per node)* \u2014 rewriting an in-memory tree node whose only on-disk copy was removed by an earlier rotation.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[SHAMapStoreImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/SHAMapStoreImp.cpp)\n\n###### Function:\n`SHAMapStoreImp::copyNode`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#node-re-store)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 284 + }, + "id": 67, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(rotation_copy_node_restore_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range])), \"series\", \"Nodes Re-Stored\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Rotation Node Re-Stores (Count)", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 296 + }, + "id": 63, + "panels": [], + "title": "Spans & traces", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*95th-percentile duration of each of the three phases of one ledger acquisition: the header wait, the account-state tree fetch, and the transaction tree fetch.*\n\n###### How it's computed:\n*95th percentile of the span-derived duration histogram for the three ledger.acquire.header / .astree / .txtree child spans, one series per phase. The parent ledger.acquire span is flat and cannot separate them, which matters because the account-state tree is nearly all of the work in a real fresh sync.*\n\n###### Reading it:\n*The astree series dominating is expected and healthy. The value of the split is the comparison: a header series that is large means the node is waiting to be told what to fetch, which is a peer-supply problem upstream of either tree.*\n\n###### Healthy range:\n*The astree series largest, the txtree series small, the header series near zero.*\n\n###### Watch for:\n*A hot astree band that keeps growing means account-state nodes are not being served \u2014 check Ledger Acquire Phase Outcomes beside it for timed_out, and Outbound Dial Outcomes (Count By Outcome) in the Bootstrap row for refused or timed-out dials. A large header series instead means no peer is answering the header request at all.*\n\n###### Keywords:\n- **Acquire phase** *(per node)* \u2014 one of the three sequential fetches a ledger acquisition is made of; the header gates both trees, because it is what names their root hashes.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::syncPhaseSpans`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#acquire-phase)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 297 + }, + "id": 48, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, span_name, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_duration_milliseconds_bucket{outcome=\"complete\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.acquire.header\"}[$__rate_interval]))), \"series\", \"P95 header\", \"span_name\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, span_name, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_duration_milliseconds_bucket{outcome=\"complete\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.acquire.astree\"}[$__rate_interval]))), \"series\", \"P95 account-state tree\", \"span_name\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, span_name, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_duration_milliseconds_bucket{outcome=\"complete\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.acquire.txtree\"}[$__rate_interval]))), \"series\", \"P95 transaction tree\", \"span_name\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + } + ], + "title": "Ledger Acquire Phase Duration (p95 by phase)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How many ledger-acquire phases finished each way over the selected range, as whole counts.*\n\n###### How it's computed:\n*`increase(span_calls_total[$__range])` per `span_name` and `outcome`, so the number shown is the count of phases that ended that way inside the dashboard time range.*\n\n###### Reading it:\n*Counts, not rates. These are discrete, low-frequency events: a count of 0.14 is 278 phases over 33 minutes, which no reader can infer from the rate. Compare the Complete and Abandoned bars for the same phase. Header needs one round trip; both tree phases need many.*\n\n###### Healthy range:\n*Abandoned near zero for every phase, and Complete rising with ledger progress.*\n\n###### Watch for:\n*Abandoned dominating the tree phases while Header still completes. That means requests reach peers and headers arrive, but the many-round-trip phases never finish. `Abandoned` here means the acquire object was destroyed while still fetching, not that it timed out \u2014 see Acquire Stalls \u2014 No Progress (Count) and the runbook for the sweep mechanism.*\n\n###### Keywords:\n- **Acquire phase** *(per node)* \u2014 one of the three sequential fetches of a ledger: header, account-state tree, transaction tree.\n- **Abandoned** *(per node)* \u2014 the acquire was destroyed with no result, i.e. dropped while still fetching rather than failing or timing out.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Spans are emitted by xrpld code; the collector's spanmetrics connector turns them into `span_calls_total`; the Grafana query counts the increase over the range.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::endPhaseSpan`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#fresh-node-sync-diagnostics)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Acquire Phase [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 297 + }, + "id": 49, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_join(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(sum by (span_name, outcome, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$span_outcome\", span_name=~\"ledger\\\\.acquire\\\\..*\"}[$__range]))), \"phase\", \"Header\", \"span_name\", \"ledger\\\\.acquire\\\\.header\"), \"phase\", \"Account State Tree\", \"span_name\", \"ledger\\\\.acquire\\\\.astree\"), \"phase\", \"Transaction Tree\", \"span_name\", \"ledger\\\\.acquire\\\\.txtree\"), \"outcome_t\", \"Complete\", \"outcome\", \"complete\"), \"outcome_t\", \"Abandoned\", \"outcome\", \"abandoned\"), \"outcome_t\", \"Timeout\", \"outcome\", \"timeout\"), \"outcome_t\", \"Failed\", \"outcome\", \"failed\"), \"series\", \" \", \"phase\", \"outcome_t\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Ledger Acquire Phase Outcomes (Count By Phase & Outcome)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate at which attempts to fetch a transaction set referenced by a consensus proposal reach each terminal state.*\n\n###### How it's computed:\n*Rate of span-derived call counts for the txset.acquire span, split by outcome. The span is new: transaction-set acquisition had no telemetry at all before, so a consensus round stalled waiting on a set looked identical to an idle one. Every exit stamps an outcome, including the sweep that drops a set which never arrived.*\n\n###### Reading it:\n*Only complete is healthy. Timeout means peers never supplied the set. Abandoned means the round moved on and the set was dropped mid-fetch. Failed means a peer served data that would not build.*\n\n###### Healthy range:\n*A low rate of complete during consensus, everything else at zero. Zero everywhere is also normal on a node that already holds every proposed set locally.*\n\n###### Watch for:\n*Timeout or abandoned climbing: proposed sets are not arriving, so rounds are waiting on data rather than on agreement. Read with the Tx-Set Acquire Duration panel beside it \u2014 a rising p95 with outcomes still complete is slow-but-working, while the same p95 with timeouts is the set never arriving at all.*\n\n###### Keywords:\n- **Tx-set acquire** *(per node)* \u2014 one attempt to fetch the transaction set a consensus proposal referenced but this node did not hold.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[TransactionAcquire.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/TransactionAcquire.cpp)\n\n###### Function:\n`TransactionAcquire::finalizeAcquireSpan`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#tx-set-acquire)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Acquisitions / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 309 + }, + "id": 46, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$span_outcome\", span_name=\"txset.acquire\"}[$__rate_interval])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Tx-Set Acquire Outcomes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*95th-percentile wall time of one transaction-set acquisition, split by how it ended.*\n\n###### How it's computed:\n*95th percentile of the span-derived duration histogram for the txset.acquire span, grouped by outcome.*\n\n###### Reading it:\n*Splitting by outcome is what makes this readable: the complete series is how long a successful fetch takes, while the timeout series is pinned near the retry budget by construction and carries no information about speed.*\n\n###### Healthy range:\n*A complete-series p95 well under a consensus round interval.*\n\n###### Watch for:\n*A complete-series p95 approaching the round interval: sets are arriving, but so late that they delay the round they belong to. This is the case that a pure outcome rate cannot show, because those acquisitions do succeed.*\n\n###### Keywords:\n- **Tx-set acquire** *(per node)* \u2014 one attempt to fetch the transaction set a consensus proposal referenced but this node did not hold.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[TransactionAcquire.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/TransactionAcquire.cpp)\n\n###### Function:\n`TransactionAcquire::init`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#tx-set-acquire)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 309 + }, + "id": 47, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, outcome, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$span_outcome\", span_name=\"txset.acquire\"}[$__rate_interval]))), \"series\", \"P95 $1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Tx-Set Acquire Duration (p95)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate of outbound peer dials reaching each terminal outcome, derived from the per-attempt peer.dial span.*\n\n###### How it's computed:\n*Rate of span-derived call counts for the peer.dial span, split by outcome. The same six outcome values the overlay_connect_total counter carries, set from the same funnel in the dial state machine so the two cannot disagree.*\n\n###### Reading it:\n*Read alongside Outbound Dial Outcomes (Count By Outcome) in the Bootstrap row, which is the native counter for the same events. This panel exists for what the counter cannot do: each point here is backed by traces, so clicking through gives the individual attempt and the peer address it was dialling, which is never a metric label because one series per peer address would be unbounded.*\n\n###### Healthy range:\n*The connected series non-zero, failure series at or near zero.*\n\n###### Watch for:\n*A failure series dominating while connected stays at zero means the node has no outbound peers and cannot sync at all. Use the trace drill-down to find which endpoint keeps failing \u2014 the aggregate rate cannot tell one bad peer from a broken local network.*\n\n###### Keywords:\n- **Outbound dial** *(per node)* \u2014 one attempt by this node to open a peer connection, spanning the TCP connect, the TLS handshake and the protocol upgrade.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[ConnectAttempt.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/ConnectAttempt.cpp)\n\n###### Function:\n`ConnectAttempt::reportOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Dials / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 321 + }, + "id": 50, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$span_outcome\", span_name=\"peer.dial\"}[$__rate_interval])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Outbound Dial Outcomes (span-derived, per attempt)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate at which this node answers peers' ledger-data requests, split by what was asked for and how the reply ended.*\n\n###### How it's computed:\n*Rate of span-derived call counts for the ledger.serve span, split by object_type (header, transaction tree, account-state tree, or a proposed transaction set) and by outcome. The outcome is derived from the reply itself, so it is refused whenever nothing was sent.*\n\n###### Reading it:\n*This is the supply side \u2014 what this node does for its peers, not what it receives. The account-state series is the one that matters to a syncing peer, since that tree is the bulk of a fresh sync.*\n\n###### Healthy range:\n*Non-zero complete series on the types peers ask for, refused near zero.*\n\n###### Watch for:\n*A refused series climbing means this node is declining to serve; the paired serve_refused_total counter on Ledger/Object Serve Refusals (Count By Reason & Request) gives the specific cause. A partial series means replies keep hitting the size cap, so peers must make repeated round trips for one tree. Note this panel does not explain this node's own sync \u2014 it explains its peers'.*\n\n###### Keywords:\n- **Ledger serve** *(per node)* \u2014 this node answering a peer's request for ledger data, as opposed to requesting data for itself.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::processLedgerRequest`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-serve)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Requests / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 321 + }, + "id": 51, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_join(sum by (object_type, outcome, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", object_type=~\"$object_type\", outcome=~\"$span_outcome\", span_name=\"ledger.serve\"}[$__rate_interval])), \"series\", \" \", \"object_type\", \"outcome\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Ledger Serve Rate by Object Type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate of trusted validations reaching the ledger-acceptance gate, split by what the validation store did with each one and by whether it actually reached the gate.*\n\n###### How it's computed:\n*Rate of span-derived calls for the consensus.validation.accept span, by validation_status and accept_gated. The span is emitted once per trusted validation and carries the trace id of the ledger it validates, so any point here can be opened as the full trace for that ledger \u2014 the validation, the acceptance decision it drove, and the acquire and store spans for the same ledger.*\n\n###### Reading it:\n*Nearly all of the rate should be validation_status=current, which is the only status that continues to the gate. accept_gated=true means another thread was already accepting that ledger, so no acceptance followed this validation; a modest share is normal when validations for one ledger arrive together.*\n\n###### Healthy range:\n*Dominated by current, at roughly the trusted-validator count per ledger close.*\n\n###### Watch for:\n*Rate concentrated in stale, bad_seq, multiple or conflicting: validations are arriving and being counted for nothing, which is the difference between a node that is slow to validate and one that never will \u2014 from the outside the two look identical. Confirm on Trusted Validations vs Quorum Target and Pre-Accept Quorum Shortfalls (Count By Stage), and check UNL Quorum Headroom in the Bootstrap row first, since a trusted list that cannot satisfy quorum starves everything below it. A flat zero on a peered node means no trusted validations are arriving at all.*\n\n###### Keywords:\n- **Validation status** *(per node)* \u2014 what this node's validation store did with an arriving validation; only current counts toward accepting a ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[RCLValidations.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLValidations.cpp)\n\n###### Function:\n`handleNewValidation`\n\n###### References:\n[Negative UNL and validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-status)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Validations / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 333 + }, + "id": 54, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_join(sum by (validation_status, accept_gated, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.validation.accept\", validation_status=~\"$validation_status\"}[$__rate_interval])), \"series\", \" Gated=\", \"validation_status\", \"accept_gated\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Trusted Validation Accept Rate by Status", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": ["ledger", "sync", "diagnostics"], + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "query": "prometheus", + "regex": "", + "current": {}, + "hide": 0, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [] + }, + { + "name": "service_name", + "label": "Service Name", + "description": "Filter by service.name (e.g. xrpld, xrpld-validator)", + "type": "query", + "query": "label_values(service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "xrpld", + "value": "xrpld" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "deployment_environment", + "label": "Environment", + "description": "Filter by deployment tier [local / ci / test / prod]", + "type": "query", + "query": "label_values(deployment_environment)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_network_type", + "label": "Network", + "description": "Filter by XRPL network [mainnet / testnet / devnet / perf / unknown]", + "type": "query", + "query": "label_values(xrpl_network_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_work_item", + "label": "Work Item", + "description": "Filter by perf-iac work item / ticket (e.g. RIPD-7455)", + "type": "query", + "query": "label_values(xrpl_work_item)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_branch", + "label": "Branch", + "description": "Filter by comparison side (baseline:: / test::)", + "type": "query", + "query": "label_values(xrpl_branch)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "xrpl_node_role", + "label": "Node Role", + "description": "Filter by node role (validator / peer)", + "type": "query", + "query": "label_values(xrpl_node_role)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "node", + "label": "Node", + "description": "Filter by xrpld node (service.instance.id)", + "type": "query", + "query": "label_values(target_info, service_instance_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "dial_outcome", + "label": "Dial Outcome", + "description": "Filter outbound dial attempts by terminal outcome [connected / tcp_fail / tls_fail / self_connection / upgrade_fail / timeout]", + "type": "query", + "query": "label_values(overlay_connect_total, outcome)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "handshake_reason", + "label": "Handshake Reason", + "description": "Filter handshake negotiation failures by the check that rejected the peer (e.g. wrong_network, clock_skew)", + "type": "query", + "query": "label_values(handshake_negotiation_fail_total, reason)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "unl_site", + "label": "UNL Site", + "description": "Filter validator-list fetches by configured UNL site URI", + "type": "query", + "query": "label_values(unl_fetch_total, site)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "unl_outcome", + "label": "UNL Fetch Outcome", + "description": "Filter validator-list fetches by outcome (accepted is the only success)", + "type": "query", + "query": "label_values(unl_fetch_total, outcome)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "mode_from", + "label": "Mode From", + "description": "Filter mode transitions by the state being left [disconnected / connected / syncing / tracking / full]", + "type": "query", + "query": "label_values(state_changes_total, from)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "mode_to", + "label": "Mode To", + "description": "Filter mode transitions by the state being entered [disconnected / connected / syncing / tracking / full]", + "type": "query", + "query": "label_values(state_changes_total, to)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "acquire_metric", + "label": "Acquire Metric", + "description": "Filter the acquire-progress gauge sub-series [missing_state_nodes_max / missing_tx_nodes_max / received_data_depth / in_flight]", + "type": "query", + "query": "label_values(sync_acquire, metric)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "addnode_outcome", + "label": "Add-Node Outcome", + "description": "Filter received SHAMap nodes by outcome [good / duplicate / invalid]", + "type": "query", + "query": "label_values(sync_addnode_total, outcome)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "acquire_source", + "label": "Acquire Source", + "description": "Filter ledger acquires by where the data came from [local / network]", + "type": "query", + "query": "label_values(sync_acquire_source_total, source)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "saturation_metric", + "label": "Saturation Metric", + "description": "Filter the worker-pool gauge sub-series [running_tasks / worker_threads / total_waiting]", + "type": "query", + "query": "label_values(jobq_saturation, metric)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "accept_outcome", + "label": "Accept Outcome", + "description": "Filter inbound peer handoffs by terminal outcome [accepted / no_slot / resource_limit / ...]", + "type": "query", + "query": "label_values(peer_accept_total, outcome)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "disconnect_reason", + "label": "Disconnect Reason", + "description": "Filter peer disconnects by cause [graceful / large_sendq / not_useful / ping_timeout / ...]", + "type": "query", + "query": "label_values(peer_disconnect_total, reason)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "disconnect_direction", + "label": "Disconnect Direction", + "description": "Filter peer disconnects by which side opened the connection [inbound / outbound]", + "type": "query", + "query": "label_values(peer_disconnect_total, direction)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "serve_reason", + "label": "Serve Reason", + "description": "Filter serve refusals by cause [sendq_full / load_shed / not_found / no_map / ...]", + "type": "query", + "query": "label_values(serve_refused_total, reason)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "serve_request", + "label": "Serve Request", + "description": "Filter serve refusals by what the peer asked for [ledger / tx-set / object / fetchpack]", + "type": "query", + "query": "label_values(serve_refused_total, request)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "replay_stage", + "label": "Replay Stage", + "description": "Filter replay fallbacks by sub-task stage [skiplist / delta]", + "type": "query", + "query": "label_values(ledger_replay_fallback_total, stage)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "replay_outcome", + "label": "Replay Outcome", + "description": "Filter replay tasks by terminal outcome [success / timeout / build_failed / parameter_failed]", + "type": "query", + "query": "label_values(ledger_replay_outcome_total, outcome)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "shortfall_stage", + "label": "Shortfall Stage", + "description": "Filter quorum-shortfall rejections by gate stage [pre_accept]", + "type": "query", + "query": "label_values(ledger_quorum_shortfall_total, stage)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "span_outcome", + "label": "Span Outcome", + "description": "Filter the span-derived sync panels by terminal outcome [complete / failed / timeout / abandoned / partial / refused / connected / tcp_fail / tls_fail / self_connection / upgrade_fail]", + "type": "query", + "query": "label_values(span_calls_total, outcome)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "object_type", + "label": "Served Object Type", + "description": "Filter served ledger-data requests by what was asked for [header / tx / as / txset]", + "type": "query", + "query": "label_values(span_calls_total{span_name=\"ledger.serve\"}, object_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "timed_out", + "label": "Phase Timed Out", + "description": "Filter ledger-acquire phases by whether the retry budget expired [true / false]", + "type": "query", + "query": "label_values(span_calls_total{span_name=~\"ledger.acquire..*\"}, timed_out)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + }, + { + "name": "validation_status", + "label": "Validation Status", + "description": "Filter the validation-accept panel by what the validation store did with the arriving validation [current / stale / bad_seq / multiple / conflicting / unknown]", + "type": "query", + "query": "label_values(span_calls_total, validation_status)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "refresh": 2, + "sort": 1 + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "title": "Ledger Sync Health", + "uid": "ledger-sync-health", + "refresh": "30s" +} diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index 206fad7228b..01b7829551e 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -5107,7 +5107,7 @@ }, { "title": "Job Queue Saturation (Running vs Limit)", - "description": "###### What this is:\n*How close each concurrency-capped job type is to its ceiling. JobQueue enforces a per-type limit on how many jobs of that type may run at once, and the tight ones carry ledger-sync traffic: makeFetchPack 1, ledgerRequest 3, ledgerData 3, updatePaths 1, fetchTxnData 5. A type at its ceiling cannot start more work no matter how many workers are idle, so this is a different kind of limit from CPU or disk.*\n\n###### How it's computed:\n*Each jobq__running gauge divided by that type's own limit from JobTypes.h, so every line shares one 0-to-1 axis. 1.0 means running equals the limit. Multiply a reading by the limit shown in its legend to recover the raw job count. JobQueue::collect snapshots all three per-type counters under the queue's own lock and publishes them after releasing it, on the 1-second export cycle.*\n\n###### Reading it:\n*Read the distance to 1.0, not the absolute height. Below 1.0 the type has spare slots and its queue wait is not the limit's fault. Touching 1.0 briefly is normal work. Sitting at 1.0 means the type is pinned at its ceiling and every further job of that type is being deferred rather than started, which is what turns into queue wait downstream. Because the limits differ, a raw count of 3 is saturation for ledgerRequest but only 60 percent for fetchTxnData; normalizing is what makes the lines comparable.*\n\n###### Healthy range:\n*Below 1.0, with brief touches under load.*\n\n###### Watch for:\n*A line flat at 1.0: that type is the binding constraint. ledgerRequest pinned means the 3 slots shared by RcvGetLedger and RcvGetObjByHash are full, so peer ledger and object requests are queueing behind each other; the Ledger Data and Sync dashboard splits that wait by handler and shows the matching deferred depth. ledgerData or fetchTxnData pinned means inbound ledger data cannot be absorbed and validated ledger age will grow. makeFetchPack or updatePaths pinned at their limit of 1 means a single long job is blocking the whole type. These are sampled gauges, so a line that never reaches 1.0 is not proof the type was never momentarily saturated.*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* — xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Concurrency limit** *(per node)* — the cap on how many jobs of one type may run at once; a type at its cap cannot start more work.\n- **Deferred job** *(per node)* — a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[core/JobQueue.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)\n\n###### Function:\n`JobQueue::getNextJob (limit enforcement) / JobQueue::collect (publication)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#concurrency-limit)", + "description": "###### What this is:\n*How close each concurrency-capped job type is to its ceiling. JobQueue enforces a per-type limit on how many jobs of that type may run at once, and the tight ones carry ledger-sync traffic: makeFetchPack 1, ledgerRequest 3, ledgerData 3, updatePaths 1, fetchTxnData 5. A type at its ceiling cannot start more work no matter how many workers are idle, so this is a different kind of limit from CPU or disk.*\n\n###### How it's computed:\n*Each jobq__running gauge divided by that type's own limit from JobTypes.h, so every line shares one 0-to-1 axis. 1.0 means running equals the limit. Multiply a reading by the limit shown in its legend to recover the raw job count. JobQueue::collect snapshots all three per-type counters under the queue's own lock and publishes them after releasing it, on the 10-second export cycle.*\n\n###### Reading it:\n*Read the distance to 1.0, not the absolute height. Below 1.0 the type has spare slots and its queue wait is not the limit's fault. Touching 1.0 briefly is normal work. Sitting at 1.0 means the type is pinned at its ceiling and every further job of that type is being deferred rather than started, which is what turns into queue wait downstream. Because the limits differ, a raw count of 3 is saturation for ledgerRequest but only 60 percent for fetchTxnData; normalizing is what makes the lines comparable.*\n\n###### Healthy range:\n*Below 1.0, with brief touches under load.*\n\n###### Watch for:\n*A line flat at 1.0: that type is the binding constraint. ledgerRequest pinned means the 3 slots shared by RcvGetLedger and RcvGetObjByHash are full, so peer ledger and object requests are queueing behind each other; the Ledger Data and Sync dashboard splits that wait by handler and shows the matching deferred depth. ledgerData or fetchTxnData pinned means inbound ledger data cannot be absorbed and validated ledger age will grow. makeFetchPack or updatePaths pinned at their limit of 1 means a single long job is blocking the whole type. These are sampled gauges, so a line that never reaches 1.0 is not proof the type was never momentarily saturated.*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* — xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Concurrency limit** *(per node)* — the cap on how many jobs of one type may run at once; a type at its cap cannot start more work.\n- **Deferred job** *(per node)* — a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[core/JobQueue.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)\n\n###### Function:\n`JobQueue::getNextJob (limit enforcement) / JobQueue::collect (publication)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#concurrency-limit)", "type": "timeseries", "gridPos": { "h": 8, diff --git a/docker/telemetry/otel-collector-config.grafanacloud.yaml b/docker/telemetry/otel-collector-config.grafanacloud.yaml index 20b93c7c615..4f33b580ac4 100644 --- a/docker/telemetry/otel-collector-config.grafanacloud.yaml +++ b/docker/telemetry/otel-collector-config.grafanacloud.yaml @@ -203,8 +203,32 @@ connectors: - name: method - name: grpc_role - name: grpc_status + # ledger.acquire dimensions. Must stay identical to the local + # otel-collector-config.yaml dimension set; see the cardinality note + # there for why ledger_hash / ledger_seq are span-only (per-ledger values + # would mint one metric series per ledger) and are indexed in Tempo + # instead. `outcome` includes `abandoned`, recorded when an acquire is + # swept or shut down before reaching a result. - name: outcome - name: acquire_reason + # Sync-diagnostic span dimensions (WP-B2). Must stay identical to the + # local otel-collector-config.yaml dimension set; see the cardinality + # note there for why txset_hash / remote_endpoint / missing_nodes / + # served_nodes / duration_ms are span-only (per-round, per-peer or + # per-integer values would each mint their own metric series) and are + # indexed in Tempo instead. `outcome` above is reused verbatim by all of + # txset.acquire, ledger.acquire.*, peer.dial and ledger.serve. + - name: timed_out + - name: object_type + # Trace-causality span dimensions (WP-B3), on + # consensus.validation.accept. Must stay identical to the local + # otel-collector-config.yaml dimension set; see the reasoning there for + # what each value set is and why the ledger_hash join key stays span-only + # (a per-ledger value would mint one metric series per ledger validated) + # and is indexed in Tempo instead. consensus_round_duration_ms needs no + # entry: it is a native histogram, so the collector only forwards it. + - name: validation_status + - name: accept_gated exporters: otlp/tempo: diff --git a/docker/telemetry/otel-collector-config.yaml b/docker/telemetry/otel-collector-config.yaml index 03ddeac31ec..75a5ea82bc9 100644 --- a/docker/telemetry/otel-collector-config.yaml +++ b/docker/telemetry/otel-collector-config.yaml @@ -187,8 +187,76 @@ connectors: - name: grpc_role - name: grpc_status # ledger.acquire dimensions (bounded: outcome, acquire reason). + # `outcome` carries the four terminal values complete|failed|timeout| + # abandoned. `abandoned` is the one recorded when an acquire is destroyed + # (swept or shut down) before reaching a result, so a stuck-then-swept + # fetch is counted here rather than silently dropped from the rate. + # Deliberately NOT promoted: ledger_hash and ledger_seq. Both are + # per-ledger values, so a dimension on either would mint a new metric + # series for every ledger acquired -- the same unbounded-cardinality rule + # already applied to close_time above. They stay span-only and are + # searchable in Tempo instead (see the parquet_dedicated_columns block in + # tempo.yaml), which gives per-ledger lookup with no metric series. - name: outcome - name: acquire_reason + # Sync-diagnostic span dimensions (WP-B2). Each is a small, closed value + # set, so each adds a bounded number of series per span name: + # timed_out - 2 values, on the ledger.acquire.* phase spans. Split + # by it and a phase-duration panel separates "slow but + # arriving" from "the retry budget ran out". + # object_type - 4 values (header|tx|as|txset), on ledger.serve. Which + # request kind this node serves its peers, so shedding + # account-state tree requests is visible apart from the + # cheap header replies. + # `outcome` above is reused verbatim by the four new spans + # (txset.acquire, the three ledger.acquire.* phases, peer.dial, + # ledger.serve), so it needs no second entry -- one dimension covers + # every span's outcome rate. + # + # Deliberately NOT promoted, same unbounded-cardinality rule as + # ledger_hash / ledger_seq above: + # txset_hash - one value per proposed tx set (per consensus round) + # remote_endpoint - one value per peer address dialled + # missing_nodes - an arbitrary node count + # served_nodes - an arbitrary reply size + # duration_ms - an arbitrary elapsed time + # A dimension on any of these would mint a metric series per round, per + # peer or per distinct integer. They stay span-only and are searchable in + # Tempo instead (see parquet_dedicated_columns in tempo.yaml); the + # aggregate view of the same quantities comes from the spanmetrics + # duration histogram and from the WP-A3 native gauges. + - name: timed_out + - name: object_type + # Trace-causality span dimensions (WP-B3), on + # consensus.validation.accept. Both are closed value sets: + # validation_status - 6 values (the 5 ValStatus outcomes plus an + # unknown sentinel). Only `current` reaches the + # acceptance gate, so splitting by it separates a + # node whose arriving validations are counting from + # one whose validations are all rejected -- which + # from the outside look identical, both being a + # node that receives validations and never + # validates. + # accept_gated - 2 values. True when this validation did not reach + # the gate because another thread was already + # accepting the same ledger, which is why a trace + # can show a validation with no ledger.validate + # after it. Without the dimension that gap looks + # like a lost span. + # + # Deliberately NOT promoted, same rule as ledger_hash / ledger_seq above: + # nothing here carries a per-ledger value. The join key itself + # (ledger_hash) is exactly such a value and stays span-only, indexed in + # tempo.yaml, because the per-ledger lookup it exists for is a trace + # search, not a metric query -- and as a dimension it would mint one + # series per ledger validated. + # + # consensus_round_duration_ms needs NO entry here at all: it is a native + # histogram from the OTel SDK, so it rides the existing OTLP -> Prometheus + # path and the collector only forwards it. Only span-derived signals need + # a dimension. + - name: validation_status + - name: accept_gated exporters: debug: diff --git a/docker/telemetry/tempo.yaml b/docker/telemetry/tempo.yaml index 24c092ca95f..ef0b99db2ee 100644 --- a/docker/telemetry/tempo.yaml +++ b/docker/telemetry/tempo.yaml @@ -82,3 +82,61 @@ storage: path: /var/tempo/wal local: path: /var/tempo/blocks + block: + # Give the highest-value span attributes their own Parquet columns so + # TraceQL scopes a search to one ledger without scanning the generic + # attribute list. Tempo indexes every attribute for search regardless; + # a dedicated column only changes the cost of filtering on it. + # + # ledger_hash is here rather than in the collector's spanmetrics + # `dimensions:` on purpose: it is a per-ledger value, so as a metric + # dimension it would mint one series per ledger acquired (unbounded + # cardinality). As a trace column it gives the same per-ledger lookup at + # zero metric cost -- which is how a single stuck or abandoned + # ledger.acquire is found: {name="ledger.acquire" && span.outcome= + # "abandoned" && span.ledger_hash=""}. + # + # txset_hash and remote_endpoint are here for the same reason and by the + # same rule: a tx-set root hash is one value per consensus round and a + # peer endpoint is one value per address dialled, so either as a + # spanmetrics dimension would mint an unbounded number of metric series. + # As trace columns they answer the per-object questions the metrics + # cannot -- which proposed set stalled a round: + # {name="txset.acquire" && span.outcome="timeout" && + # span.txset_hash=""} + # and which peer a dial keeps failing against: + # {name="peer.dial" && span.outcome="tcp_fail" && + # span.remote_endpoint="10.0.0.5:51235"} + # + # missing_nodes, served_nodes and duration_ms are deliberately NOT given + # columns. They are unbounded integers, so they are useless as equality + # filters; Tempo indexes every attribute for search regardless, so they + # remain queryable by range (e.g. span.missing_nodes > 0) at no + # storage cost, which is how they are actually read. + # + # The ledger_hash column below carries a second job as of the per-ledger + # trace join (WP-B3), which is why it stays the highest-value column here. + # Every span for one ledger -- ledger.acquire, consensus.validation.accept, + # ledger.validate, ledger.store -- now derives its TRACE ID from that same + # ledger hash, so the whole story of one slow ledger is one trace: + # {span.ledger_hash=""} + # returns all of them across the three threads that produced them. The + # column is what makes that lookup cheap; it also stays the way a reader + # confirms two spans really are the same ledger rather than a trace-id + # coincidence, since the attribute holds the full 32 bytes while the trace + # id is only the leading 16. + # + # validation_status and accept_gated get no column: both are small closed + # value sets promoted to spanmetrics dimensions instead (see the collector + # configs), so the aggregate question is answered by a metric query and no + # per-value trace lookup is needed. + parquet_dedicated_columns: + - name: ledger_hash + type: string + scope: span + - name: txset_hash + type: string + scope: span + - name: remote_endpoint + type: string + scope: span diff --git a/docker/telemetry/workload/README.md b/docker/telemetry/workload/README.md index eb926af6b9c..147ac79be0f 100644 --- a/docker/telemetry/workload/README.md +++ b/docker/telemetry/workload/README.md @@ -203,7 +203,7 @@ Automated validation that all expected telemetry data exists. Every metric and s - **Span validation**: All span types from `expected_spans.json` with required attributes and parent-child hierarchies - **Metric validation**: All metrics from `expected_metrics.json` — SpanMetrics, StatsD gauges/counters/histograms, Phase 9 OTLP metrics. Every listed metric must have > 0 series. Uses the Prometheus `/api/v1/series` endpoint (not instant queries) to avoid false negatives from stale gauges. - **Log-trace correlation**: trace_id/span_id in Loki logs (requires Loki) -- **Dashboard validation**: All 10 Grafana dashboards load with panels +- **Dashboard validation**: All 15 Grafana dashboards load with panels ```bash # Run all validations diff --git a/docker/telemetry/workload/expected_metrics.json b/docker/telemetry/workload/expected_metrics.json index 8a5b2be439c..2735e97eae2 100644 --- a/docker/telemetry/workload/expected_metrics.json +++ b/docker/telemetry/workload/expected_metrics.json @@ -47,7 +47,7 @@ "metrics": ["rpc_requests_total", "ledger_fetches_total"] }, "overlay_traffic": { - "description": "Overlay traffic metrics (subset — full list has 45+ categories).", + "description": "Overlay traffic metrics (subset \u2014 full list has 45+ categories).", "metrics": [ "total_bytes_in", "total_bytes_out", @@ -127,6 +127,76 @@ "description": "External dashboard parity: storage detail metrics (MetricsRegistry).", "metrics": ["storage_detail{metric=\"stored_object_bytes\"}"] }, + "sync_diagnostics": { + "description": "Fresh-node sync diagnostics (native metrics). Bootstrap (Domain 0) and acquire-pipeline signals rendered by the ledger-sync-health dashboard. Names are appended one per signal. Histograms are listed by their Prometheus _bucket series (the bare instrument name is not a series). The two observable gauges carry an inline metric= selector so the specific sub-series is asserted, matching the parity_* groups.", + "metrics": [ + "dns_resolve_total", + "dns_resolve_latency_ms_bucket", + "overlay_connect_total", + "overlay_dial_latency_ms_bucket", + "handshake_negotiation_fail_total", + "unl_fetch_total", + "unl_quorum{metric=\"trusted_keys\"}", + "unl_quorum{metric=\"quorum\"}", + "clock_close_offset_seconds{metric=\"offset\"}", + "sync_state{metric=\"initial_full_duration_us\"}", + "sync_state{metric=\"network_ledger_gate\"}", + "sync_state{metric=\"server_stall_seconds\"}", + "sync_state{metric=\"ledgers_behind\"}", + "server_stall_events_total", + "state_changes_total{from!=\"\",to!=\"\"}", + "sync_acquire{metric=\"missing_state_nodes_max\"}", + "sync_acquire{metric=\"missing_tx_nodes_max\"}", + "sync_acquire{metric=\"received_data_depth\"}", + "sync_acquire{metric=\"in_flight\"}", + "shamap_cache_hit_rate{metric=\"treenode\"}", + "jobq_saturation{metric=\"running_tasks\"}", + "jobq_saturation{metric=\"worker_threads\"}", + "jobq_saturation{metric=\"total_waiting\"}", + "peer_ledger_supply{metric=\"peers_reporting\"}", + "peer_ledger_supply{metric=\"peers_serving_validated\"}", + "peer_ledger_supply{metric=\"peers_serving_next\"}", + "peer_ledger_supply{metric=\"supply_min_seq\"}", + "peer_ledger_supply{metric=\"supply_max_seq\"}", + "peerfinder_slot_census{metric=\"out_active\"}", + "peerfinder_slot_census{metric=\"out_max\"}", + "peerfinder_slot_census{metric=\"in_active\"}", + "peerfinder_slot_census{metric=\"in_max\"}", + "peerfinder_slot_census{metric=\"connecting\"}", + "peerfinder_slot_census{metric=\"fixed_configured\"}", + "peerfinder_slot_census{metric=\"fixed_active\"}", + "peerfinder_slot_census{metric=\"bootcache\"}", + "peerfinder_slot_census{metric=\"livecache\"}", + "amendment_block{metric=\"warned\"}", + "amendment_block{metric=\"seconds_to_block\"}", + "peer_accept_total", + "nodestore_state{metric=\"node_writes\"}", + "nodestore_state{metric=\"node_reads_total\"}", + "nodestore_state{metric=\"read_mean_us\"}", + "ledger_quorum_publish{metric=\"trusted_validation_tally\"}", + "ledger_quorum_publish{metric=\"quorum_target\"}", + "ledger_quorum_publish{metric=\"time_to_first_validated_us\"}", + "ledger_quorum_publish{metric=\"publish_lag\"}", + "ledger_quorum_shortfall_total{stage=\"pre_accept\"}", + "consensus_round_duration_ms_bucket", + "consensus_round_duration_ms_count", + "nodestore_state{metric=\"node_writes_duration_us\"}", + "nodestore_state{metric=\"node_reads_duration_us\"}", + "unl_quorum{metric=\"quorum_disabled\"}", + "sweep_malloc_trim_us_bucket", + "sweep_malloc_trim_us_count" + ], + "_acquire_note": "The four sync_acquire sub-series and shamap_cache_hit_rate are unconditional: both are observable gauges whose callbacks observe every series on each collection tick, so each is present even when the value is 0 (an idle node reports in_flight=0 and missing_state_nodes_max=0, and a cold cache reports a 0.0 hit rate). Absence, not a zero, is the regression. The three WP-A3 counters (sync_acquire_source_total, sync_addnode_total, sync_acquire_no_progress_total) are deliberately NOT asserted here: all three are emitted only from InboundLedger, which runs only when a node must fetch a ledger it lacks. expected_spans.json already marks the ledger.acquire span optional for exactly this reason (\"A healthy local cluster rarely back-fills history\"), and the metric validator has no per-metric optional flag, so listing them would fail the harness red on a healthy run. They are covered by exact-value unit tests in src/tests/libxrpl/telemetry/MetricMacros.cpp and by the ledger-sync-health panels; add them here only alongside a harness step that forces a real acquire (e.g. starting a node against an existing ledger history).", + "_jobq_note": "The three jobq_saturation series are unconditional: it is an observable gauge whose callback observes all three fields on every collection tick, so each series exists even when the value is 0, and absence rather than a zero is the regression. worker_threads is asserted because it is the denominator of the dashboard saturation ratio, and it is always at least 1 (the JobQueue ctor gives standalone mode exactly one worker), so a zero or missing reading there means the accessor regressed rather than the node being idle. The per-job-type waiting/running/deferred counts are published separately by JobQueue::collect() as the beast::insight gauges jobq__waiting / _running / _deferred, which the collector translates; they are covered by the StatsD-derived groups, not here.", + "_conditional_note": "handshake_negotiation_fail_total and unl_fetch_total are conditional under the local harness: the first only exists once a handshake is rejected, and the second needs a [validator_list_sites] entry (run-full-validation.sh generates a static [validators] file instead). The validator has no per-metric optional flag, so if either reports 0 series in a harness run, move it out of this group rather than weakening the check.", + "_sync_state_note": "The four sync_state sub-series are unconditional: the gauge observes all four on every collection tick, so each is present as a series even when its value is 0 (a node that never reached FULL reports initial_full_duration_us=0, and a healthy node reports server_stall_seconds=0). The check asserts series presence, not a non-zero value, which is exactly right here \u2014 a zero is a meaningful reading for these signals, and absence is the regression. server_stall_events_total is likewise always present because the observable counter reports the tally (0 or more) every tick. state_changes_total is asserted here with a from!=\"\",to!=\"\" selector rather than bare (parity_counters already asserts the bare name): the selector is what proves the WP-A2 {from,to} label dimension actually reached Prometheus, so a regression to the old unlabelled counter fails this check instead of silently passing on the bare name. It needs at least one real mode transition, which any node reaching connected/syncing produces during startup.", + "_a7_note": "WP-A7 adds three observable gauges and four counters. The 16 gauge sub-series (peer_ledger_supply, peerfinder_slot_census, amendment_block) are unconditional and asserted individually: each callback in MetricsRegistry.cpp calls observe() for every field on every collection tick with no early return between them, so the series exists whatever the value. That includes the two sentinel readings \u2014 a node whose peers have advertised nothing reports peer_ledger_supply{metric=\"supply_min_seq\"} = 0 meaning unknown, and a node with no pending amendment reports amendment_block{metric=\"seconds_to_block\"} = -1 meaning healthy. Absence, not the sentinel, is the regression. Of the four counters only peer_accept_total is asserted: run-full-validation.sh gives every node a [port_peer] on 0.0.0.0 and lists the other four nodes in [ips], so all 5 nodes dial each other and each one is also dialled, which means OverlayImpl::onHandoff runs and reports outcome=accepted (or slot_refused/no_slot on the duplicate half of each mutual dial) on every node. It is asserted bare rather than with an outcome= selector because which outcome a given node records depends on dial ordering, which the harness does not control. The other three counters are deliberately NOT asserted. peer_disconnect_total is emitted only from PeerImp::close, and a healthy 5-node localhost cluster holds its 4 fixed peers for the whole run: the timer-driven reasons need maxUnknownTime (600 s) or maxDivergedTime (300 s) to elapse (Config.h) while the full-validation profile totals well under that, and the shutdown reasons only fire during teardown, which happens in run-full-validation.sh after Step 5 has already scraped. serve_refused_total needs a peer to ask this node for a ledger, tx set or object it cannot serve \u2014 on a cluster where every node has the same complete history from genesis, getLedger()/getTxSet() succeed and the send queues never approach Tuning::kDropSendQueue. ledger_jump_total needs NetworkOPsImp::switchLastClosedLedger, reached only when consensus reports an LCL this node did not build on; a healthy 5-node cluster agrees every round, so it never jumps. The metric validator has no per-metric optional flag, so listing any of the three would fail the harness red on a healthy run \u2014 the same reasoning _acquire_note applies to the WP-A3 InboundLedger counters. All four counters are covered by exact-value unit tests in src/tests/libxrpl/telemetry/MetricMacros.cpp and rendered by the ledger-sync-health panels Peer Disconnects by Reason, Ledger/Object Serve Refusals and Byzantine Ledger Jumps. To make them assertable the harness would need a fault-injection step: kill one node mid-run and re-scrape before teardown (peer_disconnect_total, reason=read_error/graceful), request a ledger sequence outside the cluster's history or drive a node past its send-queue limit (serve_refused_total), and start a node on a divergent chain tip or partition the cluster and heal it (ledger_jump_total).", + "_a6_note": "WP-A6 originally added a separate nodestore_latency gauge; it was retired because every one of its sub-metrics had an exact counterpart on nodestore_state computed from the same Database accessor, so the five entries above are the surviving equivalents (write_count -> node_writes, read_count -> node_reads_total, write_duration_us -> node_writes_duration_us, read_duration_us -> node_reads_duration_us, read_mean_us unchanged). All four totals are unconditional: MetricsRegistry::observeNodeStoreTotals observes them on every collection tick with no early return before them, so a series exists whatever the value and a node that has written nothing reports node_writes=0 rather than dropping the series. read_mean_us is safe because any node that has opened a ledger has already fetched objects, so the fetch count is non-zero. write_mean_us is NOT asserted, but only because the two means are the sub-series that scaledMean() omits when their denominator is zero, and this validator has no per-metric optional flag -- not because the numerator is missing. That older caveat is gone: all three concrete store paths now time themselves through Database::recordStoreDuration() (DatabaseNodeImp::store, DatabaseRotatingImp::store and Database::importInternal), so write_mean_us is live on an ordinary node and only a node that has performed literally zero stores would lack it. It can be promoted to an assertion once a harness run confirms it present. WP-A6's two replay counters (ledger_replay_fallback_total, ledger_replay_outcome_total) are likewise NOT asserted, for the same reason _acquire_note gives for the WP-A3 InboundLedger counters: both are emitted only from the ledger-replay path, which requires the [ledger_replay] config stanza AND a real historical back-fill against peers that support the LedgerReplay protocol feature. run-full-validation.sh starts a fresh local cluster with no history to back-fill, so no replay task is ever created and neither counter can produce a series. Both are covered by exact-value unit tests in src/tests/libxrpl/telemetry/MetricMacros.cpp and rendered by the ledger-sync-health panels Replay Fallback to Full Acquire and Replay Outcomes. To make them assertable the harness would need to enable [ledger_replay] and start a node against an existing ledger history so it back-fills through the replay path.", + "_a5_note": "WP-A5 adds one observable gauge (ledger_quorum_publish) and one counter (ledger_quorum_shortfall_total). All four gauge sub-series are asserted and are unconditional: registerLedgerQuorumPublishGauge's callback in MetricsRegistry.cpp calls observe() for every field on every collection tick with no early return between them, and each accessor is a plain relaxed atomic load that always returns a value, so the series exists whatever the reading. That deliberately includes the three diagnostic zeros: a node that has never had a gate evaluated reports trusted_validation_tally=0 and quorum_target=0, one that has never fully validated reports time_to_first_validated_us=0, and one that is caught up reports publish_lag=0. Absence, not the zero, is the regression -- the same reasoning _sync_state_note gives for initial_full_duration_us. Note the sentinel: when the trusted list disables quorum entirely, getNeededValidations() returns SIZE_MAX and LedgerMaster reports quorum_target as int64 max rather than letting the cast wrap to -1, so the target reads far above any tally instead of inverting the comparison (the same fix as the unl_quorum gauge). ledger_quorum_shortfall_total IS asserted, which differs from the WP-A3/A6/A7 counters, and the reason is that this counter does not need a fault to fire. RCLConsensus::Adaptor::doAccept issues this node's own validation and then calls ledgerMaster_.consensusBuilt immediately (RCLConsensus.cpp), which calls checkAccept on the freshly built ledger (LedgerMaster.cpp) BEFORE the peers' validations for that same ledger have arrived. With the harness's 5 validators the quorum is max(ceil(5*0.8), ceil(5*0.6)) = 4 (ValidatorList::calculateQuorum), so that first evaluation of each round tallies short of 4 and takes the shortfall early return; the gate is then re-entered from RCLValidations handleNewValidation as each trusted validation arrives and eventually passes. A HEALTHY 5-node cluster therefore emits this counter every round, which is why it is safe to assert on a clean run -- unlike peer_disconnect_total or ledger_replay_fallback_total, it needs no fault injection, no [ledger_replay] stanza and no historical back-fill. The stage=\"pre_accept\" selector is asserted rather than the bare name so that the label dimension is proven to have reached Prometheus, matching the state_changes_total{from,to} pattern. Consequence for readers of the panels: a non-zero rate on Pre-Accept Quorum Shortfall Rate is NOT by itself a fault, and the panel description says so; the fault signature is that rate climbing well above the ledger-close rate while the tally on Trusted Validations vs Quorum Target stays flat below its target. If a future harness change makes the cluster single-node or standalone this assertion must move to a note: standalone_ short-circuits consensusBuilt before checkAccept, and getNeededValidations() returns 0 in standalone mode, so the gate can never report a shortfall.", + "_b5_sweep_note": "WP-B5 Suspect 3 (per-sweep malloc_trim) adds one histogram and two counters. Only the histogram is asserted, by its Prometheus _bucket and _count series because the bare instrument name is not a series. It is unconditional on any running node: ApplicationImp::start arms the sweep timer before the workload begins, the interval is SizedItem::SweepInterval (Config.cpp: 10 s at nodeSize 0 through 120 s at nodeSize 4), and the full-validation profile runs 270 s of workload before Step 5 scrapes -- so at least two sweeps complete even in the slowest case, and each one records exactly one sample. The measurement itself is now unconditional too: it used to sit inside `if (journal.debug())` in MallocTrim.cpp, so a node at ordinary log level measured nothing; that gate now covers only the JLOG. The two counters are deliberately NOT asserted. sweep_malloc_trim_minor_faults_total is emitted only when the trim's minor-fault delta is greater than zero, and a trim on the small heap of a fresh localhost node routinely faults zero times -- the emit site publishes nothing rather than a zero, because a zero-valued series would claim the trim was measured as free when the honest statement is that there was nothing to fault on. sweep_malloc_trim_reclaimed_kb_total is emitted only when resident memory actually FELL across the trim, which on a node whose caches are still filling frequently does not happen (glibc has nothing above the top of the heap to release, and mmap-backed allocations are returned on free regardless of trimming). The validator has no per-metric optional flag, so asserting either would ship a permanently red CI check for a healthy run. Both are covered by exact-value unit tests in src/tests/libxrpl/telemetry/MetricMacros.cpp -- including the skip paths -- and rendered by the ledger-sync-health panel Sweep Heap-Trim Faults & Reclaim Rate. To make them assertable the harness would need a node with a large enough resident heap for a trim to reclaim, e.g. starting against an existing populated database rather than from genesis.", + "_b5_rotation_note": "WP-B5 Suspect 4 (online_delete rotation extra writes) adds one observable gauge (rotation_state, sub-series in_flight and copy_forward) and one counter (rotation_copy_node_restore_total). NONE is asserted, because the 5-node localhost harness structurally CANNOT produce any of them -- this is a documented note rather than a check that would fail CI red. Two independent reasons. First, no rotation ever runs: xrpld-validator.cfg.template sets online_delete=256 and does not set advisory_delete, so SHAMapStoreImp's gate is validatedSeq >= lastRotated + 256 (SHAMapStoreImp.cpp), which needs 256 validated ledgers; at the network's several-seconds-per-ledger close rate that is on the order of 15-20 minutes, while the full-validation profile totals 270 s of workload before Step 5 scrapes. Second, even the in_flight flag needs a rotation to have started, and copy_forward additionally needs an ARCHIVE holding data that a fetch actually reads during the rotation window -- which requires a populated, already-rotated database, exactly the condition the hypothesis says is why this slowdown never appears on a fresh node. rotation_copy_node_restore_total is narrower still: it fires only for a clean tree node reachable from the validated state map whose sole on-disk copy was removed by an EARLIER rotation, so it needs at least two rotations plus real prior data loss. Note that rotation_state publishes no series at all when online_delete is not configured, by design: MetricsRegistry::registerRotationStateGauge dynamic_casts the node store to DatabaseRotating and returns early on failure, so an absent series means 'rotation is not configured' rather than the false 'rotation is free' a zero would report. All four signals are covered by exact-value unit tests in src/tests/libxrpl/telemetry/MetricMacros.cpp (including the not-configured and between-rotations cases) and rendered by the ledger-sync-health panels Online-Delete Rotation Window & Copy-Forward Writes and Rotation Node Re-Store Rate. To make them assertable the harness would need a step that starts a node against a pre-populated database that has already rotated at least once, or that lowers online_delete and advisory_delete far enough to force a rotation inside the run window and then re-scrapes before teardown.", + "_round_histogram_note": "consensus_round_duration_ms is a native OTel histogram recorded once per consensus round in RCLConsensus, so it needs no collector configuration -- it rides the existing OTLP -> Prometheus path. It is asserted by its Prometheus _bucket and _count series because the bare instrument name is not a series. Both are unconditional on any running cluster: every node closes ledgers continuously, so a round completes within the harness window and the histogram is populated. Absence means the record site or the explicit-bucket view regressed, not that the node was idle. The instrument carries NO labels, so exactly one series exists per node and per bucket boundary." + }, "grafana_dashboards": { "description": "All Grafana dashboards that must render data (UIDs as provisioned on disk under docker/telemetry/grafana/dashboards/).", "uids": [ @@ -143,7 +213,8 @@ "network-traffic", "rpc-pathfinding", "overlay-traffic-detail", - "ledger-data-sync" + "ledger-data-sync", + "ledger-sync-health" ] } } diff --git a/docker/telemetry/workload/expected_spans.json b/docker/telemetry/workload/expected_spans.json index f663f303cf3..ec6cf28935e 100644 --- a/docker/telemetry/workload/expected_spans.json +++ b/docker/telemetry/workload/expected_spans.json @@ -1,5 +1,5 @@ { - "description": "Expected span inventory for xrpld telemetry validation. Attribute keys follow the 2026-05-13 span-attr naming redesign (bare/underscore form; dotted xrpl.* reserved for resource attributes). Sourced from the *SpanNames.h headers. Spans marked \"optional\": true are conditional — they only fire under traffic the harness may not produce (e.g. gRPC client, missing-ledger fetch, mode transitions) and are not failed when absent.", + "description": "Expected span inventory for xrpld telemetry validation. Attribute keys follow the 2026-05-13 span-attr naming redesign (bare/underscore form; dotted xrpl.* reserved for resource attributes). Sourced from the *SpanNames.h headers. Spans marked \"optional\": true are conditional \u2014 they only fire under traffic the harness may not produce (e.g. gRPC client, missing-ledger fetch, mode transitions) and are not failed when absent.", "spans": [ { "name": "rpc.ws_message", @@ -22,7 +22,7 @@ "parent": "rpc.process", "required_attributes": ["command", "version", "rpc_role", "rpc_status"], "config_flag": "trace_rpc", - "note": "Wildcard — matches rpc.command.server_info, rpc.command.ledger, etc." + "note": "Wildcard \u2014 matches rpc.command.server_info, rpc.command.ledger, etc." }, { "name": "rpc.http_request", @@ -87,7 +87,7 @@ "required_attributes": ["tx_hash", "tx_type", "txq_status"], "config_flag": "trace_transactions", "optional": true, - "note": "Only fires when a tx is queued (fee below open-ledger level). Requires fee escalation — driven by the txq-burst workload phase. tx_hash/tx_type/txq_status are set on every code path; fee_level_paid/required_fee_level are conditional (TxQ.cpp ~895-898, after the rejected and applied_direct early exits), so they are NOT guaranteed on every txq.enqueue span and cannot be required." + "note": "Only fires when a tx is queued (fee below open-ledger level). Requires fee escalation \u2014 driven by the txq-burst workload phase. tx_hash/tx_type/txq_status are set on every code path; fee_level_paid/required_fee_level are conditional (TxQ.cpp ~895-898, after the rejected and applied_direct early exits), so they are NOT guaranteed on every txq.enqueue span and cannot be required." }, { "name": "txq.apply_direct", @@ -259,6 +259,20 @@ "config_flag": "trace_consensus", "note": "Context-propagated from the sending peer. No required local attributes." }, + { + "name": "consensus.validation.accept", + "category": "consensus", + "parent": null, + "required_attributes": [ + "ledger_hash", + "ledger_seq", + "validation_status", + "accept_gated", + "full_validation" + ], + "config_flag": "trace_ledger", + "note": "Emitted once per TRUSTED validation as it reaches the ledger-acceptance gate (handleNewValidation), so its rate is bounded by the UNL size per ledger close. Required, not optional: every node in the harness cluster validates and every other node receives those validations, so the span fires continuously. config_flag is trace_ledger, not trace_consensus, because the span is created through the ledger-hash trace join (TraceCategory::Ledger) -- it belongs to the ledger's trace, not the round's. Its trace id is derived from the VALIDATED ledger hash, the same key ledger.validate and ledger.store use, which is what makes one slow ledger read as one connected trace across the three threads that produce those spans. validation_status is one of the 6 ValStatus values and accept_gated says whether the validation actually reached the gate; both are spanmetrics dimensions. See trace_join_groups below." + }, { "name": "consensus.mode_change", "category": "consensus", @@ -303,12 +317,85 @@ "ledger_seq", "acquire_reason", "timeouts", - "peer_count", + "outcome", + "ledger_hash" + ], + "config_flag": "trace_ledger", + "optional": true, + "note": "Only fires when a node must fetch a missing ledger (InboundLedger). A healthy local cluster rarely back-fills history. outcome is one of complete|failed|abandoned and is stamped on every exit path, including the sweep/shutdown path where the fetch never finished (abandoned). ledger_hash identifies the target ledger from the first moment, since a by-hash acquire starts with ledger_seq 0. peer_count is not required: the destructor path deliberately skips the peer lookup to avoid taking the Overlay lock under the InboundLedgers collection lock." + }, + { + "name": "ledger.acquire.header", + "category": "ledger", + "parent": "ledger.acquire", + "required_attributes": ["ledger_hash", "outcome", "timed_out"], + "config_flag": "trace_ledger", + "optional": true, + "note": "Child of ledger.acquire, so it is optional for the same reason the parent is: it exists only while a node is fetching a missing ledger, and a healthy 5-node cluster that has agreed from genesis rarely back-fills. Covers the wait for the ledger header, which gates both tree phases -- until it arrives the account-state and transaction root hashes are unknown, so nothing else can be requested. missing_nodes is deliberately absent: a header is a single object, not a tree." + }, + { + "name": "ledger.acquire.astree", + "category": "ledger", + "parent": "ledger.acquire", + "required_attributes": [ + "ledger_hash", + "outcome", + "timed_out", + "missing_nodes" + ], + "config_flag": "trace_ledger", + "optional": true, + "note": "Child of ledger.acquire (optional for the same reason). The account-state SHAMap phase, which is nearly all of the work in a real fresh sync -- the flat parent span could not separate it from the small transaction tree. outcome=timeout with a non-zero missing_nodes is the 'peers are not serving this tree' signature. missing_nodes is read from the count getMissingNodes() already produced during its sweep; no extra tree walk." + }, + { + "name": "ledger.acquire.txtree", + "category": "ledger", + "parent": "ledger.acquire", + "required_attributes": [ + "ledger_hash", + "outcome", + "timed_out", + "missing_nodes" + ], + "config_flag": "trace_ledger", + "optional": true, + "note": "Child of ledger.acquire (optional for the same reason). The transaction SHAMap phase. Usually completes long before the astree phase, and that asymmetry is the point of splitting them: the parent span's duration is the state tree's, not this one's." + }, + { + "name": "ledger.serve", + "category": "ledger", + "parent": null, + "required_attributes": [ + "object_type", + "peer_id", + "served_nodes", "outcome" ], "config_flag": "trace_ledger", + "note": "The supply side: this node answering a peer's TMGetLedger request on the JtLedgerReq worker. Required (not optional) because every node in the harness cluster is listed in the others' [ips], so they exchange ledger and tx-set requests continuously throughout the run. A fresh trace root -- the request arrives from the wire on a shared worker, so it must not inherit an unrelated span active there. object_type is header|tx|as|txset and outcome is complete|partial|refused, both derived by shared rules in LedgerSpanNames.h so the eight exits of processLedgerRequest cannot disagree. ledger_seq is present only once getLedger() succeeded, so it is not required." + }, + { + "name": "txset.acquire", + "category": "ledger", + "parent": null, + "required_attributes": [ + "txset_hash", + "outcome", + "timeouts", + "duration_ms", + "peer_count" + ], + "config_flag": "trace_ledger", "optional": true, - "note": "Only fires when a node must fetch a missing ledger (InboundLedger). A healthy local cluster rarely back-fills history." + "note": "One attempt to fetch a transaction set a consensus proposal referenced (TransactionAcquire, which had zero telemetry before WP-B2). Optional because it only fires when a node does NOT already hold a proposed set: in the harness cluster every node sees the same relayed transactions and builds the same set locally, so InboundTransactions::getSet finds it in its map and never constructs a TransactionAcquire. It is the sibling of ledger.acquire -- same TimeoutCounter base, same trigger/onTimer/takeNodes shape -- and shares the trace_ledger flag so the two halves of a stuck sync cannot be enabled apart. outcome is complete|failed|timeout|abandoned, stamped on both exits (done() and the destructor when the round sweep drops the set)." + }, + { + "name": "peer.dial", + "category": "peer", + "parent": null, + "required_attributes": ["remote_endpoint", "outcome", "duration_ms"], + "config_flag": "trace_peer", + "note": "One outbound connect attempt (ConnectAttempt), a fresh trace root because a dial is the first thing a starting node does and there is nothing to parent it to. Required: run-full-validation.sh lists the other four nodes in each node's [ips], so every node dials and the span always fires. Telemetry is live in time to catch it -- ApplicationImp::setup() calls startTelemetry() before start() calls overlay_->start(). outcome carries the same six values as the overlay_connect_total counter (connected|tcp_fail|tls_fail|self_connection|upgrade_fail|timeout) and is set from the same reportOutcome() funnel, so span and counter cannot disagree. remote_endpoint is the span-only dimension the counter cannot carry, since one series per peer address would be unbounded cardinality." }, { "name": "peer.proposal.receive", @@ -375,7 +462,7 @@ "required_attributes": ["method", "grpc_role", "grpc_status"], "config_flag": "trace_rpc", "optional": true, - "note": "Wildcard — grpc.. The harness has no gRPC client, so these do not fire. Tracked for completeness." + "note": "Wildcard \u2014 grpc.. The harness has no gRPC client, so these do not fire. Tracked for completeness." } ], "parent_child_relationships": [ @@ -412,8 +499,29 @@ "description": "Pathfind request contains the compute sub-span", "skip": true, "skip_reason": "pathfind.compute only fires when a path computation actually runs; the self-to-self XRP probe in a fresh cluster with no liquidity returns before computing, so the child is not emitted under the harness workload." + }, + { + "parent": "ledger.acquire", + "child": "ledger.acquire.astree", + "description": "Ledger acquire contains the account-state tree fetch phase", + "skip": true, + "skip_reason": "The parent ledger.acquire is itself optional: it only fires when a node must fetch a missing ledger, and a healthy 5-node cluster agreeing from genesis rarely back-fills history. The hierarchy check has no optional handling and fails when the parent produces no traces, so it is skipped rather than shipped red. The parenting itself is explicit and not thread-dependent -- beginPhaseSpan() parents through the acquire span's own captured SpanContext, not the ambient thread context -- so it holds on whichever worker opens a phase. Un-skip once the harness gains a step that forces a back-fill (start a sixth node with an empty database against the running cluster)." } ], - "total_span_types": 40, - "total_unique_attributes": 58 + "trace_join_groups": { + "description": "Groups of spans that share ONE trace id without any parent/child link between them, because each derives its trace id deterministically from the same hash (SpanGuard::hashSpan). This is how spans produced on unrelated threads are joined: no context is propagated, so there is no parent to assert -- the assertion is that the spans co-occur in a single trace. Checked by assert_trace_join_groups() in validate_telemetry.py, which searches for the anchor span and requires at least one of its traces to also contain every member listed in required_members.", + "groups": [ + { + "name": "per_ledger", + "join_key": "ledger_hash", + "anchor": "ledger.validate", + "required_members": ["ledger.store"], + "optional_members": ["consensus.validation.accept", "ledger.acquire"], + "note": "One ledger's spans, keyed on its own 32-byte hash: the acceptance decision (ledger.validate, LedgerMaster::checkAccept), the persist (ledger.store), the trusted validation that drove acceptance (consensus.validation.accept) and the network fetch (ledger.acquire). Each runs on a different thread, so before the join each was its own single-span trace and a slow ledger could not be read as one unit. ledger.validate is the anchor because it fires for every validated ledger. ledger.store is required with it: both run for every ledger the node accepts, and checkAccept reaches storeLedger on the acquire path while consensus reaches it via buildLCL, so at least one ordering always produces both. consensus.validation.accept is optional here only because a ledger this node built itself is accepted via switchLCL without a peer validation arriving first, so the two need not land in the same trace on every ledger. ledger.acquire is optional for the reason its own entry gives: a healthy cluster agreeing from genesis rarely back-fills." + } + ] + }, + "_conditional_attributes_note": "Five attributes documented in the 'Fresh-node sync diagnostics' table of OpenTelemetryPlan/09-data-collection-reference.md are deliberately absent from required_attributes above, because each is emitted only when its value is known and _validate_span_attributes_otlp() has no per-attribute optional flag -- listing one would fail CI red on a healthy run. ledger.acquire/peer_count is set only when finalizeAcquireSpan() is passed a peer count (InboundLedger.cpp), which the sweep and shutdown paths cannot supply. ledger_seq on the three ledger.acquire.header/.astree/.txtree phase spans is set only when seq_ != 0 (InboundLedger.cpp startPhaseSpan), and a by-hash acquire starts with seq_ == 0 and learns the sequence only when the header arrives -- so a phase that opens before the header legitimately carries no sequence. ledger_seq on ledger.serve is set only when the reply carries one (PeerImp.cpp), which an object-by-hash request does not. All five ARE indexed in the 09-reference table and rendered by the Ledger Sync Health board; the honest encoding is to document them here rather than assert a conditional attribute as required.", + "total_span_types": 47, + "total_unique_attributes": 76 } diff --git a/docker/telemetry/workload/validate_telemetry.py b/docker/telemetry/workload/validate_telemetry.py index 40f7e507a82..a6e1c9524dd 100644 --- a/docker/telemetry/workload/validate_telemetry.py +++ b/docker/telemetry/workload/validate_telemetry.py @@ -8,9 +8,11 @@ Validation categories: 1. Span validation — All 16+ span types present with required attributes 2. Metric validation — SpanMetrics, StatsD, and Phase 9 metrics are non-zero - 3. Log-trace correlation — Loki logs contain trace_id/span_id fields - 4. Dashboard validation — All 14 Grafana dashboards render data - 5. External parity — Span attrs, metric existence, and value sanity for + 3. Sync diagnostics — Fresh-node sync signals (bootstrap + acquire + pipeline) declared in the "sync_diagnostics" group + 4. Log-trace correlation — Loki logs contain trace_id/span_id fields + 5. Dashboard validation — All 15 Grafana dashboards render data + 6. External parity — Span attrs, metric existence, and value sanity for external dashboard parity (validator-health, peer-quality, node-health) @@ -62,6 +64,12 @@ METRIC_POLL_TIMEOUT_SEC = 45.0 METRIC_POLL_INTERVAL_SEC = 5.0 +# Group key in expected_metrics.json holding the fresh-node sync-diagnostics +# metrics (bootstrap + acquire pipeline). Owned by +# assert_sync_diagnostics_metrics() so those signals get their own report +# category and a single, explicit failure per missing metric. +SYNC_DIAGNOSTICS_GROUP = "sync_diagnostics" + # --------------------------------------------------------------------------- # Data classes @@ -514,10 +522,152 @@ async def _validate_parent_child( ) +# --------------------------------------------------------------------------- +# Trace-join Validation (Tempo API) +# --------------------------------------------------------------------------- + + +async def assert_trace_join_groups( + session: aiohttp.ClientSession, + tempo_url: str, + report: ValidationReport, +) -> None: + """Assert each declared trace-join group really lands in ONE trace. + + A trace-join group is a set of spans that share one trace id with NO + parent/child link between them: each derives its trace id deterministically + from the same hash (SpanGuard::hashSpan over a ledger hash), which is how + spans produced on unrelated threads are joined without propagating any + context. The parent/child check above cannot express that -- there is no + parent to look for -- so the assertion here is co-occurrence: search for the + group's anchor span, then require at least one of its traces to also contain + every span in required_members. + + A regression this catches: if the join key or the deterministic-root + mechanism breaks, each span reverts to its own single-span trace and no trace + contains the members together, so a slow ledger is no longer readable as one + unit. That is invisible to every other check in this harness -- the spans are + all still emitted with all their attributes. + + An absent "trace_join_groups" key is a genuine no-op (nothing declared yet), + not a failure: unlike the sync_diagnostics metric group, this key is optional + and older expected_spans.json files predate it. + + Args: + session: aiohttp client session. + tempo_url: Base URL for the Tempo API. + report: ValidationReport to accumulate results. + """ + logger.info("--- Trace-Join Validation (Tempo) ---") + + with open(EXPECTED_SPANS_FILE) as f: + expected = json.load(f) + + groups = expected.get("trace_join_groups", {}).get("groups", []) + if not groups: + logger.info("[SKIP] span.trace_join: no join groups declared") + return + + for group in groups: + await _validate_trace_join_group(session, tempo_url, group, report) + + +async def _validate_trace_join_group( + session: aiohttp.ClientSession, + tempo_url: str, + group: dict[str, Any], + report: ValidationReport, +) -> None: + """Validate one trace-join group: anchor and members share a trace. + + Args: + session: aiohttp client session. + tempo_url: Tempo API base URL. + group: One entry from expected_spans.json trace_join_groups.groups. + report: ValidationReport to accumulate results. + """ + name = group.get("name", "") + anchor = group["anchor"] + required = list(group.get("required_members", [])) + check_name = f"span.trace_join.{name}" + + try: + query = '{resource.service.name="xrpld" && name="' + anchor + '"}' + traces = await _tempo_search(session, tempo_url, query, limit=10) + + if not traces: + report.add( + CheckResult( + name=check_name, + category="span", + passed=False, + message=f"{name}: no {anchor} traces to check the join against", + details={"anchor": anchor, "required_members": required}, + ) + ) + return + + # Walk the anchor's traces and keep the best result: the join holds as + # soon as ONE trace carries every required member. Several traces are + # examined because a given ledger may legitimately be missing an + # optional member (e.g. a self-built ledger has no arriving validation). + best_missing = required + for summary in traces: + trace_id = summary.get("traceID", "") + if not trace_id: + continue + spans = await _tempo_get_trace(session, tempo_url, trace_id) + present = {s.get("name", "") for s in spans} + missing = [m for m in required if m not in present] + if len(missing) < len(best_missing): + best_missing = missing + if not missing: + break + + report.add( + CheckResult( + name=check_name, + category="span", + passed=not best_missing, + message=( + f"{name}: {anchor} shares a trace with {required}" + if not best_missing + else ( + f"{name}: no {anchor} trace contained {best_missing} " + f"-- the per-{group.get('join_key', 'hash')} trace join " + f"is broken (spans are landing in separate traces)" + ) + ), + details={ + "anchor": anchor, + "join_key": group.get("join_key"), + "required_members": required, + "missing": best_missing, + "traces_examined": len(traces), + }, + ) + ) + except Exception as exc: + report.add( + CheckResult( + name=check_name, + category="span", + passed=False, + message=f"{name}: trace-join check failed ({exc})", + ) + ) + + # --------------------------------------------------------------------------- # Metric Validation (Prometheus API) # --------------------------------------------------------------------------- +# Top-level keys of expected_metrics.json that validate_metrics() must not walk +# with its generic group loop: "description" is prose, "grafana_dashboards" +# holds dashboard UIDs (checked by validate_dashboards), and +# "sync_diagnostics" has its own validator, assert_sync_diagnostics_metrics(). +SKIPPED_METRIC_GROUPS = ("description", "grafana_dashboards", SYNC_DIAGNOSTICS_GROUP) + async def validate_metrics( session: aiohttp.ClientSession, @@ -565,6 +715,42 @@ async def validate_metrics( "ledger_economy", "state_tracking", "storage_detail", + # Fresh-node sync diagnostics. Two of these only exist + # under specific conditions (a rejected handshake, a + # configured UNL site), so listing them here is how a + # failed run shows whether the metric was absent or + # merely misnamed. + "dns_resolve", + "overlay_connect", + "overlay_dial", + "handshake_", + "unl_", + "clock_close_offset", + # Sync-state signals. sync_state carries the gate, + # stall-seconds, ledgers-behind and time-to-first-FULL + # sub-series; state_changes_total is now labelled with + # the {from,to} transition edge, and + # server_stall_events_total is the stall episode count. + "sync_state", + "state_changes_total", + "server_stall_events", + # Acquire + SHAMap signals. sync_acquire carries the + # missing-node, stash-depth and in-flight sub-series and + # shamap_cache_hit_rate the tree-node cache rate; both + # are asserted. The sync_acquire_* / sync_addnode_total + # counters are listed here for diagnosis only -- they + # need a real ledger acquire, so they are not asserted + # (see _acquire_note in expected_metrics.json). + "sync_acquire", + "sync_addnode_total", + "shamap_cache_hit_rate", + # JobQueue saturation signals. jobq_saturation carries + # the pool running_tasks/worker_threads/total_waiting + # sub-series and is asserted. No new prefix entry is + # needed -- the "jobq_" prefix above already matches it + # (it was added for the StatsD jobq_job_count gauge), + # so listing it again would only duplicate the + # diagnostic output. ) ) ] @@ -580,9 +766,12 @@ async def validate_metrics( with open(EXPECTED_METRICS_FILE) as f: expected = json.load(f) - # Check each metric category. + # Check each metric category. SKIPPED_METRIC_GROUPS keys are either not + # metric groups at all or are owned by a dedicated validator below, so + # skipping them here keeps each group to a single owner (no duplicate + # Prometheus queries and no duplicate report entries). for category_key, category_data in expected.items(): - if category_key in ("description", "grafana_dashboards"): + if category_key in SKIPPED_METRIC_GROUPS: continue metrics = category_data.get("metrics", []) @@ -657,6 +846,63 @@ async def _check_prometheus_metric( ) +async def assert_sync_diagnostics_metrics( + session: aiohttp.ClientSession, + prometheus_url: str, + report: ValidationReport, +) -> None: + """Assert every metric in the 'sync_diagnostics' group is present. + + Fresh-node sync-diagnostics work packages append native metric names to the + "sync_diagnostics" group in expected_metrics.json; this check fails the run + if any listed metric regresses to absent, so a dropped signal cannot pass + CI silently. An empty group is a genuine no-op: nothing is queried and no + check is recorded, which is the state before any signal has landed. + + A missing group key is itself a failure — the key is the anchor the sync + work packages append to, so its absence means the harness lost the contract + rather than that there is nothing to check. + + Args: + session: aiohttp client session. + prometheus_url: Prometheus API base URL. + report: ValidationReport to accumulate results. + """ + logger.info("--- Fresh-Node Sync Diagnostics Metrics ---") + + with open(EXPECTED_METRICS_FILE) as f: + expected = json.load(f) + + group = expected.get(SYNC_DIAGNOSTICS_GROUP) + if group is None: + report.add( + CheckResult( + name=f"metric.{SYNC_DIAGNOSTICS_GROUP}.group_present", + category="metric", + passed=False, + message=( + f"'{SYNC_DIAGNOSTICS_GROUP}' group missing from " + f"{EXPECTED_METRICS_FILE.name}" + ), + ) + ) + return + + metrics = group.get("metrics", []) + if not metrics: + logger.info( + "[SKIP] metric.%s: group is empty (no sync-diagnostics signals " + "declared yet)", + SYNC_DIAGNOSTICS_GROUP, + ) + return + + for metric_name in metrics: + await _check_prometheus_metric( + session, prometheus_url, metric_name, SYNC_DIAGNOSTICS_GROUP, report + ) + + # --------------------------------------------------------------------------- # Log-Trace Correlation Validation (Loki API) # --------------------------------------------------------------------------- @@ -1186,7 +1432,9 @@ async def run_validation( async with aiohttp.ClientSession() as session: await validate_spans(session, tempo_url, report) await validate_span_durations(session, tempo_url, report) + await assert_trace_join_groups(session, tempo_url, report) await validate_metrics(session, prometheus_url, report) + await assert_sync_diagnostics_metrics(session, prometheus_url, report) if not skip_loki: await validate_log_trace_correlation(session, loki_url, tempo_url, report) await validate_dashboards(session, grafana_url, report) diff --git a/docs/telemetry-glossary.md b/docs/telemetry-glossary.md index 62e0e42160c..b3df34ccdca 100644 --- a/docs/telemetry-glossary.md +++ b/docs/telemetry-glossary.md @@ -198,6 +198,16 @@ A consensus round is one iteration in which validators relay and revise proposal **See also:** [Consensus round on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) + + +### Consensus round duration + +The wall-clock time one consensus round took, from its start to the ledger it accepted, as this node measured it. Recorded as a distribution rather than a single number, because the interesting question is not what one round took but how round times are spread and whether that spread is moving: a healthy network sits in a tight band a few seconds wide, and a band drifting upward delays every ledger behind it. Distinct from convergence time, which is how long validators took to agree on the transaction set — a round can converge quickly and still be slow overall if it spent its time waiting for the transaction set to arrive. + +**Scope:** per node — measured on and specific to this individual server. The round is a network-wide process, but this is one node's own timing of it. + +**See also:** [Consensus round](#consensus-round) · [Convergence time](#convergence-time) · [Tx-set acquire](#tx-set-acquire) · [Consensus round on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) + ### Consensus stall @@ -492,14 +502,6 @@ Each job type declares how many of its jobs may run at the same time. Sync-criti **Scope:** per node — measured on and specific to this individual server. - - -### Deferred job - -When a job arrives for a type that is already at its concurrency limit, the queue holds it back rather than refusing it. Adding a job never fails for queue pressure, so a deferred count is the earliest signal that a type is oversubscribed — latency only shows the problem afterwards. Each completing job releases one deferred job, so a count that stays high means arrivals are outpacing completions. - -**Scope:** per node — measured on and specific to this individual server. - ### Handler label @@ -528,6 +530,26 @@ When a node is missing ledgers (at startup, after an outage, or to extend histor **Scope:** per node — measured on and specific to this individual server. + + +### Byzantine ledger jump + +Being told that the network's last closed ledger is not the one this node built on, and discarding its own chain tip to follow the network instead. It is an abnormal event by construction: the node had already closed a ledger, and it is now throwing that work away because the peers it listens to agree on a different one. A single jump while a fresh node is still settling onto the network's chain can be benign. Repeated jumps are wrong-chain thrash — the node keeps switching between chains and never settles — and the cause is upstream of the sync pipeline, in which peers it is listening to or which network it thinks it is on, so nothing in ledger acquisition can fix it. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Fork](#fork) · [Ledger history mismatch](#ledger-history-mismatch) · [Insane / diverged peers](#insane-diverged-peers) + + + +### Clock close offset + +The difference between the network's agreed ledger close time and this node's own clock, negative when the local clock runs ahead and positive when it lags. A magnitude above a second that does not decay delays consensus participation and is a local time-sync fault rather than a network one; the server-info API only reports the offset once the magnitude reaches 60 seconds, so the metric sees skew far earlier. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Clock drift](#clock-drift) · [Ledger close times on xrpl.org](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) + ### Complete ledger ranges @@ -536,6 +558,86 @@ A node stores ledger history as one or more contiguous ranges. One continuous ra **Scope:** per node — measured on and specific to this individual server. + + +### DNS resolve + +Turning each configured peer hostname into IP addresses, which happens before any connection is attempted. An empty outcome means the name resolved to nothing, so that peer is never dialled at all; slow resolution delays every dial behind it even when it eventually succeeds. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Outbound dial latency](#outbound-dial-latency) · [Peer protocol on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) + + + +### Deferred job + +A job the queue accepted but withheld from a worker thread because its job type is already running at that type's concurrency limit. This is a third state alongside waiting and running, and it is counted in neither: the work exists and is being actively denied a thread, which is starvation rather than idleness or overload. The distinction matters because the sync-critical types run at very small limits — ledger requests and inbound ledger data are each capped at three concurrent jobs — so during a fresh sync those types routinely have work withheld while the queue looks shallow and the queue-wait quantiles look unremarkable. A sustained non-zero deferred count names the job type whose limit is the bottleneck: each completing job releases one withheld job, so a count that stays high means arrivals are outpacing completions. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Job queue occupancy](#job-queue-occupancy) · [Worker-pool saturation](#worker-pool-saturation) + + + +### Acquire phase + +One of the three sequential fetches a ledger acquisition is made of: the ledger header, then the account-state tree, then the transaction tree. The header comes first and gates the other two, because it is what names their root hashes — until it arrives the node does not yet know what to ask for. The distinction matters because the three fail for different reasons and at wildly different scales: on a fresh node the account-state tree is nearly all of the work, so an acquisition measured as a whole reports essentially that tree alone, and a node stuck waiting for the header or for the far smaller transaction tree looks identical to one making normal progress. Measured per phase, the phase that is stuck names itself. A phase can also end because its retry budget expired while the acquisition as a whole is still alive and retrying, which is why running out of time is recorded separately from failing. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Ledger acquire (inbound fetch)](#ledger-acquire-inbound-fetch) · [Missing SHAMap node](#missing-shamap-node) · [Acquire stall](#acquire-stall) + + + +### Acquire source + +Whether a ledger acquire was satisfied entirely from the local node store or required fetching the data from peers. During a genuine fresh sync almost every acquire is network-sourced, because nothing is local yet. The signal becomes diagnostic on a node that should already hold the range: acquires that still go to the network mean the local store is not retaining data, so the slowness is disk-bound rather than peer-bound. This is the pairing that explains why a node with a large existing database can start slower than a fresh one. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [SHAMap cache hit rate](#shamap-cache-hit-rate) · [Ledger acquire (inbound fetch)](#ledger-acquire-inbound-fetch) + + + +### Acquire stall + +A ledger-acquire timeout in which not a single new tree node arrived since the previous timeout, so the acquire made no progress at all. Distinct from a slow acquire, which still receives data between timeouts. A sustained stall rate alongside a missing-node count that never falls is the definitive "this sync will never complete" signature: the node keeps asking and no peer answers. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Missing SHAMap node](#missing-shamap-node) · [Ledger acquire (inbound fetch)](#ledger-acquire-inbound-fetch) + + + +### Add-node outcome + +The result of applying one SHAMap node received from a peer during a ledger acquire: good (new and valid), duplicate (already held), or invalid (failed validation). The split matters because traffic-level metrics count all three as healthy throughput. Only good represents progress; a duplicate share that swamps it means peers keep re-sending data the node already has, and a rising invalid share points at one misbehaving peer rather than a local fault. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Received-data stash](#received-data-stash) · [Missing SHAMap node](#missing-shamap-node) + + + +### Fresh-node sync diagnostics + +The set of signals that explain why a freshly-started node is slow to reach, or never reaches, a validated ledger. They split into pre-quorum bootstrap (DNS, peer dial, protocol negotiation, UNL fetch and quorum, clock skew) and the post-peering acquire pipeline (sync state, ledger and tx-set acquire, job queue, quorum and publish lag, back-fill, persistence). Rendered by the Ledger Sync Health dashboard; individual terms are defined below as each signal lands. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Diagnosing slow/stuck fresh sync](./telemetry-runbook.md#diagnosing-slowstuck-fresh-sync) (operator flow) · [Server states on xrpl.org](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) + + + +### Handshake negotiation failure + +A peer connection rejected after TLS succeeds, while the two sides check network identifier, clock, keys and reported addresses. The rejection reason names the failing check: a wrong network identifier means the node can never reach a quorum with those peers, a clock reason points at local time sync, and key or signature reasons point at the peer. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Outbound dial latency](#outbound-dial-latency) · [Clock close offset](#clock-close-offset) · [Peer protocol on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) + ### Historical fetch rate @@ -544,6 +646,16 @@ The rate at which the node fetches older ledgers to extend or repair its stored **Scope:** per node — measured on and specific to this individual server. + + +### Job queue occupancy + +How many jobs of a given type are queued or executing at the instant the queue is sampled, as opposed to how many passed through it over a period. Occupancy answers "what is sitting there now"; the job counters and queue-wait quantiles answer "what already moved and how long it had waited". The two can disagree in the way that matters most: a type whose jobs are all still queued produces no completed-job samples at all, so a latency quantile can look healthy precisely because nothing is finishing. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Deferred job](#deferred-job) · [Worker-pool saturation](#worker-pool-saturation) + ### Ledger acquire (inbound fetch) @@ -552,15 +664,345 @@ Acquiring a ledger means requesting it and its contents from peers when the node **Scope:** per node — measured on and specific to this individual server. + + +### Ledger replay + +An optional faster way to rebuild a run of historical ledgers: instead of downloading each ledger whole, the node fetches one starting ledger plus the list of ledger hashes that links the range, then fetches only what changed in each subsequent ledger and applies those changes on top of its predecessor. It is only available when enough connected peers support the protocol feature that serves those pieces, so whether it is used at all depends on the peer set rather than on local configuration alone. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Replay fallback](#replay-fallback) · [Ledger acquire (inbound fetch)](#ledger-acquire-inbound-fetch) + + + +### Ledger serve + +This node answering a peer's request for ledger data, as opposed to requesting data for itself. It is the supply side of the same exchange every other sync term describes from the receiving side, and it explains a peer's sync rather than this node's own: a server that answers nothing is why some other operator sees no peer able to serve the range they need. Worth reading in two ways. Against the receiving side, healthy serving alongside starved receiving points at peer selection rather than at this node's capacity. On its own, the kind of object asked for matters, because a request for the account-state tree is the expensive one a syncing peer actually depends on, while header replies are cheap. A reply can also be cut short by a size limit rather than refused, which is not a failure but does mean the requesting peer has to come back for the remainder — several round trips for one tree. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Ledger acquire (inbound fetch)](#ledger-acquire-inbound-fetch) · [Acquire phase](#acquire-phase) · [Peer ledger supply](#peer-ledger-supply) + + + +### Ledgers behind network + +How many ledgers this node's validated sequence trails the network's. The network figure is the highest ledger sequence any connected peer reports holding, so the gap is what the node still has to close to reach the tip. Trending down to zero is healthy convergence; flat or rising means the node acquires slower than the network advances and will not converge on its own. Because the value is floored at zero and the target comes from peer reports, a node with no peers — or whose peers have not reported a range yet — also reads zero, so a zero is only "at the tip" once there are peers. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Time to first FULL](#time-to-first-full) · [Ledger acquire (inbound fetch)](#ledger-acquire-inbound-fetch) · [Validated ledger on xrpl.org](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) + + + +### Missing SHAMap node + +A node of a ledger's account-state or transaction tree that this server needs in order to complete the ledger but does not yet hold. The count of outstanding missing nodes is the clearest available answer to "is this acquire progressing?": a count falling toward zero is progress, while a count that stays flat and non-zero means no peer is serving that tree and the acquire will never finish. Reported per tree, as the maximum across in-flight acquires, and capped per sweep — so a value sitting at the cap means the real backlog is at least that large, and only the trend distinguishes a large-but-progressing tree from a stuck one. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Acquire stall](#acquire-stall) · [Received-data stash](#received-data-stash) · [Ledger acquire (inbound fetch)](#ledger-acquire-inbound-fetch) + + + +### Network ledger gate + +The startup guard that holds a node back until it has seen a complete ledger from the network. While the gate is closed the node refuses submitted transactions and cannot reach the full state, no matter how healthy the rest of the sync pipeline looks. It normally opens within the first minutes of startup; a gate that stays closed means the node never obtained a full network ledger, which is a peering or quorum fault upstream rather than a sync-pipeline one. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Operating mode / server state](#operating-mode-server-state) · [UNL quorum headroom](#unl-quorum-headroom) · [Operating mode / server state on xrpl.org](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) + + + +### Node-store read latency + +How long the node store takes to return one stored object. Every ledger traversal that is not already answered from an in-memory cache pays this cost, so it is the floor under ledger acquisition and under most queries. It is reported as an average over an interval rather than as a distribution, which means a slow minority of reads shows up as a raised average rather than as a separate tail figure. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Node-store write latency](#node-store-write-latency) · [SHAMap cache hit rate](#shamap-cache-hit-rate) + + + +### Node-store operation rate + +How many objects per second the node store is storing and retrieving. It is the companion an average latency needs in order to be read correctly: latency measured over an interval with almost no operations in it is a stale number rather than a good one, and a node writing nothing at all while still behind the network is stalled somewhere upstream of storage rather than slowed by it. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Node-store write latency](#node-store-write-latency) + + + +### Node-store write latency + +How long the node store takes to persist one object. This is the cost that governs how fast a node can absorb ledger history, because filling in history is dominated by writing rather than by reading. It is the measurement that distinguishes the two ways a sync can be slow: starved of data from peers, or unable to write down the data it already has. A node with a large existing database can be slower to start and catch up than an empty one for exactly this reason, and no read-side measurement reveals it. Like the read figure it is an interval average, not a distribution. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Node-store read latency](#node-store-read-latency) · [Node-store operation rate](#node-store-operation-rate) + + + +### Heap trim + +Asking the memory allocator to return free pages from its own pools back to the operating system. The node does this at the end of every periodic cache sweep, because a sweep is exactly when a large amount of memory has just been released. The cost is not fixed: the allocator has to walk its pools to find what is returnable, so the work grows with how much memory the process is holding — which is why a node with a large existing database pays more for it, every sweep, than an empty one does. The pages handed back are not gone for good; the next access to that memory has to take them again, which is the reason a trim is a trade rather than a pure saving. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Minor page fault](#minor-page-fault) · [Reclaimed resident memory](#reclaimed-resident-memory) · [Sweep interval](#sweep-interval) + + + +### Minor page fault + +A memory access the operating system satisfies without touching a disk, by attaching a page it already had available. Cheap next to a disk read but not free, and taken in volume it becomes a real cost. Faults counted during a heap trim show the trim doing its own work of releasing memory. They deliberately say nothing about the faults paid afterwards, when caches refill and touch the memory that was given back — that later cost is the reason a trim can slow other work down, and counting the faults inside the trim does not capture it. Reading the figure as the total price of trimming overstates what was measured. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Heap trim](#heap-trim) · [Reclaimed resident memory](#reclaimed-resident-memory) + + + +### Reclaimed resident memory + +How much memory a heap trim actually handed back to the operating system, as opposed to how long it spent looking. This is what makes the trim's cost judgeable: time spent with memory returned is a trade, and time spent with nothing returned is pure loss. Only memory the allocator holds in its own pools can be returned at all, so a trim can legitimately reclaim nothing — and a reading of zero is a real answer rather than a missing one. Memory can also grow across a trim, when other threads allocate faster than it releases; that is reported as no reclaim rather than as a negative amount, since a running total cannot go backwards. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Heap trim](#heap-trim) · [Sweep interval](#sweep-interval) + + + +### Sweep interval + +How often the node runs its periodic pass over the in-memory caches, expiring what is stale. The period is chosen from the configured node size, so a larger node sweeps less often. It sets the cadence of everything the sweep does, including the heap trim at the end of it, and it is therefore the number against which any per-sweep cost has to be judged: the same expense is negligible at one interval and significant at another. The sweep runs as a queued job, so its cost is paid on a worker thread and competes with other work rather than happening in the background. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Heap trim](#heap-trim) · [Job queue occupancy](#job-queue-occupancy) + + + +### Rotation window + +The interval during which the node is swapping the pair of storage backends that make online deletion of old history possible. New data goes to one backend while the older one is kept for reading; on a swap the older one is discarded and a fresh one takes over. The window matters because it is the only time certain extra writes happen, so a cost seen inside it and a cost seen outside it have different explanations. A node that is not configured for online deletion has no window at all, which is a different situation from a window that costs nothing. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Copy-forward write](#copy-forward-write) · [Node re-store](#node-re-store) · [Node-store write latency](#node-store-write-latency) + + + +### Copy-forward write + +Rewriting a stored object out of the backend that is about to be discarded and into the one replacing it. An ordinary read would not write anything; this one must, because the copy it just read is about to be deleted and would otherwise survive only in memory. The volume scales with how much of the outgoing backend gets read during the swap, so it is a cost only a node that already holds history can incur — the reason this competes with catching up on a populated database and never appears on a fresh one. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Rotation window](#rotation-window) · [Node re-store](#node-re-store) + + + +### Node re-store + +Writing a tree node back to storage from memory because it could not be found in either storage backend. It signals more than cost. The node is still reachable from the current validated state, yet its only stored copy was in a backend an earlier swap discarded, and it was never rewritten because nothing had modified it. Rescuing it is an extra write, and skipping the rescue would leave the node unresolvable later. A sustained rate therefore reports two things at once: added write pressure now, and history quietly dropped by an earlier swap. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Rotation window](#rotation-window) · [Copy-forward write](#copy-forward-write) · [Missing SHAMap node](#missing-shamap-node) + ### Operating mode / server state -The server state describes how fully the node is participating, in ascending order: disconnected, connected, syncing, tracking, full (caught up), and for validators validating and proposing. A healthy non-validator sits in Full; frequent transitions out of Full indicate instability. +The server state describes how fully the node is participating, in ascending order: disconnected, connected, syncing, tracking, full (caught up), and for validators validating and proposing. A healthy non-validator sits in Full; frequent transitions out of Full indicate instability. Transitions are recorded as a from-to edge rather than a bare count, which is what distinguishes a clean one-way climb to Full from flapping in and out of it. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Time to first FULL](#time-to-first-full) · [Network ledger gate](#network-ledger-gate) · [Mode flapping](#mode-flapping) · [Operating mode / server state on xrpl.org](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) + + + +### Mode flapping + +Mode flapping is a node repeatedly reaching the full server state and losing it again, rather than climbing to it once and staying. It is visible only because state transitions are recorded as a from-to edge: a clean fresh sync traverses each climb edge roughly once, whereas flapping shows repeated counts on a reverse edge paired with its forward partner. A bare transition count cannot distinguish the two, which is why the edge labels exist. Flapping alongside healthy ledger acquisition points away from the acquire pipeline and at whatever drops a node out of full once it has arrived — a stalling main loop, a clock disagreeing with the network, or a trusted list that keeps failing quorum. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Operating mode / server state](#operating-mode-server-state) · [Server stall](#server-stall) · [Clock close offset](#clock-close-offset) · [Validation quorum](#validation-quorum) + + + +### Sentinel reading + +A sentinel reading is a deliberately out-of-range value a gauge reports to mean "this condition does not apply", chosen so the healthy or unknown state is a distinct value rather than a missing series. The distinction matters because these are observable gauges: they report on every collection tick whatever the value, so absence is a regression while an unusual number may be the intended answer. Three appear in the sync diagnostics: an amendment-block countdown of -1 means nothing is pending, and is not a negative duration; a quorum target at the signed 64-bit maximum means quorum has been switched off entirely because too many list publishers are unavailable, reported as that maximum rather than allowed to wrap negative so it cannot be misread as a target already exceeded; and a peer supply window of zero means no peer has advertised a range yet, meaning unknown rather than the start of history. A one-shot duration reading of zero is the related case — it means the milestone was never reached, not that it took no time. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Amendment block countdown](#amendment-block-countdown) · [Validation quorum](#validation-quorum) · [Peer ledger supply](#peer-ledger-supply) · [Time to first FULL](#time-to-first-full) · [Time to first validated ledger](#time-to-first-validated-ledger) + + + +### Outbound dial latency + +The elapsed time of one outbound peer connection attempt, from starting the TCP connect through TLS to the protocol upgrade, measured to whichever outcome ends it. Every attempt ends in exactly one outcome, so the outcome names the stage that broke; because the timing covers failures too, a value pinned near the dial timeout means peers accept the connection but never finish the handshake. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [DNS resolve](#dns-resolve) · [Handshake negotiation failure](#handshake-negotiation-failure) · [Peer protocol on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) + + + +### Peer ledger supply + +The idea that a connected peer set collectively offers a window of ledger sequences, rather than being simply present or absent. Each peer advertises the oldest and newest ledger it holds, so the set as a whole can serve some range and nothing outside it. This turns "how many peers do I have" into the question that actually matters during a sync: does any connected peer hold the next ledger this node needs. Being unable to advance because nobody holds that sequence is a fundamentally different fault from being slow — it is a supply gap fixed only by changing the peer set, whereas slowness with the data available is a throughput problem fixed locally, and the two are indistinguishable from inside the acquire itself. The shape of a gap matters too: needing a sequence below the window means asking for history nobody kept, while needing one above it means asking for a tip nobody has reached. Peers that have not advertised a range yet are excluded from the counts entirely, so a zero window means unknown rather than empty, and the count of peers that have reported anything is what makes the rest readable. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Ledgers behind network](#ledgers-behind-network) · [Slot census](#slot-census) · [Acquire stall](#acquire-stall) · [Complete ledger ranges](#complete-ledger-ranges) + + + +### Received-data stash + +Peer packets held for later processing because a ledger acquire cannot apply them as fast as they arrive. A shallow stash means node data is applied as it lands. A growing stash means the bottleneck is local processing — job-queue depth or disk latency — rather than peer supply, which is the opposite conclusion from an acquire that receives nothing at all. Read alongside the in-flight acquire count, since an empty stash on an idle node says nothing about acquire health. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Add-node outcome](#add-node-outcome) · [Acquire stall](#acquire-stall) + + + +### Replay fallback + +A replay sub-task giving up on the delta shortcut and acquiring the entire ledger instead, which happens when too few connected peers support the feature that serves the pieces replay needs. Nothing fails when this occurs and no error is raised — the node still completes its back-fill, just on the slower path — which is why it is easy to miss: the optimisation is simply absent. It is counted separately for each of the two sub-tasks, because the one that fetches the list of historical ledger hashes and the one that fetches a single ledger's changes can fail independently of each other. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Ledger replay](#ledger-replay) + + + +### SHAMap cache hit rate + +The share of SHAMap tree-node lookups answered from the in-memory tree-node cache rather than the node store. This is the layer above the node store's own hit ratio: a miss here is what causes a node-store read there. A low rate is expected during a fresh sync while the cache fills. A persistently low rate on a node that should be warm means the working set does not fit the cache, or continuous re-acquisition is churning it, so every tree walk pays disk latency. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Acquire source](#acquire-source) · [Missing SHAMap node](#missing-shamap-node) + + + +### Server stall + +The server's main loop failing to check in with the load monitor, measured as seconds of unresponsiveness. A stall means main-loop overload, not a shortage of sync data, so the cause is downstream work such as job-queue backlog or slow disk rather than peer supply. Two readings mean different things: a large duration with a flat episode count is one long unresolved stall, while a small duration with a rising episode count is repeated short stalls the server keeps recovering from. Episodes are counted once per stall, not once per stalled second, which is what keeps those two cases distinguishable. A stall that persists long enough is treated as unrecoverable and deliberately ends the process. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Ledgers behind network](#ledgers-behind-network) · [Consensus stall](#consensus-stall) + + + +### Time to first FULL + +The elapsed time from process start until the node first reached the full server state. It is a one-shot measurement: it is set on the first transition to full and never changes afterwards, so it has no trend to read. That leaves exactly two meaningful readings — a duration, meaning the node synced and this is how long it took, or zero, meaning it has never reached full at all. The zero is the diagnostic signal rather than absent data, and it is the starting point for working through the rest of the sync pipeline. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Operating mode / server state](#operating-mode-server-state) · [Network ledger gate](#network-ledger-gate) · [Ledgers behind network](#ledgers-behind-network) + + + +### Tx-set acquire + +One attempt to fetch the set of transactions a consensus proposal referred to but this node did not already hold. It is the consensus-path sibling of a ledger acquire — the same retry-and-timeout machinery, fetching a transaction tree rather than a whole ledger — and it is on the critical path of a round: until the set arrives, the node cannot evaluate the position that referenced it. This makes it a distinct kind of sync problem from history back-fill, and one that was previously invisible: a round waiting on a set that never arrives is indistinguishable from an idle node unless the attempt itself is measured. Two readings are diagnostic. Attempts that never complete mean rounds are blocked on data rather than on disagreement. Attempts that do complete but take about as long as a round means sets arrive so late that they delay the round they belong to — which the completion rate alone cannot reveal, because those attempts succeed. Zero attempts is normal and healthy: it means the node already held every set proposed to it. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Ledger acquire (inbound fetch)](#ledger-acquire-inbound-fetch) · [Consensus round](#consensus-round) · [Acquire phase](#acquire-phase) + + + +### Per-ledger trace join + +The scheme that makes every span touching one ledger appear in a single trace, so one slow ledger can be read as one unit. The spans are produced on threads that share nothing — the network fetch on a job-queue worker, the acceptance decision on whichever thread received the validation that triggered it, the persist on a third — so none of them can inherit a parent from the others. Instead each derives its trace identifier from the same value they all already hold: the ledger's own hash. Spans keyed on the same ledger therefore land in one trace, and spans for different ledgers stay apart, with nothing passed between the threads. The spans appear as siblings rather than as a chain, which is the honest shape: none directly causes another, and their order varies with the path the ledger took through the node. + +**Scope:** per node — measured on and specific to this individual server. Each node builds its own trace for the same ledger. + +**See also:** [Ledger acquire (inbound fetch)](#ledger-acquire-inbound-fetch) · [Validation status](#validation-status) · [Time to first validated ledger](#time-to-first-validated-ledger) · [Following ONE slow ledger as a single trace](./telemetry-runbook.md#following-one-slow-ledger-as-a-single-trace) + + + +### Validation status + +What this node's validation store did with an arriving validation. Only one result — current, meaning the validation is new and usable — counts toward accepting a ledger; the rest mean the validation was recorded and then counted for nothing, because it was stale, carried a sequence number that violates the increasing-sequence rule, or duplicated or contradicted another validation from the same validator. The distinction is what separates a node that is slow to validate from one that never will: both receive validations continuously, and without this split the two are indistinguishable. A companion flag says whether the validation actually reached the acceptance gate, since a validation arriving while another thread is already accepting the same ledger is set aside rather than acted on — which is why a trace can show a validation with no acceptance after it. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Validation quorum](#validation-quorum) · [Quorum shortfall](#quorum-shortfall) · [Per-ledger trace join](#per-ledger-trace-join) · [Negative UNL and validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) + + + +### UNL fetch outcome + +The result of retrieving a validator list from one configured UNL site and applying it. Accepted is the only success; same-sequence and known-sequence are normal no-op refreshes of a list the node already holds; fetch, status and parse errors are transport or content faults; and expired, stale, untrusted, invalid or unsupported-version mean the list arrived but was rejected, so no trusted keys are loaded from that site. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [UNL quorum headroom](#unl-quorum-headroom) · [UNL (Unique Node List)](#unl-unique-node-list) · [UNL (Unique Node List) on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/unl) + + + +### UNL quorum headroom + +The trusted UNL key count minus the quorum a ledger needs, so it reads as spare validator keys. At or below zero — including a trusted key count of zero, meaning no usable list loaded — the node can track ledgers but can never declare one validated, however healthy the rest of the sync pipeline looks. A UNL site that keeps failing to fetch is the usual cause. **Scope:** per node — measured on and specific to this individual server. -**See also:** [Operating mode / server state on xrpl.org](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) +**See also:** [UNL fetch outcome](#unl-fetch-outcome) · [Validation quorum](#validation-quorum) · [Validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) + + + +### Worker-pool saturation + +The share of the job-queue worker threads currently executing a job. The thread count is fixed at startup from configuration, node size and hardware concurrency, so it is a real ceiling rather than an elastic one. Saturation is read together with the total number of jobs queued, because the ratio alone is ambiguous: every thread busy with nothing queued is a busy instant, while every thread busy with work piling up behind them is an exhausted pool. The distinction is what makes this a pool-level signal — when the pool is exhausted, every subsystem whose jobs are queued behind it slows at the same time, so each one appears to have its own independent fault. Reading saturation first attributes that whole pattern once, instead of once per victim. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Deferred job](#deferred-job) · [Job queue occupancy](#job-queue-occupancy) · [Server stall](#server-stall) + + + +### Quorum shortfall + +A candidate ledger being refused the status of validated because the agreeing trusted validations counted for it fell short of the quorum it needed. It is the last gate of the sync pipeline and the one that can fail while everything upstream looks healthy: the node can hold every ledger it needs, apply them all, and still never declare one validated. Two shapes mean opposite things. A tally accumulating toward the quorum is slow and will get there, so the shortfall is transient. A tally that plateaus below the quorum is stuck, and the causes are upstream of ledger acquisition entirely — too few trusted validators reachable, or a validator-list or negative-UNL configuration that excludes the ones that are — so nothing in acquisition can fix it. A shortfall is also expected briefly on every healthy round, because the gate is first evaluated the moment this node finishes building a ledger, before its peers' validations for that ledger have arrived, and is retried as they come in. That makes the bare occurrence of a shortfall uninformative; only its persistence alongside a flat tally is a fault. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Validation quorum](#validation-quorum) · [UNL quorum headroom](#unl-quorum-headroom) · [Validated ledger](#validated-ledger) · [Time to first validated ledger](#time-to-first-validated-ledger) · [Negative UNL on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) + + + +### Publish lag + +The number of ledgers a node has fully validated but not yet published to its clients and subscribers. Publication trails validation by design, so a small lag that drains each round is the normal state; the diagnostic reading is a lag that stays positive or grows. That is a distinct fault from anything the quorum or acquisition signals describe: validation is working, the node itself is current, and only the pipeline that hands finished ledgers to subscribers is behind — so the visible symptom is stale data for API clients on a server that is not itself behind the network. Because it is local processing rather than peer supply that falls behind, the causes sit in job-queue starvation and main-loop stalls. One reading trap: a lag of zero is only healthy on a node that is validating. On a node that never has, the zero means there is nothing validated to publish at all, and the quorum gate is the thing to read instead. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Published ledger](#published-ledger) · [Validated ledger](#validated-ledger) · [Quorum shortfall](#quorum-shortfall) · [Deferred job](#deferred-job) · [Worker-pool saturation](#worker-pool-saturation) + + + +### Time to first validated ledger + +The elapsed time from process start until the node first declared a ledger fully validated. Like the time to first reaching the full server state, it is a one-shot measurement: it is set the first time the quorum gate passes and never changes afterwards, so it has no trend to read. That leaves exactly two meaningful readings — a duration, meaning the node got there and this is how long it took, or zero, meaning it never has. The zero is the diagnostic signal rather than absent data. Its value is in the pairing: a duration for reaching the full server state beside a zero here means the node reached that state but has still never fully validated a ledger, which places the fault at the quorum gate rather than in ledger acquisition. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Time to first FULL](#time-to-first-full) · [Quorum shortfall](#quorum-shortfall) · [Validated ledger](#validated-ledger) · [Operating mode / server state](#operating-mode-server-state) @@ -576,6 +1018,16 @@ A cluster is a set of servers run by the same operator that trust each other, ex **See also:** [Cluster on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/clustering) + + +### Disconnect reason + +The cause recorded when a peer connection is torn down, kept alongside the direction the connection was originally opened in. A single disconnect count cannot separate the two situations that matter, because they produce the same number: a node shedding load, which drops peers deliberately because it could not keep up with what it owed them or because a peer exceeded its resource allowance, and a network or topology fault, where the peer became unreachable, stopped answering keepalives, or turned out to be following a different chain. The first is a local capacity problem and the peer list is not the fix; the second is the opposite. A third group is neither — clean teardown at shutdown and peers closing their own side are ordinary churn, and a count dominated by those is healthy. The direction matters separately, since churn among the peers a node dials points somewhere different from churn among the peers that dial it. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Resource disconnect](#resource-disconnect) · [Slot census](#slot-census) · [Insane / diverged peers](#insane-diverged-peers) + ### Fetch-pack @@ -682,6 +1134,26 @@ Set-get (fetch) and set-share messages exchange transaction-set data between pee **Scope:** per node — measured on and specific to this individual server. + + +### Serve refusal + +A peer data request that this node declined to answer — the supply side of the sync exchange, as opposed to everything a node measures about its own fetching. It matters because a node that refuses everything it is asked for looks, from the outside, exactly like a node nobody asks: both serve nothing. From the asking peer's point of view a refusal is indistinguishable from a peer that does not hold the data, so refusals directly slow the sync of every peer that depends on this node. The reason divides them into two kinds. Self-inflicted refusals mean the node was too loaded to answer — its outgoing queue to that peer had grown past its limit, or the local fee track showed it under load, or too much bulk-transfer work was already queued — and these are the serving-side symptom of the same overload that shows up as stalls and job-queue backlog locally. A refusal because the data was simply not held is different: that is a genuine history gap, a question of what this node retains rather than how busy it is. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Fetch-pack](#fetch-pack) · [GetObject / object fetch](#getobject-object-fetch) · [Complete ledger ranges](#complete-ledger-ranges) · [Peer ledger supply](#peer-ledger-supply) + + + +### Slot census + +A single consistent reading of everything PeerFinder knows about this node's peering position: how many outbound and inbound slots are occupied against how many exist, how many outbound attempts are in flight, how many configured fixed peers are connected against how many were configured, and the depth of the two address stores. Taken together at one instant, so the numbers can be compared against each other. The three terms worth defining plainly: an occupied outbound slot is a peer this node dialled and is now connected to; the bootstrap address store is a persisted list of addresses kept across restarts purely so a starting node has somewhere to dial; and the live address store holds addresses learned from peers during this session and exists only in memory. Occupancy alone cannot explain a peering failure, which is the reason the census exists. A node with no outbound peers might be dialling continuously and never completing, or not dialling at all because it has no addresses to try, or dialling only configured peers that are unreachable — three different faults with three different fixes, and the occupancy count is identical in all of them. It is the attempt count, the address-store depths, and the configured-versus-connected comparison that tell them apart. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Overlay](#overlay) · [Disconnect reason](#disconnect-reason) · [Peer ledger supply](#peer-ledger-supply) · [DNS resolve](#dns-resolve) · [Outbound dial latency](#outbound-dial-latency) + ### Squelch @@ -758,6 +1230,16 @@ The NodeStore serves reads through a pool of read threads (optionally bundling r ## Validator Health + + +### Amendment block countdown + +The window between an amendment this build does not understand reaching majority among validators and that amendment actually activating. It exists because amendment activation is not instantaneous: once an unsupported amendment has majority support it becomes expected to activate at a known future time, and until then the node still works normally. That window is the only actionable part of an otherwise terminal condition — after activation the node stops validating and cannot resume without a software upgrade, so there is no operational fix left, only a rebuild and restart. Read as a countdown it therefore outranks every other sync signal in urgency: a node counting down is going to stop validating at a knowable moment, and anything else that looks wrong is secondary. The healthy state is reported as an explicit sentinel value rather than as absent data, so a node with nothing pending is distinguishable from a node whose reporting has broken, and the countdown is held at zero rather than going negative once activation is due. The identity of the blocking amendment is deliberately not carried on the metric — the network can vote on any amendment identifier, including ones this build has never heard of, which would make it an unbounded label — so the hash comes from the log line that records the amendment reaching majority. + +**Scope:** per node — measured on and specific to this individual server. + +**See also:** [Amendment blocked](#amendment-blocked) · [UNL blocked](#unl-blocked) · [Amendments on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/amendments) + ### Amendment blocked diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 084a63fda98..1b6bdd7b661 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -1729,23 +1729,48 @@ Use the call-site macros in `src/xrpld/telemetry/MetricMacros.h` -- no | Last-value snapshot (not a distribution) | `XRPL_METRIC_GAUGE_RECORD` [+ `_LABELED`] -- requires an ABI v2 opentelemetry-cpp build; this repo currently builds ABI v1, so use the observable-gauge row below instead | | Value your own code already tracks, sampled on a timer | `XRPL_METRIC_OBSERVABLE_GAUGE_REGISTER` / `_COUNTER_REGISTER` / `_UPDOWN_REGISTER` | +First declare the name in `src/xrpld/telemetry/MetricNames.h` -- the emit site +must reference a constant, never a string literal, and CI Rule I enforces that +for any metric family that already has constants: + +```cpp +// in src/xrpld/telemetry/MetricNames.h, namespace metric: +inline constexpr char myNewThingTotal[] = "my_new_thing_total"; +inline constexpr char myInFlightRequests[] = "my_in_flight_requests"; +inline constexpr char myThingSize[] = "my_thing_size"; +``` + +Then emit against it: + ```cpp #include +#include // Monotonic counter: -XRPL_METRIC_COUNTER_INC(app_, "my_new_thing_total", "Description of what this counts"); +XRPL_METRIC_COUNTER_INC( + app_, metric::myNewThingTotal, "Description of what this counts"); // Value that can go up and down, e.g. in-flight work (no _total suffix -- that // is reserved for monotonic counters; an UpDownCounter is a current value): -XRPL_METRIC_UPDOWN_ADD(app_, "my_in_flight_requests", "Currently executing", 1); // on start -XRPL_METRIC_UPDOWN_ADD(app_, "my_in_flight_requests", "Currently executing", -1); // on finish +XRPL_METRIC_UPDOWN_ADD(app_, metric::myInFlightRequests, "Currently executing", 1); +XRPL_METRIC_UPDOWN_ADD(app_, metric::myInFlightRequests, "Currently executing", -1); + +// Labelled: the KEY is a constant too, and so is the VALUE when it comes from a +// fixed set. Only runtime data stays a plain expression. +XRPL_METRIC_COUNTER_INC_LABELED( + app_, + metric::myNewThingTotal, + "Description of what this counts", + {{label::outcome, std::string(lval::dns_resolve::resolved)}}); // Sampled from your own state, on the OTel export timer (register ONCE, in init code): -XRPL_METRIC_OBSERVABLE_GAUGE_REGISTER(app_, "my_thing_size", "Current size", +XRPL_METRIC_OBSERVABLE_GAUGE_REGISTER(app_, metric::myThingSize, "Current size", [this] { return static_cast(myThing_.size()); }); ``` -Counters use a `_total` suffix by convention. A histogram whose values can +Naming rules (counter `_total`, duration `_us`/`_ms`/`_seconds`, no `xrpld_` +prefix, bounded label cardinality) are listed in CONTRIBUTING.md -> +"Telemetry metric naming" and enforced by CI Rules I/J/K. A histogram whose values can exceed ~10,000 units (e.g. a microsecond duration beyond 10ms) still needs one line added to `addMicrosecondHistogramView()` in `MetricsRegistry.cpp` -- the only case that still touches a central file. There is no way to read a metric's @@ -2749,6 +2774,952 @@ not a sign the cache is working. - Verify Loki is running: `curl http://localhost:3100/ready` - Check the filelog receiver glob `/var/log/xrpld/*/debug.log` matches your log layout — the log file must sit one subdirectory below the mount root +### Diagnosing slow/stuck fresh sync + +A fresh node that is slow to reach `server_state=full`, or never reaches it, is +diagnosed from the **Ledger Sync Health** dashboard (uid `ledger-sync-health`). +Its nine rows are ordered the way a fresh node progresses, so reading the board +top-to-bottom walks the same path a sync does: + +| # | Row | Question it answers | +| --- | ----------------------------- | ---------------------------------------------------- | +| 1 | Bootstrap (Domain 0) | Can the node reach peers and form a quorum at all? | +| 2 | Peer supply | Does any peer hold what this node needs? | +| 3 | Sync state | Is the node advancing through the mode machine? | +| 4 | Ledger acquire & SHAMap fetch | Is ledger data arriving and being applied? | +| 5 | Job queue | Does arrived work ever get a worker thread? | +| 6 | Quorum & publish | Does a held ledger ever validate, and reach clients? | +| 7 | Terminal blockers & serving | Will the node stop validating for good? | +| 8 | Back-fill & persistence | Is an existing database the bottleneck? | +| 9 | Spans & traces | _Which_ fetch, peer or object — not how many? | + +**Start from the symptom, not from row 1.** Match what you observe to a branch +below; each branch names the panels that discriminate it, what healthy and +unhealthy look like, and what to conclude. The branch then points at the +numbered ordered-diagnosis steps further down, which are the detail. + +```mermaid +flowchart TD + S["Node is not reaching
server_state = full"] --> Q1{"Amendment Block Countdown
at -1?"} + Q1 -->|"No: counting down"| T["Branch F
Terminal blocker"] + Q1 -->|"Yes: healthy"| Q2{"Which sync mode
is it stuck in?"} + + Q2 -->|"disconnected"| A["Branch A
DNS / dial / negotiation"] + Q2 -->|"connected or syncing,
no validated ledger"| B["Branch B
UNL, quorum, clock"] + Q2 -->|"acquiring,
never finishing"| C["Branch C
SHAMap fetch and job queue"] + Q2 -->|"reaches full,
slowly or flapping"| D["Branch D
Publish, back-fill, rounds"] + Q2 -->|"was fine when the
DB was empty"| E["Branch E
Node-store and cache"] + + classDef start fill:#1f3b57,stroke:#8ab4d8,stroke-width:2px,color:#ffffff + classDef gate fill:#4a3a10,stroke:#d8b45a,stroke-width:2px,color:#ffffff + classDef leaf fill:#f2f6fa,stroke:#4a6f8a,stroke-width:1px,color:#12232e + classDef stop fill:#5c1f1f,stroke:#e08a8a,stroke-width:2px,color:#ffffff + class S start + class Q1,Q2 gate + class A,B,C,D,E leaf + class T stop +``` + +**Check branch F first, always.** It is the only branch whose window closes: +once an unsupported amendment activates there is no operational fix, so it +outranks every other symptom regardless of what the mode machine says. + +#### Branch A — stuck at `disconnected` + +Peer count flat at zero; _Mode Transitions by Edge_ shows the node never leaving +`disconnected`, or churning straight back to it. + +| Look at | Healthy | Unhealthy | Conclude | +| ------------------------------------------ | -------------------------------------- | ---------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| _DNS Resolve Outcome Rate_ | all rate on `outcome=resolved` | any rate on `empty`, or both flat at zero | a name in `[ips]`/`[ips_fixed]` returns no address, or the list is empty — fix the hostname or use an IP | +| _DNS Resolve Latency (p95)_ | milliseconds | seconds-scale | the resolver is timing out and delaying every dial behind it | +| _Outbound Dial Outcome Rate_ | `connected` non-zero | all attempts on one failure outcome | `tcp_fail` = route/firewall/closed port · `tls_fail` = TLS · `self_connection` = we dialled our own address · `upgrade_fail` = negotiation, go to the next row · `timeout` = never terminal | +| _Outbound Dial Latency (p95)_ | well under the dial timeout | pinned near it | peers accept TCP but never finish the handshake | +| _Handshake Negotiation Failures by Reason_ | flat, or a low background rate | any sustained `reason` | `wrong_network`/`invalid_network_id` is the most common fresh-node fault — the node is on a different network and can never reach quorum; `clock_skew` sends you to branch B | +| _PeerFinder Slot Census_ | `out_active` climbing toward `out_max` | `connecting` non-zero with `out_active` low; or `bootcache` and `livecache` both 0 | dials never complete; or there is nothing to dial at all | + +**Conclusion:** the node has no usable overlay. Nothing downstream can be +diagnosed until `connected` on _Outbound Dial Outcome Rate_ is non-zero. Detail: +[Bootstrap ordered diagnosis](#bootstrap-domain-0--ordered-diagnosis) steps 1-3 +and [Sync-pipeline](#sync-pipeline--ordered-diagnosis) step 12. + +#### Branch B — stuck at `connected`/`syncing`, no validated ledger + +The node has peers, `server_state` reaches `connected` or `syncing`, and +_Time to First Validated Ledger_ stays flat at zero. + +| Look at | Healthy | Unhealthy | Conclude | +| ---------------------------------------------------- | ------------------------------------------------------------------------- | ------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| _UNL Fetch Rate by Site & Outcome_ | `accepted` once per site, then `same_sequence`/`known_sequence` refreshes | `fetch_error`/`bad_status`/`parse_error` | the publisher site is unreachable, so no keys load from it | +| | | `expired` | the list applied but is past its validity window — refresh the blob and check the local clock, do not replace `validators.txt` | +| _UNL Trusted Keys vs Quorum_ + _UNL Quorum Headroom_ | `trusted_keys` above `quorum`, headroom positive | headroom zero or negative (red) | **the node will never validate**: the trusted list is too small to satisfy quorum. It can track ledgers forever and never declare one validated | +| | | `quorum` about 9.2e18 | the **quorum-disabled** sentinel — too many publishers unavailable. Fix publisher reachability; the key count is irrelevant until quorum is re-enabled | +| _Clock Close Offset_ | magnitude decaying under 1 s | above 1 s and not decaying | local NTP fault delaying consensus participation, and the cause behind a `clock_skew` handshake reason | +| _Network Ledger Gate_ | clears within minutes of startup | persistent 1 (red) | the node has never seen a complete network ledger — go back to branch A, not deeper | +| _Trusted Validations vs Quorum Target_ | tally climbing toward the target | tally flat below the target | **stuck at the quorum gate**: validations arrive and never reach quorum. Judge by the sustained floor over minutes, never one sample — the first evaluation of each round runs before peers' validations arrive, so a healthy node sawtooths | +| | | both flat at 0 | the gate has never been evaluated; nothing has been offered — upstream problem, back to branch A | +| _Trusted Validation Accept Rate by Status_ (row 9) | nearly all `current` | concentrated in `stale`, `bad_seq`, `multiple`, `conflicting` | validations arrive and are counted for nothing — this is what separates "slow to validate" from "never will", and it is invisible in the tally. Bulk `stale` = clock/timing; bulk `multiple`/`conflicting` = a misbehaving trusted validator or two chains | + +**Conclusion:** the fault is in trust and time, not in data supply. Detail: +[Bootstrap](#bootstrap-domain-0--ordered-diagnosis) steps 4-5 and +[Sync pipeline](#sync-pipeline--ordered-diagnosis) steps 2 and 16. + +#### Branch C — acquiring but never finishing + +Ledger acquires are in flight, _Ledgers Behind Network_ is flat or rising, and +`full` never arrives. + +| Look at | Healthy | Unhealthy | Conclude | +| ------------------------------------------------------------------------------------------------------------- | ------------------------------------------------- | --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| _Missing SHAMap Nodes per Acquire (state/tx)_ | falling toward zero (read the trend over minutes) | **flat and non-zero** | **no peer is serving that tree** — the node will sit here forever. Pinned at 256 is the per-sweep cap, meaningful only with the trend | +| _Acquire Stall Rate (no progress)_ | flat | sustained rate **together with** a flat missing-node count | the definitive stuck-sync signature: requesting and nobody answering | +| _Peers Able to Serve Needed Sequence_ | `peers_serving_next` above zero | `peers_serving_next` = 0 while `peers_reporting` > 0 | **decisive**: peers are connected and none holds the next needed ledger. Waiting cannot finish it — the peer set must change. Everything else in branch C will look starved as a consequence, so do not chase it | +| _Peer Supply Window Margin (history headroom vs tip gap)_ | _History Headroom_ positive, _Tip Gap_ near zero | _History Headroom_ **below zero** | asking for history nobody kept — needs a full-history peer | +| | | _Tip Gap_ growing steadily | the peer set lags the real network; not a history problem | +| _Ledger Acquire Phase Outcomes (by phase & timeout)_ + _Ledger Acquire Phase Duration (p95 by phase)_ (row 9) | `header` short, `astree` the bulk | `astree` hot with `timed_out=true` and non-zero `missing_nodes` | the common stuck shape — peers are not supplying account-state nodes | +| | | `header` hot | the node is waiting to be **told what to fetch**; invisible in the missing-node counts, which are both still zero | +| _Add-Node Outcomes_ | `good` dominates | `duplicate` swamps `good` | bandwidth busy, acquire standing still — peers re-sending known data | +| | | `invalid` rising | a specific misbehaving peer, not a local fault | +| _Received-Data Stash Depth & In-Flight Acquires_ | stash drains | stash growing | data arrives faster than it is applied — a job-queue or disk problem, the **opposite** conclusion from a stall rate, and only this panel separates them | +| `jobq__deferred` | flat at 0 | sustained non-zero on `ledgerdata`/`ledgerrequest` | a job the queue accepted then **withheld** at its concurrency limit of 3 — it appears in neither `waiting` nor `running`, so no other signal can show it. Starved `ledgerdata` is exactly why the stash grows while missing nodes stay flat | +| _Worker Pool Saturation_ + _Worker Pool Capacity & Total Backlog_ | under 80% | 100% with `total_waiting` climbing | the pool is **exhausted** — every stage looks slow at once. Stop here; no per-subsystem fix helps while no thread is free | +| Acquire outcome `abandoned` in Tempo (`{name="ledger.acquire" && span.outcome="abandoned"}`) | absent | present | the acquire was swept or shut down before reaching a result — without this value a stuck-then-swept fetch had no `outcome` at all and vanished from every outcome rate | + +**Conclusion:** distinguish "nobody is serving it" (peer supply) from "it arrives +and we cannot process it" (job queue / disk). The two look identical in a log and +are separated only by the stash-depth and per-type deferred gauges. Detail: +[Sync pipeline](#sync-pipeline--ordered-diagnosis) steps 6-12 and 17. + +#### Branch D — reaching `full` but slowly, or falling back out of it + +_Time to First FULL_ has a value, or the node flaps between `full` and +`connected`. + +| Look at | Healthy | Unhealthy | Conclude | +| ------------------------------------------------------------------------------ | ------------------------------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| _Publish Lag (validated minus published)_ | flat at 0 or 1 | positive and growing | validation is healthy and the **publish pipeline is not** — the node is current while its clients see stale data. Local processing fault: go to the job-queue and stall panels | +| | | flat at 0 on a node that never validated | not healthy, merely empty — read _Trusted Validations vs Quorum Target_ first | +| _Server Stall_ + _Server Stall Event Rate_ | both zero | large seconds, **flat** event rate | one long unresolved stall; past 600 s the server deliberately fails | +| | | small seconds, **rising** event rate | repeated short stalls — periodic work (sweeps, large writes), not one stuck operation | +| _Mode Transitions by Edge_ | each climb edge roughly once | repeated `full`→`connected` paired with `connected`→`full` | flapping: reaching `full` and losing it. Sends you to the stall panels or to branch B's clock and quorum panels — those are what drop a node out of `full` | +| _Consensus Round Duration (p50/p95)_ + _Consensus Round Duration Distribution_ | band steady | band drifting up, or a second high band | rounds are taking longer; read against _Tx-Set Acquire Duration (p95)_ (rounds waiting on data) and _Trusted Validations vs Quorum Target_ (validations arriving too late) | +| | | p95 climbing, p50 flat | a minority of rounds stall — the early form of a second band | +| | | both rising together | the network is slowing, not this node. Compare another node via `$node` first | +| _Tx-Set Acquire Outcomes_ + _Tx-Set Acquire Duration (p95)_ (row 9) | flat at zero, or all `complete` | `timeout`/`abandoned` climbing | proposed sets never complete — rounds wait on data, not on agreement. This is the consensus path, not history back-fill | +| | | `complete` p95 approaching the round interval | sets arrive but so late they delay their own round; the outcome rate cannot show this because they succeed | +| _Replay Fallback to Full Acquire (by stage)_ (row 8) | flat | any sustained rate | too few peers support the `LedgerReplay` feature, so every historical ledger is fetched whole. Nothing fails — the optimisation is simply gone, which is why it is easy to miss. `stage` names the sub-task: `skiplist` or `delta` | +| _Replay Outcomes (by terminal state)_ (row 8) | `success` climbing | `timeout` climbing | deltas never arrived — treat as peer supply, read with branch C | +| | | `build_failed`/`parameter_failed` | **data** faults from the serving peers, not slowness — the peer set is suspect | + +**Conclusion:** the pipeline works; something behind it is not keeping up. +Publish lag and stalls are local, round duration is often network-wide, and +replay fallback is a silent loss of an optimisation rather than a failure. +Detail: [Sync pipeline](#sync-pipeline--ordered-diagnosis) steps 3, 5, 15, 16 +and 18. + +#### Branch E — an existing database syncs slower than a fresh one + +The specific symptom: a node with history starts and is slower than the same node +was when empty. Back-fill is **write**-bound, so no read-side panel shows it. +Read the **Back-fill & persistence** row. + +| Look at | Healthy | Unhealthy | Conclude | +| ----------------------------------------------------------- | ------------------------------------------------------------------------- | --------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| _NodeStore Write vs Read Latency (us/op)_ | write line flat and low | write line rising during back-fill | the backend cannot absorb writes. Adding peers will not help — check storage IOPS, the `[node_db]` backend and whether online-delete/rotation competes with the back-fill | +| | | read line far above write | the read path is the cost; read with the cache-hit panel below | +| _NodeStore Operation Rate (writes vs reads)_ | write rate non-zero while behind | write rate zero while still behind the network | nothing is being persisted — the stall is **upstream** of the node store. Go to branch C; storage is not the problem | +| _SHAMap TreeNode Cache Hit Rate_ | rising as the cache warms | persistently low | the working set does not fit the cache, or re-acquisition churns it, so every tree walk pays disk latency | +| _Acquire Source (local vs network)_ | `local` dominant on a warm node | sustained `network` on a range the node should hold | the local store is not retaining data | +| paired with _NuDB Cache Hit Ratio_ (Ledger Data Sync board) | both healthy | low on both | disk-bound sync | +| | | low here, NuDB healthy | cache pressure alone — this is the pairing that explains the whole symptom | +| _Sweep Heap-Trim Duration (p50/p95)_ | sub-millisecond | tens of milliseconds and rising with database size | the per-sweep heap trim is walking a large resident heap. It runs **on the sweep job**, so the cost lands on the job queue, not in the background — read it next to the sweep job's queue wait | +| | | flat and low while the symptom persists | the trim is not the cause; the remaining rows in this branch are | +| _Sweep Heap-Trim Faults & Reclaim Rate_ | reclaim rate tracking cache turnover, faults near zero | reclaim near zero while the duration panel shows real time | the trim is walking the heap and freeing nothing — pure cost, and the clearest case for tuning the sweep interval up | +| | | fault rate moving with the reclaim rate | pages are being handed back and immediately taken again. **Do not over-read this:** the fault delta covers the trim call only, so it shows the trim faulting — it does NOT prove the trim caused the later faults as caches refill. That is the mechanism, but it is not what this counter measures | +| _Online-Delete Rotation Window & Copy-Forward Writes_ | flag briefly 1 once per delete interval, writes only inside those windows | copy-forward rate large enough to move node-store write latency | rotation is competing with sync I/O — the extra writes exist only on a populated, already-rotated database, which is why the symptom is specific to an existing one | +| | | no series at all on either query | `online_delete` is not configured on this node, which is **not** the same as rotation costing nothing — rule the whole rotation hypothesis out and move on | +| | | copy-forward writes while the flag reads 0 | the window flag leaked; treat the rate as unattributed rather than concluding rotation is cheap | +| _Rotation Node Re-Store Rate_ | flat at zero | any sustained rate | an earlier rotation removed the only on-disk copy of clean nodes the current state map still reaches. Two consequences: each rescue is an extra write competing with sync, and without it the node would later hit an unresolvable missing-node error. Get the hashes from the `copyNode` warning in Loki — they are deliberately not labels | + +**Conclusion:** the tree-node cache sits one layer **above** the node store, so a +miss here is what produces a node-store read there; reading the two together is +what tells cache pressure from a disk bottleneck. The last four rows add the two +costs that are _specific_ to a node that already has data — the per-sweep heap +trim, whose price scales with the resident heap, and online-delete rotation, +whose extra writes need an archive to read from. Both are absent by construction +on a fresh node, which is what makes them candidate explanations for this branch's +symptom rather than general slowness. + +Two limits to respect here. The node-store numbers are **means, not +percentiles**, so a tail that matters will move them but there is no p99. And +the trim's fault counter is scoped to the **trim call only**: it shows +that the trim itself faults, and it cannot show the faults paid later as the +caches refill and touch the pages the trim returned. That later re-fault cost is +the actual mechanism by which a trim would slow a sync, and no metric here +measures it — so correlate the trim **duration** against the sweep job's queue +wait, and do not present the fault rate as proof the trim caused a slow sync. +Detail: [Sync pipeline](#sync-pipeline--ordered-diagnosis) steps 9 and 14. + +#### Branch F — terminal: the node will stop validating for good + +Check this branch on **every** slow sync, before the mode machine, because it is +the only one with a deadline. + +| Look at | Healthy | Unhealthy | Conclude | +| ------------------------------ | ------------------------------------------------------------------------------------------------------ | --------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| _Amendment Block Countdown_ | **-1** — an explicit sentinel meaning nothing is pending. Not a negative duration and not missing data | any non-negative value | a countdown to a **terminal** state: at expiry the node becomes amendment-blocked and will never validate again without a software upgrade. Clamped at 0, so 0 means due or past due. Nothing else on this dashboard matters — plan the upgrade inside the window | +| _Amendment Warned_ | 0 | 1 | the same condition as a flag: an unsupported amendment reached majority | +| _Byzantine Ledger Jumps_ | flat at zero | a single jump during a fresh sync | benign — the node is settling onto the network's chain | +| | | repeated jumps | wrong-chain thrash: the node keeps switching chains and never settles. Check the peer set (branch A) and the configured network id. Nothing in the acquire pipeline can fix it | +| _Ledger/Object Serve Refusals_ | near zero | `sendq_full`/`load_shed` climbing | self-inflicted: this node is too loaded to answer. It does not explain **this** node's sync — it explains its peers', and is the serving-side symptom of the same overload branches C-E cover | +| | | `not_found` climbing | a genuine history gap — a retention and configuration question, not a load one | + +**Conclusion:** the countdown is the only actionable amendment signal. The +existing `validator_health{metric="amendment_blocked"}` on the Validator Health +dashboard reports the block after it has happened, when nothing can be done. The +blocking amendment's hash is deliberately not a label (unbounded cardinality) — +get it from the `AmendmentTableImpl::doValidatedLedger` log line via Loki, +correlated by node and time. Detail: +[Sync pipeline](#sync-pipeline--ordered-diagnosis) step 13. + +--- + +Signal definitions: +[telemetry-glossary.md](./telemetry-glossary.md) "Fresh-node sync diagnostics". +Signal index (instrument, emit site, panel): +[09-data-collection-reference.md](../OpenTelemetryPlan/09-data-collection-reference.md) +"Fresh-node sync diagnostics". + +Note: rows 1-8 are native metrics, which are never sampled — unlike the +span-derived (spanmetrics) series in row 9, they are always complete. An absent +series in row 9 can mean "tracing not enabled" (`trace_ledger` / `trace_peer`) +rather than "not happening", so use row 9 to **localise** a fault the native +rows have already established, not to detect one. + +The two sections below are the ordered detail the branches point into. They walk +every panel in sequence; the branches above are the fast path to the right step. + +#### Bootstrap (Domain 0) — ordered diagnosis + +Work the Bootstrap row in this order. Each step gates the next, so stop at the +first one that is wrong and fix it before reading further panels. + +1. **DNS — did the node resolve its configured peers?** + Panel _DNS Resolve Outcome Rate_ (`dns_resolve_total`). Any rate on + `outcome=empty` means a name in `[ips]` or `[ips_fixed]` returned no address, + so that peer is never dialled. Fix the hostname or point at an IP. + If both outcomes are flat at zero the node resolved nothing at all — check + that `[ips]`/`[ips_fixed]` is actually populated. + Then check _DNS Resolve Latency (p95)_ (`dns_resolve_latency_ms`): a + seconds-scale p95 is a resolver timing out, which delays every dial behind + it even when the resolution eventually succeeds. + +2. **Outbound dial — did the connections complete?** + Panel _Outbound Dial Outcome Rate_ (`overlay_connect_total`). The `outcome` + label names the stage that broke: + - `connected` — success; this is the line that must be non-zero. + - `tcp_fail` — no route, refused, or the peer port is closed or firewalled. + - `tls_fail` — the TLS handshake failed. + - `self_connection` — TLS succeeded and PeerFinder then recognised the + remote address as one of this node's own, so it had dialled itself. A + local misconfiguration (own address in `[ips_fixed]`, or behind the + advertised endpoint), not an unreachable peer; reported separately so a + rising `tls_fail` is not confused with it. + - `upgrade_fail` — TLS succeeded but the HTTP upgrade or protocol + negotiation was rejected. This is the outcome that pairs with step 3. + - `timeout` — the attempt never reached a terminal state. + If every attempt lands on one failure outcome and `connected` stays at + zero, the node has no outbound peers and cannot sync at all. + _Outbound Dial Latency (p95)_ (`overlay_dial_latency_ms`) covers successes + and failures together, so a p95 pinned near the dial timeout means peers + accept the TCP connection but never finish the handshake. + +3. **Handshake — why were peers rejected?** + Panel _Handshake Negotiation Failures by Reason_ + (`handshake_negotiation_fail_total`). This is where an `upgrade_fail` from + step 2 gets its cause. The `reason` label is the check that rejected the + peer; the ones worth acting on first: + - `wrong_network` or `invalid_network_id` — **most common fresh-node + misconfiguration.** The node is on a different network than its peers, so + it will never reach a quorum however healthy everything else looks. Fix + `[network_id]` and the peer list together. + - `clock_skew` — the peer rejected this node's clock; go to step 5. + - `session_verify_failed`, `bad_public_key`, `no_session_signature` — a + misbehaving or mismatched peer rather than a local fault. + - `self_connection`, `local_ip_mismatch`, `remote_ip_mismatch` — a low + background rate is normal on a NAT'd host. + +4. **UNL — can the node ever form a quorum?** + Panel _UNL Fetch Rate by Site & Outcome_ (`unl_fetch_total`), grouped by + `site` and `outcome`. Read the outcome carefully — `accepted` is the only + success value: + - `accepted` — the list was applied. + - `same_sequence`, `known_sequence` — normal no-op refreshes of a list the + node already holds. + - `fetch_error`, `bad_status`, `parse_error` — transport or content faults; + the site is effectively unreachable. + - `stale`, `untrusted`, `invalid`, `unsupported_version` — the list arrived + but was rejected, so no keys are loaded from that site. + - `expired` — the list was applied and its keys were loaded, but it is past + its validity window, so the expiry sweep drops them again and the + publisher does not count as available. Refresh the publisher blob, and + check the local clock, rather than replacing `validators.txt`. + - `pending` — the list is valid only from a future date and is held for + rotation. Normal, not a fault. + Then read _UNL Trusted Keys vs Quorum_ and _UNL Quorum Headroom_ + (`unl_quorum`, `metric=trusted_keys` against `metric=quorum`). **This is + the "will never validate" check.** If `trusted_keys` is zero, or sits at or + below `quorum` (headroom zero or negative, shown red), the trusted UNL is + too small to ever satisfy quorum: the node can track ledgers but will + never declare one validated, and no amount of healthy acquire traffic + changes that. A site stuck on `fetch_error` or `expired` in the panel above + is the usual cause. A very large `quorum` with a deeply negative headroom + is the distinct "quorum disabled" state: too many publishers are + unavailable, so quorum has been switched off entirely rather than merely + set high. Fix publisher reachability first — the key count is irrelevant + until quorum is enabled again. + +5. **Clock — is local time disagreeing with the network?** + Panel _Clock Close Offset_ (`clock_close_offset_seconds`, `metric=offset`). + The signed series is negative when the local clock runs ahead of the network + and positive when it lags; the magnitude series carries the threshold bands. + A magnitude above 1 second that does not decay is suspicious and delays + consensus participation. Note that `server_info` only reports + `close_time_offset` once the magnitude reaches 60 seconds, so this panel sees + skew long before the API does. A persistent offset is a local NTP fault, not + a network one — and it is also the cause behind a `clock_skew` reason in + step 3. + +If all five steps are clean the bootstrap stage is healthy, and the problem is +in the sync pipeline — rows 2 to 9 — instead. + +#### Sync pipeline — ordered diagnosis + +Once bootstrap is clean, work rows 2 to 9 in this order. As above, each step +gates the next: stop at the first one that is wrong. The step order is the +diagnosis order, which crosses rows deliberately — a step names the row and +panel it reads. + +1. **Did the node ever sync at all?** + Panel _Time to First FULL_ (`sync_state`, `metric=initial_full_duration_us`, + shown in seconds). This is a one-shot measurement: it fills in the moment the + node first reaches `full` and never changes again. So there are exactly two + readings that matter — a value, meaning the node synced and this is how long + it took, or a flat zero (red), meaning it has **never** reached `full`. A + flat zero is the signal, not missing data; everything below explains why. + Do not read a rising or falling trend into this panel — it cannot have one. + +2. **Is the node gated on the network ledger?** + Panel _Network Ledger Gate_ (`sync_state`, `metric=network_ledger_gate`). A + persistent 1 (red) means the node has never seen a complete ledger from the + network, so it refuses submitted transactions and cannot reach `full` however + healthy the acquire pipeline looks. This normally clears within the first + minutes of startup; a 1 that never clears sends you back to the Bootstrap row + (no peers, or no quorum) rather than deeper into the pipeline. + +3. **Is the server stalling rather than starving?** + Panels _Server Stall_ (`sync_state`, `metric=server_stall_seconds`) and + _Server Stall Event Rate_ (`server_stall_events_total`). A non-zero stall + means the main loop missed its heartbeat for at least 10 seconds, which is + main-loop **overload**, not sync data starvation — so the fix is in job-queue + depth and disk latency, not peer supply. Read the two panels together, as + they separate two different faults that look identical in a log: + - Large stall seconds with a **flat** event rate — one long unresolved + stall. Note that past 600 seconds the server deliberately fails with a + logic error, so a stall approaching that is about to end the process. + - Small stall seconds with a **rising** event rate — repeated short stalls; + the server keeps recovering and re-stalling, which points at periodic work + (sweeps, large writes) rather than a single stuck operation. + +4. **How far behind is it, and is it converging?** + Panel _Ledgers Behind Network_ (`sync_state`, `metric=ledgers_behind`). The + target is the highest ledger any connected peer reports holding, so this is + the gap the node must close. Trending down to 0 is healthy convergence; flat + or rising means the node acquires slower than the network advances and will + never converge on its own. One caveat when reading a zero: the value is + floored at 0 and the target comes from peer reports, so a node with no peers + — or whose peers have reported no range yet — also reads 0. Confirm against + the peer count before treating a 0 here as "at the tip". + +5. **Which state edges is it actually taking?** + Panel _Mode Transitions by Edge_ (`state_changes_total`, grouped by `from` + and `to`). A clean fresh sync traverses each climb edge + (`disconnected`→`connected`→`syncing`→`tracking`→`full`) roughly once. + Repeated counts on a reverse edge such as `full`→`connected`, paired with + `connected`→`full`, is flapping: the node keeps reaching `full` and losing + it. Flapping with a healthy step 4 points back at step 3 (stalls) or at the + Bootstrap row's clock and quorum panels, since those are what drop a node out + of `full` once it has arrived. Use the _Mode From_ and _Mode To_ template + variables to isolate one edge. + +6. **Is ledger acquisition progressing, or permanently stuck?** + Panel _Missing SHAMap Nodes per Acquire (state/tx)_ (`sync_acquire`, + `metric=missing_state_nodes_max` and `missing_tx_nodes_max`). **This is the + panel that separates "sync is slow" from "sync will never finish."** Read the + shape over several minutes, not the instantaneous value: + - **Falling toward zero** — the acquire is progressing. Slow is not stuck. + - **Flat and non-zero** — no peer is serving that tree. The node will sit + here forever; no amount of waiting fixes it. Go to step 7. + - **Pinned at 256** — that is the per-sweep cap (`kMissingNodesFind`), so the + real backlog is at least that large. Meaningful only in combination with + the trend: pinned-and-falling is a large but progressing tree. + - **Zero on one tree, non-zero on the other** — that tree is already + complete; concentrate on the one still reporting nodes. + One caveat: zero on both is only healthy if the `in_flight` series on + _Received-Data Stash Depth & In-Flight Acquires_ (step 8) + is non-zero. Zero everywhere with zero acquires in flight is an idle node, + which says nothing about acquire health. + +7. **Is the node asking and getting nothing back?** + Panel _Acquire Stall Rate (no progress)_ (`sync_acquire_no_progress_total`). + This counts acquire timeouts in which not a single node arrived. A sustained + rate here **together with** a flat missing-node count from step 6 is the + definitive stuck-sync signature: the node is requesting and no peer is + answering. Check the peer count and whether any connected peer actually holds + the ledger range being requested — a peer set that cannot serve the range + looks identical to no peers at all from inside the acquire. + +8. **Is the data arriving useful, or wasted?** + Panel _Add-Node Outcomes_ (`sync_addnode_total`, stacked by `outcome`). + Traffic-level metrics count all three outcomes as healthy throughput, which + is why this split matters: + - `good` — new, valid nodes. This is the only line that represents progress. + - `duplicate` — nodes already held. A share that swamps `good` means peers + keep re-sending known data, so bandwidth is busy while the acquire stands + still. + - `invalid` — nodes that failed validation. A rising share points at a + specific misbehaving peer rather than a local fault. + Then read _Received-Data Stash Depth & In-Flight Acquires_ + (`sync_acquire`, `metric=received_data_depth` and `in_flight`). A growing + stash means node data is arriving faster than it can be applied, which is a + job-queue or disk problem, not a peer-supply one — the opposite conclusion + from step 7, and the two are distinguished only by this panel. + +9. **Is it disk-bound rather than peer-bound?** + Panels _Acquire Source (local vs network)_ (`sync_acquire_source_total`) and + _SHAMap TreeNode Cache Hit Rate_ (`shamap_cache_hit_rate`). During a genuine + fresh sync `network` dominates and the cache hit rate is low — both expected, + because nothing is local yet. The diagnostic case is a node that should + already be warm: + - Sustained `network` on a range the node should already hold means the local + store is not retaining data. + - A persistently low tree-node hit rate means the working set does not fit + the cache, or continuous re-acquisition is churning it, so every tree walk + pays disk latency. + Note this is the in-memory tree-node cache, one layer **above** the + _NuDB Cache Hit Ratio_ panel on the Ledger Data Sync dashboard: a miss here + is what produces a node-store read there. A low rate on both is disk-bound + sync; a low rate here with a healthy NuDB ratio is cache pressure alone. + This is also the pairing that explains a large existing database syncing + slower than a fresh one. + +10. **Is the work arriving but never getting a worker thread?** + Steps 6 to 9 all assume the node is at least trying to process what it + receives. This step covers the case where it is not: the data arrived, the + job was queued, and no thread ever ran it. Read the two levels in order, + because the pool-wide answer supersedes the per-type one. + - **Pool first** — _Worker Pool Saturation_ (`jobq_saturation`, + `running_tasks / worker_threads`) with _Worker Pool Capacity & Total + Backlog_ beside it. Below 80% the pool has spare capacity, so skip to the + per-type read. At 100% the answer depends on `total_waiting`: + - 100% with `total_waiting` near zero — the pool is merely busy at this + instant. Not a fault. + - 100% with `total_waiting` climbing — the pool is **exhausted**. Every + job type is starved by it, so every other stage in this row will look + slow at once. Stop here: fixing an individual subsystem cannot help + while no thread is free to run its jobs. Look at what is holding the + threads (long-running jobs, disk waits from step 9) or at the + `[workers]` setting for the node size. + - **Then per type** — the `jobq__deferred` gauges, one series per + job type (see + [Per-Job-Type Queue Saturation](#per-job-type-queue-saturation)). This is + the signal that exists nowhere else. A deferred job is one the queue + **accepted and then withheld** because its type is already running at its + concurrency limit, so it appears in neither `waiting` nor `running`, and + neither the job counters nor the queue-wait histograms can show it. Any + sustained non-zero value names the job type whose limit is the + bottleneck; find it with + `topk(5, {__name__=~"jobq_.*_deferred", service_instance_id="$node"})`. + During a fresh sync watch `jobq_ledgerdata_deferred` and + `jobq_ledgerrequest_deferred` first: both types run at a limit of 3, so + they are the ones that starve soonest, and starved `ledgerdata` is + exactly why the received-data stash in step 8 grows while the + missing-node count in step 6 stays flat. The matching + `jobq__running` and `_waiting` gauges give the context — + `running` pinned at the limit with `waiting` above zero confirms the + limit, not the supply, is what is holding the type back. + Note the difference from _Job Queue Wait p95 By Type_ on the Ledger Data + Sync dashboard: that panel measures how long jobs that already ran had + waited, so it reports the past; these gauges report what is sitting in + the queue right now, including the part being actively withheld. + +11. **Can the network even serve the ledgers this node needs?** + Steps 6 to 10 all assume some peer holds what the node is asking for. This + step tests that assumption, and it is the one that separates "slow" from + "impossible". Panel _Peers Able to Serve Needed Sequence_ + (`peer_ledger_supply`, `metric=peers_reporting`, + `peers_serving_validated` and `peers_serving_next`). Read the two counts + together — `peers_reporting` is the denominator that makes the rest + meaningful: + - **`peers_serving_next` at zero with `peers_reporting` above zero** — + the decisive reading. Peers are connected and have advertised their + ranges, and **none of them holds the next ledger this node must + acquire.** No amount of waiting finishes the sync; the peer set itself + has to change. Add peers that hold the range, or point the node at a + full-history server. Everything in steps 6 to 10 will look starved as a + consequence, so do not chase them. + - **`peers_serving_next` above zero but the sync is still slow** — supply + is fine and the fault is downstream. Go back to steps 6 to 10: the data + is available, so the limit is acquire progress, local processing or + worker threads. + - **`peers_reporting` at zero** — nothing has advertised a range yet. + This is not a supply gap; it means the node has no peers, or its peers + have not sent a status change. Peers advertising an empty range are + excluded from every field, so the two window fields read 0 meaning + **unknown**, not genesis — do not read a zero window here as "peers + serve from the start of history". Go back to the Bootstrap row, and to + step 12 for why there are no peers. + Then read _Peer Supply Window Margin (history headroom vs tip gap)_, + which plots the two window fields as distances from this node's own + validated sequence rather than as absolute sequences: _History Headroom_ + is `validated_ledger_seq − supply_min_seq` and _Tip Gap_ is + `supply_max_seq − validated_ledger_seq`. This is what tells the two + shapes of a supply gap apart, and zero is the boundary in both cases: + _History Headroom_ **below zero** means the node is asking for history + nobody kept, so it needs a full-history peer; a _Tip Gap_ that grows + steadily means it is chasing a tip its peers have not reached, which is a + peer set lagging the real network rather than a history problem. Both + lines stay blank until the node has a validated ledger and a peer has + advertised a range, so an empty panel here is the `peers_reporting` = 0 + case above, not a healthy reading. + +12. **Is this node failing to get or keep peers, and why?** + Step 11 says whether the peer set can serve; this step says why the peer + set is what it is. Panel _PeerFinder Slot Census_ + (`peerfinder_slot_census`) with _PeerFinder Address Caches & Fixed Peers_ + beside it. All nine fields come from one lock acquire, so occupancy and + capacity can be compared directly — which is what separates three faults + that otherwise look identical: + - **`connecting` non-zero while `out_active` stays below `out_max`** — + the node is dialling and the dials never complete. Without the attempt + count this looks exactly like a node that is not dialling at all. Pair + it with _Outbound Dial Outcome Rate_ in the Bootstrap row for the stage + that breaks. + - **`bootcache` and `livecache` both at 0** — there is nothing to dial. + No seed addresses at all, so check `[ips]` and DNS in Bootstrap step 1. + - **`fixed_active` below `fixed_configured`** — a peer named in the + configuration is unreachable. `fixed_configured` is what was asked for + and `fixed_active` is what was obtained, so any shortfall names a + specific configured peer to check. + Then split the traffic by direction. _Inbound Peer Accept Outcomes_ + (`peer_accept_total`, by `outcome`) covers connections offered **to** + this node; the already-documented `overlay_connect_total{outcome}` in + Bootstrap step 2 covers dials **from** it. Reading both is the only way + to get the full in/out picture: a node refusing every inbound + connection looks the same as one nobody dials until these are separated. + On the inbound side `resource_limit`, `no_slot` and `slot_refused` are + this node declining (load, capacity, or a duplicate), while + `protocol_mismatch`, `bad_cookie` and `handshake_error` point at the + peer or at a network-id mismatch. + Then read _Peer Disconnects by Reason_ (`peer_disconnect_total`, by + `reason` and `direction`). One disconnect count cannot separate the two + causes; the label can: + - `large_sendq`, `charge_resources` — **our fault.** This node could not + keep up with what it owed the peer, or charged it past the resource + limit, so it shed the connection as backpressure. The fix is local + capacity, not the peer list, and it sends you back to steps 9 and 10. + - `not_useful`, `ping_timeout`, `read_error` — topology or network + faults. The peer is on a different chain or unreachable, so the fix is + the peer set. + - `graceful`, `shutdown`, `stopping` — normal churn and clean teardown, + not faults. A run dominated by these is healthy. + Use `direction` to tell churn in the peers this node dials from churn + in the peers that dial it. + Finally, the mirror-image question: _Ledger/Object Serve Refusals_ + (`serve_refused_total`, by `request` and `reason`) is what **this node + refuses to serve OTHERS**. It does not explain this node's own sync, but + it explains its peers' — and a node that refuses everything is why some + other operator is reading step 11 on their side. `sendq_full` and + `load_shed` are self-inflicted: this node is too loaded or too far + behind on its send queue to answer, so treat them as the serving-side + symptom of the same overload steps 3, 9 and 10 cover. `not_found` is + different — it is a genuine history gap, meaning the data was asked for + and this node simply does not hold it, which is a configuration and + retention question rather than a load one. + +13. **Is the node about to stop validating for good?** + Panel _Amendment Block Countdown_ (`amendment_block`, + `metric=seconds_to_block`) with _Amendment Warned_ (`metric=warned`) + beside it. **This step outranks every other step in urgency**, so check it + whenever a sync looks wrong, not only after the ten above are clean: + - `seconds_to_block` at **-1** — healthy. The -1 is an explicit sentinel + meaning nothing is pending, chosen so the healthy state is a distinct + value rather than a missing series. Do not read it as a negative + duration or as absent data. + - `seconds_to_block` at **any non-negative value** — a countdown to a + **terminal** state. When it expires the node becomes amendment-blocked: + it stops validating and will never validate again without a software + upgrade. The value is clamped at 0 rather than going negative, so a 0 + means the activation is due or past due, not that it just started. + Nothing else on this dashboard matters if this is counting down — plan + the upgrade inside the window, because after it there is no + operational fix. + `warned` reaching 1 is the same condition seen as a flag: an + unsupported amendment has reached majority. The existing + `validator_health{metric="amendment_blocked"}` on the Validator Health + dashboard is the after-the-fact companion — it reports the block once it + has happened, when nothing can be done, whereas this countdown is the + only actionable part. + The blocking amendment's hash is **not** a metric label, deliberately: + the network can vote on an arbitrary 256-bit amendment id, not drawn + from this build's known features, so an id label would be unbounded + cardinality and would mint a permanent new series per amendment. + Get the hash from the log line in `AmendmentTableImpl::doValidatedLedger` + ("Unsupported amendment ... reached majority at ...") via Loki, + correlated to this series by node and time. + Finally, read _Byzantine Ledger Jumps_ (`ledger_jump_total`) in the + same pass. Any non-zero rate means the node was fed a last-closed + ledger it had not built on and **discarded its own chain tip** to + follow. A single jump during a fresh sync can be benign as the node + settles onto the network's chain. Repeated jumps are wrong-chain + thrash: the node keeps switching between chains and never settles, so + check the peer set from step 12 and the configured network id from + Bootstrap step 3 — those are what put a node on the wrong chain in the + first place. Nothing in the acquire pipeline can fix it. + +14. **Is the node store itself the bottleneck — and is it the write side?** + This is the step for the specific symptom **"a node with a large existing + database starts and syncs slower than a fresh one"**. Back-fill is + write-bound, so no read-side panel can show it; check this step whenever a + node with existing history is the slow one. Both panels live in the + **Back-fill & persistence** row. + Panel _NodeStore Write vs Read Latency (us/op)_ (`nodestore_state`, + `metric=node_writes_duration_us` / `node_reads_duration_us` rated against + their counts) with _NodeStore Operation Rate (writes vs reads)_ + (`metric=node_writes` / `node_reads_total`) beside it: + - **Write line rising during history back-fill** — the backend cannot + absorb writes fast enough. Sync will stay slow however many peers are + available, so adding peers will not help. Check storage IOPS, the + `[node_db]` backend and its tuning, and whether the online-delete / + rotation cycle is competing with the back-fill writes. Correlate with + `nodestore_state{metric="write_load"}` on the Ledger Data Sync dashboard. + - **Read line far above the write line** — the read path, not the write + path, is the cost. Read it together with _SHAMap TreeNode Cache Hit Rate_ + (step 7): a cold in-memory cache sends every tree walk to disk, and that + shows up here as read latency rather than as a node-store fault. + - **Write rate at zero while the node is still behind the network** — + nothing is being persisted at all, so the stall is upstream of the node + store. Go back to peer supply (step 12) and the acquire panels (steps + 6-8); storage is not the problem. + - Both panels use the **rate of the mean divided by the rate of the + count**, which is why the count series exist. Read as an interval + latency, not a since-boot average — on a long-running node the raw + cumulative mean moves so slowly that a current stall is invisible in it. + - Two limits to keep in mind. First, this is a **mean, not a percentile**: + a tail that matters will move it, but there is no p99 here. That is a + deliberate cost trade — a histogram would need one `Record()` per node + object, and a single ledger write walks thousands of SHAMap nodes. + Second, a mean is **omitted rather than drawn as 0** when nothing has + been stored or fetched yet, so an absent line means "no samples", not + "instantaneous". All three concrete store paths time themselves, so + `write_mean_us` is present on any node that has written at all. + +15. **Is replay-based back-fill silently falling back to the slow path?** + Only relevant when `[ledger_replay]` is enabled. Panels _Replay Fallback to + Full Acquire (by stage)_ (`ledger_replay_fallback_total`) and _Replay + Outcomes (by terminal state)_ (`ledger_replay_outcome_total`), also in the + **Back-fill & persistence** row: + - **Any sustained fallback rate** — too few connected peers support the + `LedgerReplay` protocol feature, so every historical ledger is fetched + whole instead of as a delta. Back-fill still completes, just far slower, + which is why this is easy to miss: nothing fails, the optimisation is + simply gone. The `stage` label says which sub-task gave up — `skiplist` + (fetching the list of historical ledger hashes) or `delta` (a single + ledger's changes). Fix by peering with nodes that support the feature. + - **Failure outcomes climbing while `success` stays flat** — replay runs + but never completes. The outcome names the layer at fault: `timeout` + means the deltas never arrived, so treat it as a peer-supply problem and + read it with step 12; `build_failed` means a delta would not apply to its + parent, and `parameter_failed` means a peer served a skip list + inconsistent with the request — those two are **data** faults from the + serving peers, not slowness, so the peer set is suspect rather than the + network. + - **All series absent** — expected when `[ledger_replay]` is not + configured, or on a node with no history to back-fill. Absence here is + not a regression; it means no replay task was ever created. For the same + reason neither counter is asserted by the local validation harness. + +16. **The node has peers and a UNL, yet never declares a ledger validated — + is it slow, stuck, or is only publishing behind?** + This is the last stage of the pipeline and the one every step above + assumes away. Steps 1 to 15 all ask whether ledger data arrives and gets + applied; this step asks whether the ledgers the node already holds ever + pass the **quorum gate**, and whether the ones that pass ever reach + clients. It is the step for the specific symptom **"peers are connected, + the trusted list is loaded, acquisition looks healthy, and + `server_state` still never becomes `full`."** + - **Slow or stuck?** Panel _Trusted Validations vs Quorum Target_ + (`ledger_quorum_publish`, `metric=trusted_validation_tally` against + `quorum_target`). Both are snapshots of the most recent gate + evaluation, recorded whether it passed or failed, so a node that keeps + failing still reports both numbers — which is exactly what separates + the two cases: + - **Tally climbing toward the target** — slow, not stuck. Validations + are accumulating and the gate will pass. Keep waiting; nothing here + needs fixing. + - **Tally flat below the target** — **stuck.** Validations arrive and + never reach quorum, so this node can hold every ledger it needs and + still never declare one validated. Two causes: too few trusted + validators are reachable, or the UNL / negative-UNL configuration + excludes the ones that are. Go to Bootstrap step 4 (_UNL Trusted Keys + vs Quorum_ and _UNL Quorum Headroom_) — a trusted list that cannot + satisfy quorum makes every panel in this row look starved as a + consequence, so do not chase them. + - **Target reading about 9.2e18** (signed 64-bit maximum) — the + explicit **quorum-disabled** sentinel. Too many list publishers are + unavailable, so the trusted list switched quorum off entirely rather + than merely setting it high, and the node can never validate however + far the tally climbs. The value is reported as that maximum rather + than being allowed to wrap to -1, precisely so it cannot be misread + as a target the tally already exceeds. Fix publisher reachability + first; the key count is irrelevant until quorum is enabled again. + - **Both series flat at 0** — the gate has never been evaluated at all, + so nothing has yet been offered for validation. That is an upstream + problem, not a quorum one: go back to the Bootstrap row. + One reading caveat that matters more here than anywhere else in this + row: judge the tally by its **sustained floor over minutes**, never by + a single sample. Each series is a snapshot of the last evaluation, and + the first evaluation of every round runs before peer validations for + that ledger arrive, so a healthy node sawtooths. + - **Is the gate actually rejecting?** Panel _Pre-Accept Quorum Shortfall + Rate_ (`ledger_quorum_shortfall_total`, `stage=pre_accept`). Read this + one carefully, because **a non-zero rate is not by itself a fault.** + Consensus issues this node's own validation and evaluates the gate + immediately, before its peers' validations for that same ledger have + arrived, so the first evaluation of each round tallies short and is + retried as validations come in. A healthy cluster emits this counter + every round. The fault signature is the **combination**: this rate + climbing well above the ledger-close rate while the tally above stays + flat below its target and _Time to First Validated Ledger_ stays at + zero. That is the retry loop never converging, rather than merely + running early. + - **Did it ever validate at all?** Panel _Time to First Validated Ledger_ + (`ledger_quorum_publish`, `metric=time_to_first_validated_us`, shown in + seconds). A one-shot measurement, read exactly like _Time to First + FULL_ in step 1: a value means the node got there and this is how long + it took, a flat zero means it never has. Reading the two together is + what pins down the fault — a value on _Time to First FULL_ beside a + zero here means the node reached the `full` server state but has still + never fully validated a ledger, which points at the quorum gate rather + than at acquisition. + - **Validation fine, publishing behind?** Panel _Publish Lag (validated + minus published)_ (`ledger_quorum_publish`, `metric=publish_lag`). This + is the separate question, and the one no other panel can answer: the + published sequence was never exported before, so this gap was not + derivable from any other series. Publishing trails validation by + design, so a small lag that drains each round is normal. + - **Lag flat at 0 or 1** — healthy. + - **Lag positive and growing** — validation is healthy and the + **publish pipeline is not.** The node itself is current while its + clients and subscriptions see stale data. This is a local processing + fault, not a peer or quorum one, so go back to steps 3, 9 and 10: + a starved job queue or a stalling main loop is the usual cause. + - **Lag flat at 0 on a node that has never validated** — not healthy, + merely empty. There is nothing validated to publish, so read + _Trusted Validations vs Quorum Target_ first and ignore this panel + until the gate passes. + - **Are the arriving validations even usable?** Panel _Trusted Validation + Accept Rate by Status_ (`consensus.validation.accept`, by + `validation_status`). The tally panels above say how many trusted + validations were counted; this one says what happened to the validations + that arrived, which is the missing half when the tally sits flat: + - **Nearly all `current`** — normal. That is the only status that + continues to the acceptance gate, so the tally is limited by how many + validators are reachable, not by the validations themselves. Go back to + Bootstrap step 4. + - **Rate concentrated in `stale`, `bad_seq`, `multiple` or + `conflicting`** — validations arrive and are counted for nothing. This + is the difference between a node that is slow to validate and one that + never will, and it is invisible in the tally, which simply stays low + either way. `stale` in bulk usually means a clock or timing problem + (check _Clock Close Offset_ in Bootstrap); `multiple` or `conflicting` + in bulk means a trusted validator is misbehaving or the node is seeing + two chains. + - **`accept_gated=true` on a share of them** — normal, and not a lost + span: another thread was already accepting that ledger, so no + acceptance followed this particular validation. + - **Flat at zero on a peered node** — no trusted validations are + arriving at all, which sends you back to Bootstrap step 4 rather than + anywhere in this row. + This panel is span-derived, so unlike the native panels above it inherits + trace sampling — read it for the **shape** of the split, and the native + tally for exact counts. + +17. **Which stage of the fetch is stuck, and which peer or object is it stuck + on?** + Every step above reads a native metric, which is an aggregate: it says how + much and how often, never _which one_. This step reads the **span-derived** + panels in the **Spans & traces** row, which answer the "which" + questions the aggregates structurally cannot — and each point on them is + backed by a trace, so it can be clicked through to the individual fetch, + request or dial. + Two things to know before reading them. First, unlike every metric above, + these series come from spans and are therefore subject to sampling and to + the `trace_ledger` / `trace_peer` config flags — an absent series can mean + "not enabled" rather than "not happening". Second, use them to _localise_ + a fault that the metric steps have already established, not to detect one. + - **Which acquire phase is stuck?** Panels _Ledger Acquire Phase Duration + (p95 by phase)_ and _Ledger Acquire Phase Outcomes (by phase & timeout)_ + (`ledger.acquire.header` / `.astree` / `.txtree`). Step 6 says the + missing-node count is flat; these say _where_. A ledger acquire is three + sequential fetches, and the parent `ledger.acquire` span is flat, so its + duration is the account-state tree's and hides the other two: + - **`astree` band hot, with `timed_out=true` and a non-zero + `missing_nodes`** — peers are not supplying account-state nodes. This + is the common stuck-fresh-sync shape. Check _Outbound Dial Outcomes + (span-derived, per attempt)_ below for `tcp_fail` / `timeout` spikes, + and step 11 for whether any peer holds the range at all. + - **`header` band hot** — the node is waiting to be **told what to + fetch**. The header names both trees' root hashes, so until it arrives + nothing else can even be requested. This is a peer-supply fault + upstream of either tree, and it is invisible in step 6, whose + missing-node counts are both still zero at this point. + - **`txtree` band hot while `astree` is quiet** — unusual, and worth + noting precisely because the transaction tree is normally small: the + parent span's duration would not have shown it. + Use the `$timed_out` template variable to isolate the timed-out phases. + The two are separate dimensions on purpose: a phase can time out and + still be retried by its parent acquire, so `timed_out=true` is not the + same as the acquire having failed. + - **Are proposed transaction sets arriving?** Panels _Tx-Set Acquire + Outcomes_ and _Tx-Set Acquire Duration (p95)_ (`txset.acquire`). Nothing + measured this before, so a consensus round stalled waiting on a + transaction set looked identical to an idle one: + - **`timeout` or `abandoned` climbing** — proposed sets never complete, + so rounds wait on data rather than on agreement. Read it with the + consensus panels rather than with the acquire steps above: this is the + consensus path, not history back-fill. + - **`complete` series p95 approaching the round interval** — sets do + arrive, but so late that they delay the round they belong to. The + outcome rate alone cannot show this, because those acquisitions + succeed. + - **All series flat at zero** — normal. It means this node already held + every proposed set locally and never had to fetch one. + - **Which peer is the dial failing against?** Panel _Outbound Dial + Outcomes (span-derived, per attempt)_ (`peer.dial`). The same six + outcomes as `overlay_connect_total` in Bootstrap step 2, set from the + same code path so the two cannot disagree — read the counter first for + the rate, then come here for the **identity**. The span carries + `remote_endpoint`, which the counter deliberately cannot: one metric + series per peer address would be unbounded cardinality. Drill into a + failing point to get the endpoint, which is what separates one + unreachable peer from a broken local network — a distinction the + aggregate rate cannot make. A dial torn down by shutdown ends its span + with **no** `outcome` attribute; that is deliberate, and means "never + concluded" rather than a lost span. + - **Are we serving others while starving ourselves?** Panel _Ledger Serve + Rate by Object Type_ (`ledger.serve`). This is the **supply side** — what + this node does for its peers — so it does not explain this node's own + sync, but it does explain its peers'. Compare the `as` series here + against the inbound `astree` phase above: healthy serving with starved + receiving points at peer selection rather than at this node's capacity. + - **`refused` climbing** — this node is declining requests; _Ledger/Object + Serve Refusals_ in step 12 gives the specific cause. + - **`partial` climbing** — replies keep filling `kSoftMaxReplyNodes`, so + peers need repeated round trips for one tree. Expected while serving + large state trees; sustained on small requests it is worth a look. + +18. **Are consensus rounds themselves slowing down?** + Panels _Consensus Round Duration Distribution_ (heatmap) and _Consensus + Round Duration (p50/p95)_ (`consensus_round_duration_ms`). A round that + used to take 3-4 s and now takes 12 delays every ledger behind it, and + until now this was only a span attribute — answering it fleet-wide meant + raw trace queries. Being a native metric it is also **never sampled**, + unlike every span-derived panel above. + - **Band drifting upward, or a second band high up** — rounds are taking + longer. Read it against the two panels that explain why: _Tx-Set Acquire + Duration (p95)_ (step 16 — rounds waiting on data rather than on + agreement) and _Trusted Validations vs Quorum Target_ (step 13 — + validations arriving too late to close the round). + - **P95 climbing while P50 stays flat** — a minority of rounds stall. This + is the early form of what the heatmap later shows as a second band. + - **Both rising together** — the whole network is slowing, not this node. + Compare against another node using the `$node` variable before treating + it as a local fault. + - Buckets span 500 ms to 120 s deliberately. The SDK default stops at + 10 s, which would put every slow round in one saturated bucket and read + every quantile as exactly 10 s — and the consensus parameters allow a + round up to 120 s before it is abandoned. + +#### Following ONE slow ledger as a single trace + +The steps above find _which stage_ is slow across all ledgers. This finds why +_one specific_ ledger was slow, which is the question left when the aggregate +panels look merely mediocre. + +Every span that touches a ledger derives its trace id from that ledger's own +hash, so they all share one trace even though each runs on a different thread +and no context is passed between them. One TraceQL query returns the whole +story: + +``` +{span.ledger_hash="<64-hex-ledger-hash>"} +``` + +What that trace contains, and what each part tells you: + +| Span | Thread | What a long one means | +| ----------------------------- | ----------------------------- | ------------------------------------------------------------------ | +| `ledger.acquire` (+ phases) | `JtLedgerData` worker | The node had to fetch this ledger and peers were slow to supply it | +| `consensus.validation.accept` | validation worker | Time this trusted validation spent driving the acceptance decision | +| `ledger.validate` | whichever thread hit the gate | The acceptance decision itself (`LedgerMaster::checkAccept`) | +| `ledger.store` | caller of `storeLedger` | Persisting the ledger — a slow one points at the node store | + +Read it this way: + +1. **Get a candidate hash.** From a slow point on any per-ledger panel, or from + a log line, or by searching for the shape directly — for example an acquire + that never finished: + `{name="ledger.acquire" && span.outcome="abandoned"}`. +2. **Open the whole trace** with the query above. The spans appear as + **siblings**, not as a parent/child chain. That is deliberate and is the + honest shape: none of them directly causes another, and their order changes + with the sync path — `checkAccept` is reached from a peer thread (an arriving + validation), from the acquire-completion job, and from the consensus thread + (`switchLCL`). A chain would assert an order that does not hold. +3. **Compare the spans' durations,** which is the point of having them in one + trace: whether this ledger was slow to _arrive_, slow to be _accepted_, or + slow to be _stored_ is now one glance instead of three separate searches + that cannot be correlated. +4. **If a validation span has no `ledger.validate` beside it,** check its + `accept_gated` attribute. `true` means another thread was already accepting + that ledger, so no acceptance followed this validation — that is normal, not + a lost span. +5. **If the trace holds only one span,** the join is broken rather than the + ledger being fast: confirm with the `span.trace_join.per_ledger` check in + `validate_telemetry.py`, which fails CI on exactly this regression. + +```mermaid +flowchart TD + KEY["Ledger hash (32 bytes)
trace_id = hash[0:16]"] + + KEY --> ACQ["ledger.acquire
JtLedgerData worker
network fetch"] + KEY --> VAL["consensus.validation.accept
validation worker
trusted validation arrives"] + KEY --> CHK["ledger.validate
checkAccept
acceptance decision"] + KEY --> STO["ledger.store
storeLedger
persist"] + + classDef seed fill:#1f3b57,stroke:#8ab4d8,stroke-width:2px,color:#ffffff + classDef span fill:#f2f6fa,stroke:#4a6f8a,stroke-width:1px,color:#12232e + class KEY seed + class ACQ,VAL,CHK,STO span +``` + +Two limits worth knowing. The join needs the ledger **hash**: a stage that has +only a sequence number cannot join, so a by-sequence lookup made before the +hash is known will not appear. And the trace id is the hash's leading 16 bytes +only — the `ledger_hash` attribute carries all 32, which is how you confirm two +spans are genuinely the same ledger rather than a trace-id coincidence. + ## Performance Tuning | Scenario | Recommendation | diff --git a/include/xrpl/basics/MallocTrim.h b/include/xrpl/basics/MallocTrim.h index 2d0cf989bae..eb3244da10e 100644 --- a/include/xrpl/basics/MallocTrim.h +++ b/include/xrpl/basics/MallocTrim.h @@ -8,7 +8,10 @@ namespace xrpl { -// cSpell:ignore ptmalloc +// cSpell:ignore ptmalloc statm +// "statm" is the /proc/self/statm filename the RSS readings come from; it is a +// kernel path, not prose, so it cannot be respelled. Same directive as the two +// MallocTrim .cpp files. // ----------------------------------------------------------------------------- // Allocator interaction note: @@ -66,6 +69,19 @@ struct MallocTrimReport * * @note Intended for use after operations that free significant memory (e.g., * cache sweeps, ledger cleanup, online delete). Consider rate limiting. + * + * @note Every report field is populated on every call, whatever the journal's + * severity. Only the diagnostic JLOG is severity-gated. Measuring costs + * about 6 us (two /proc/self/statm reads and two getrusage calls) against + * a trim that costs milliseconds on a large heap, so callers on a cold + * path -- the cache sweep is the intended one -- can record the numbers + * unconditionally. A caller on a hot path should not use this function at + * all rather than expect the measurement to disappear. + * + * @note `minfltDelta` covers the trim call ONLY. It shows that the trim itself + * faults; it does NOT capture the faults later taken when the caches + * refill and touch the pages the trim returned. Do not read it as the + * total cost of trimming. */ MallocTrimReport mallocTrim(std::string_view tag, beast::Journal journal); diff --git a/include/xrpl/consensus/ConsensusSpanNames.h b/include/xrpl/consensus/ConsensusSpanNames.h index 6c79cd3132a..df743c7c616 100644 --- a/include/xrpl/consensus/ConsensusSpanNames.h +++ b/include/xrpl/consensus/ConsensusSpanNames.h @@ -73,6 +73,20 @@ * consensus.validation.receive [PeerImp I/O thread] * Created: PeerImp::onMessage(TMValidation) * + * ## Per-ledger trace (separate from the per-round trace above) + * + * consensus.validation.accept sits in a DIFFERENT trace from the spans above. + * The round trace is keyed on the PREVIOUS ledger hash (the round's seed); + * this span is keyed on the hash of the ledger the arriving validation is FOR, + * the same key LedgerMaster uses for ledger.validate and ledger.store, so a + * reader following one slow ledger sees the validation that triggered its + * acceptance beside the acceptance itself: + * + * trace_id = validatedLedgerHash[0:16] + * +-- consensus.validation.accept [JtValidationT worker / consensus thread] + * +-- ledger.validate [LedgerMaster::checkAccept] + * +-- ledger.store [LedgerMaster::storeLedger] + * * Legend: * +-- child-of relationship (same trace) * +~~~ follows-from link (separate sub-tree, causal link) @@ -80,6 +94,8 @@ #include +#include + namespace xrpl::telemetry::consensus::span { // ===== Span name segments ==================================================== @@ -105,6 +121,13 @@ inline constexpr auto modeChange = makeStr("mode_change"); inline constexpr auto proposalReceive = join(part::proposal, makeStr("receive")); inline constexpr auto validationReceive = join(part::validation, makeStr("receive")); inline constexpr auto phaseOpen = join(part::phase, makeStr("open")); +/** + * "validation.accept" — an arriving trusted validation being handed to the + * acceptance gate. Distinct from `validationReceive`, which is the overlay + * decoding a validation message: this is the later, per-ledger step where the + * validation is counted toward accepting a specific ledger. + */ +inline constexpr auto validationAccept = join(part::validation, makeStr("accept")); } // namespace op // ===== Full span names (prefix.op) =========================================== @@ -122,6 +145,12 @@ inline constexpr auto modeChange = join(seg::consensus, op::modeChange); inline constexpr auto proposalReceive = join(seg::consensus, op::proposalReceive); inline constexpr auto validationReceive = join(seg::consensus, op::validationReceive); inline constexpr auto phaseOpen = join(seg::consensus, op::phaseOpen); +/** + * "consensus.validation.accept" — full name, passed to `SpanGuard::hashSpan()` + * (which takes one complete span name) so the span adopts the trace id derived + * from the VALIDATED ledger's hash and joins that ledger's trace. + */ +inline constexpr auto validationAccept = join(seg::consensus, op::validationAccept); // ===== Attribute keys ======================================================== @@ -191,6 +220,21 @@ inline constexpr auto disputesResolvedCount = makeStr("disputes_resolved_count") * `using` re-export above.) */ inline constexpr auto validationSignTime = makeStr("validation_sign_time"); +/** + * "validation_status" — what the validation store did with an arriving + * validation, set on consensus.validation.accept. One of the bounded + * `ValStatus` names (see `val::status*`), so it is safe to aggregate: it is the + * difference between "validations are arriving and counting" and "they arrive + * and are all rejected", which on a stuck node look identical from the outside. + */ +inline constexpr auto validationStatus = makeStr("validation_status"); +/** + * "accept_gated" — true when this validation did NOT reach the acceptance gate + * because another thread was already accepting the same ledger (the + * bypass-accept path). Two values, so it is aggregatable; without it the + * absence of a following ledger.validate span in the trace is unexplained. + */ +inline constexpr auto acceptGated = makeStr("accept_gated"); /** * Receive-side hash prefixes for cross-peer correlation. */ @@ -291,6 +335,73 @@ inline constexpr auto unchanged = makeStr("unchanged"); inline constexpr auto phaseOpen = makeStr("open"); inline constexpr auto phaseEstablish = makeStr("establish"); inline constexpr auto phaseAccepted = makeStr("accepted"); + +/** + * validation_status values — the five `ValStatus` outcomes of adding an + * arriving validation to the validation store, plus a sentinel. + * + * Only `current` continues to the acceptance gate; every other value means the + * validation was counted for nothing, which is exactly the state a node stuck + * below quorum is in. Spelled here rather than reusing `to_string(ValStatus)` + * because that function returns camelCase ("badSeq"), and attribute values on + * an aggregated dimension must stay lower_snake_case. + */ +inline constexpr auto statusCurrent = makeStr("current"); +inline constexpr auto statusStale = makeStr("stale"); +inline constexpr auto statusBadSeq = makeStr("bad_seq"); +inline constexpr auto statusMultiple = makeStr("multiple"); +inline constexpr auto statusConflicting = makeStr("conflicting"); +inline constexpr auto statusUnknown = makeStr("unknown"); } // namespace val +// ===== Value rules =========================================================== + +/** + * Map a `ValStatus` to its `validation_status` attribute value. + * + * Takes a plain int rather than the enum so this header stays free of + * `Validations.h` (which pulls in the whole validation-store template) and so + * the rule can be asserted directly from the lib-only test binary, which cannot + * link xrpld. The caller passes `static_cast(status)`; the enumerator order + * is fixed by `ValStatus` and asserted by the paired unit test, which is what + * keeps this mapping honest if a value is ever inserted. + * + * @param valStatus `ValStatus` as an int: 0 Current, 1 Stale, 2 BadSeq, + * 3 Multiple, 4 Conflicting. + * @return The matching `val::status*` value, or `unknown` for anything else. + * + * Example — the two readings that matter on a stuck node: + * @code + * validationStatusValue(0); // "current" -- counted, gate will be tried + * validationStatusValue(2); // "bad_seq" -- rejected, counted for nothing + * @endcode + * + * Example — edge case: an out-of-range value still yields a usable label rather + * than an empty attribute, so the dimension never gains a blank series: + * @code + * validationStatusValue(99); // "unknown" + * @endcode + * + * @note Pure and side-effect free; safe to call from any thread. + */ +[[nodiscard]] constexpr std::string_view +validationStatusValue(int valStatus) noexcept +{ + switch (valStatus) + { + case 0: + return val::statusCurrent; + case 1: + return val::statusStale; + case 2: + return val::statusBadSeq; + case 3: + return val::statusMultiple; + case 4: + return val::statusConflicting; + default: + return val::statusUnknown; + } +} + } // namespace xrpl::telemetry::consensus::span diff --git a/include/xrpl/core/HashRouter.h b/include/xrpl/core/HashRouter.h index 20aafecc5f7..58fa9d14328 100644 --- a/include/xrpl/core/HashRouter.h +++ b/include/xrpl/core/HashRouter.h @@ -73,8 +73,6 @@ any(HashRouterFlags flags) return static_cast>(flags) != 0; } -class Config; - /** * Routing table for objects identified by hash. * diff --git a/include/xrpl/core/JobQueue.h b/include/xrpl/core/JobQueue.h index e07ab0362c9..0c1e622c6c2 100644 --- a/include/xrpl/core/JobQueue.h +++ b/include/xrpl/core/JobQueue.h @@ -215,6 +215,52 @@ class JobQueue : private Workers::Callback int getJobCountGE(JobType t) const; + /** + * Worker-pool saturation reading: work in flight against capacity. + * + * Answers "is the whole pool exhausted?" in one place. Without it, a + * pool-wide slowdown shows up separately in every subsystem whose jobs + * are queued behind it, and each one looks like its own fault. + */ + struct WorkerSaturation + { + /** + * Calls to processTask() executing right now across all job types. + */ + int runningTasks{0}; + + /** + * Worker threads the pool is configured to run — the ceiling + * `runningTasks` is measured against. + */ + int workerThreads{0}; + + /** + * Jobs queued and not yet dispatched, summed over all job types. + */ + int totalWaiting{0}; + }; + + /** + * Read the global worker-pool saturation. + * + * All three fields are produced under a single mutex acquire so the + * running/threads ratio and the backlog describe the same instant; read + * separately they could disagree and imply a saturation that never + * existed. + * + * @return The current saturation reading. + * + * @note Thread-safe with respect to the queue counters, which are read + * under the internal mutex. The configured thread count is a plain int + * that only changes at pool construction and at stop(); telemetry never + * races that write, because Application detaches the metric callbacks + * before it stops the JobQueue. + * @note Intended for a periodic observer, not a hot path. + */ + [[nodiscard]] WorkerSaturation + getWorkerSaturation() const; + /** * Return a scoped LoadEvent. */ diff --git a/include/xrpl/json/json_value.h b/include/xrpl/json/json_value.h index 47ad3ac1e0c..77c5f652108 100644 --- a/include/xrpl/json/json_value.h +++ b/include/xrpl/json/json_value.h @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -623,6 +624,12 @@ class ValueConstIterator : public ValueIteratorBase public: using size_t = unsigned int; using difference_type = int; + // std::iterator_traits needs value_type and iterator_category to classify + // this as a Cpp17InputIterator; without them it defaults to output-only, + // which breaks standard algorithms (e.g. std::all_of). The iterator walks a + // map both ways via ++/--, so it is bidirectional. + using value_type = Value; + using iterator_category = std::bidirectional_iterator_tag; using reference = Value const&; using pointer = Value const*; using SelfType = ValueConstIterator; @@ -687,6 +694,10 @@ class ValueIterator : public ValueIteratorBase public: using size_t = unsigned int; using difference_type = int; + // See ValueConstIterator: value_type and iterator_category are required for + // std::iterator_traits to treat this as a bidirectional iterator. + using value_type = Value; + using iterator_category = std::bidirectional_iterator_tag; using reference = Value&; using pointer = Value*; using SelfType = ValueIterator; diff --git a/include/xrpl/nodestore/Database.h b/include/xrpl/nodestore/Database.h index d88296e6b2b..47ef8a1aeef 100644 --- a/include/xrpl/nodestore/Database.h +++ b/include/xrpl/nodestore/Database.h @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -215,44 +216,66 @@ class Database } /** - * Cumulative time spent in backend fetches, in microseconds. - * - * Divide by getFetchTotalCount() to get the mean read latency. That mean - * is what separates a cold store from a warm one: a warm store reads in - * single-digit microseconds, a cold one in low hundreds. + * Total payload bytes returned by successful fetches. * - * @return The running microsecond total for the lifetime of this process. + * @return The running byte total for the lifetime of this process. */ std::uint64_t - getFetchDurationUs() const + getFetchSize() const { - return fetchDurationUs_; + return fetchSz_; } /** - * Cumulative time spent in backend stores, in microseconds. + * Cumulative microseconds spent inside store() calls. + * + * Pairs with getStoreCount() to derive mean write latency + * (`duration / count`), mirroring how the read side pairs + * getFetchDurationUs() with getFetchTotalCount(). The mean includes any + * time the backend spent waiting for its own internal locks, so it is wall + * time per store, not service time. * - * Divide by getStoreCount() to get the mean write latency. This includes - * any time the backend spent waiting for its own internal locks, so it is - * wall time per store, not service time. + * This is the "an existing DB syncs slower than a fresh one" signal: the + * read counters cannot show it, because back-fill is write-bound. Also + * published as the `node_writes_duration_us` field of getCountsJson(), so + * the RPC and the metric report the same number. * - * @return The running microsecond total for the lifetime of this process. + * @return Total microseconds accumulated across every completed store. + * + * @note Thread-safe: a single relaxed atomic load. Cheap enough for a + * periodic observer (the telemetry reader ticks every ~10 s). Relaxed is + * sufficient because the value is a monotonic statistic, not a + * synchronization signal — a reader that observes a slightly stale total + * simply reports a slightly stale mean. + * @note Monotonic and never reset, so a dashboard must take a rate or a + * delta of both this and getStoreCount() over the same window to see + * current latency rather than the since-boot average. */ - std::uint64_t - getStoreDurationUs() const + [[nodiscard]] std::uint64_t + getStoreDurationUs() const noexcept { - return storeDurationUs_; + return storeDurationUs_.load(std::memory_order_relaxed); } /** - * Total payload bytes returned by successful fetches. + * Cumulative microseconds spent inside fetchNodeObject() calls. * - * @return The running byte total for the lifetime of this process. + * Pairs with getFetchTotalCount() to derive mean read latency. That mean + * is what separates a cold store from a warm one: a warm store reads in + * single-digit microseconds, a cold one in low hundreds. The same + * total is already published as the `node_reads_duration_us` field of + * getCountsJson(); this accessor exposes it directly so a caller need not + * build a json::Value and parse a decimal string back to an integer. + * + * @return Total microseconds accumulated across every completed fetch. + * + * @note Same threading and monotonicity contract as + * getStoreDurationUs(). */ - std::uint64_t - getFetchSize() const + [[nodiscard]] std::uint64_t + getFetchDurationUs() const noexcept { - return fetchSz_; + return fetchDurationUs_.load(std::memory_order_relaxed); } void @@ -309,18 +332,35 @@ class Database } /** - * Add the wall time of one backend store to the cumulative total. + * Accumulate the time one completed store took. + * + * The write counterpart of the timing fetchNodeObject() already does for + * reads. `store()` is pure virtual, so unlike the read path there is no + * non-virtual wrapper in this class to time — each concrete database calls + * this once per store it completes, and the single conversion to + * microseconds lives here rather than being repeated per subclass. Each + * concrete store path times only its backend call, so the total reflects + * disk work and excludes cache bookkeeping. * - * Each concrete store path times only its backend call, so the total - * reflects disk work and excludes cache bookkeeping. Callers must pass - * microseconds. + * @param elapsed Wall time the store took, as measured by the caller. * - * @param us Wall time of the completed backend store, in microseconds. + * @note Call once per store operation, never inside a per-tree-node loop: + * a ledger write walks thousands of SHAMap nodes and this must stay a + * single atomic add on the whole write, matching the one-sample-per-fetch + * cost on the read side. + * @note Thread-safe: one relaxed atomic add, no lock. Relaxed ordering is + * correct because the total is a statistic that is only ever read by a + * periodic observer, never used to order other memory operations. + * @note A negative duration cannot occur (steady_clock is monotonic), but + * a caller passing one would be clamped to zero rather than wrapping the + * unsigned total to a huge value. */ void - storeDurationStats(std::uint64_t us) + recordStoreDuration(std::chrono::steady_clock::duration elapsed) noexcept { - storeDurationUs_ += us; + auto const us = std::chrono::duration_cast(elapsed).count(); + if (us > 0) + storeDurationUs_.fetch_add(static_cast(us), std::memory_order_relaxed); } // Called by the public import function @@ -367,7 +407,7 @@ class Database /** * Wall time spent in backend stores, in microseconds. * - * Written by each concrete store path via storeDurationStats(), which + * Written by each concrete store path via recordStoreDuration(), which * times only the backend call. */ std::atomic storeDurationUs_{0}; diff --git a/include/xrpl/nodestore/DatabaseRotating.h b/include/xrpl/nodestore/DatabaseRotating.h index 21b8b422c79..057d1486f9d 100644 --- a/include/xrpl/nodestore/DatabaseRotating.h +++ b/include/xrpl/nodestore/DatabaseRotating.h @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -54,6 +55,38 @@ class DatabaseRotating : public Database */ virtual void setRotationInFlight(bool inFlight) = 0; + + /** + * Whether an online-delete rotation is in progress right now. + * + * A rotation's extra writes only happen inside this window, so a panel + * reading the copy-forward total needs this to know when to expect it to + * move. Outside the window a flat total is correct, not a broken signal. + * + * @return true between the cache-freshen phase starting and rotate() + * completing. + */ + [[nodiscard]] virtual bool + isRotationInFlight() const = 0; + + /** + * Nodes copied forward from the archive backend into the writable one + * during rotation windows, since this process started. + * + * These are writes an ordinary fetch would not have performed: the archive + * is about to be deleted, so a body it served has to be rewritten to + * survive. The count therefore scales with how much of the archive is read + * during a rotation, which is why it appears only on a populated, + * already-rotated online_delete database. + * + * Cumulative for the lifetime of the process, deliberately: the per-rotation + * tally that `rotate()` logs is reset on every swap, and a counter that goes + * backwards cannot be rated. A panel takes the rate of this instead. + * + * @return Monotonic count of copy-forward writes. + */ + [[nodiscard]] virtual std::uint64_t + copyForwardTotal() const = 0; }; } // namespace xrpl::node_store diff --git a/include/xrpl/nodestore/detail/DatabaseRotatingImp.h b/include/xrpl/nodestore/detail/DatabaseRotatingImp.h index 22ee4f367c5..9262826afef 100644 --- a/include/xrpl/nodestore/detail/DatabaseRotatingImp.h +++ b/include/xrpl/nodestore/detail/DatabaseRotatingImp.h @@ -78,6 +78,18 @@ class DatabaseRotatingImp : public DatabaseRotating void setRotationInFlight(bool inFlight) override; + [[nodiscard]] bool + isRotationInFlight() const override + { + return rotationInFlight_.load(std::memory_order_acquire); + } + + [[nodiscard]] std::uint64_t + copyForwardTotal() const override + { + return copyForwardTotal_.load(std::memory_order_relaxed); + } + private: std::shared_ptr writableBackend_; std::shared_ptr archiveBackend_; @@ -91,6 +103,13 @@ class DatabaseRotatingImp : public DatabaseRotating std::atomic rotationInFlight_{false}; std::atomic copyForwardCount_{0}; + // Same events as copyForwardCount_, but never reset. rotate() zeroes the + // per-rotation tally above to produce its one-line summary, which makes that + // counter unusable as a metric: a series that drops to 0 on every swap + // cannot be rated. This one only ever increases, so rate() over it reads + // correctly and the two uses do not fight over one variable. + std::atomic copyForwardTotal_{0}; + std::shared_ptr fetchNodeObject(uint256 const& hash, std::uint32_t, FetchReport& fetchReport, bool duplicate) override; diff --git a/include/xrpl/peerfinder/PeerfinderManager.h b/include/xrpl/peerfinder/PeerfinderManager.h index ed683520c19..cb6d1f8471f 100644 --- a/include/xrpl/peerfinder/PeerfinderManager.h +++ b/include/xrpl/peerfinder/PeerfinderManager.h @@ -9,6 +9,7 @@ #include +#include #include #include #include @@ -17,6 +18,98 @@ namespace xrpl::PeerFinder { +//------------------------------------------------------------------------------ + +/** + * One consistent snapshot of slot occupancy and address-cache depth. + * + * Every field is read under a single acquire of the PeerFinder lock, so the + * nine numbers describe the same instant and can be compared against each + * other. Reading them through separate accessors would not give that: the + * autoconnect logic mutates counts and caches together, so two reads taken a + * moment apart can show a state that never existed. + * + * Dependency: + * + * +-----------------+ reads +--------+ + * | Logic (counts_, |--------->| Counts | + * | fixed_, caches)| +--------+ + * +--------+--------+ + * | fills + * v + * +-----------------+ returned by +---------+ + * | SlotCensus |<--------------| Manager | + * +-----------------+ +---------+ + * + * Example usage -- capacity check from a telemetry gauge callback: + * @code + * auto const census = overlay.getSlotCensus(); + * if (census.outActive < census.outMax && census.connecting > 0) + * // dials are starting but never completing + * @endcode + * + * Example usage -- edge case: a fresh node with nothing to dial: + * @code + * auto const census = overlay.getSlotCensus(); + * if (census.bootcache == 0 && census.livecache == 0) + * // no seed addresses at all; check [ips] and DNS + * @endcode + * + * @note All fields are signed so a caller can hand them straight to a metric + * without an unsigned-to-signed conversion that would turn a sentinel + * into a large positive or a negative value. + * @note Snapshot only. The values are stale the moment the lock is released, + * which is fine for diagnostics but must not be used to make a slot + * decision -- take the lock and re-read for that. + */ +struct SlotCensus +{ + /** + * Outbound slots currently occupied by an active peer. + */ + std::int64_t outActive{0}; + + /** + * Outbound slots configured, i.e. the ceiling on outActive. + */ + std::int64_t outMax{0}; + + /** + * Inbound slots currently occupied by an active peer. + */ + std::int64_t inActive{0}; + + /** + * Inbound slots configured; 0 when inbound connections are disabled. + */ + std::int64_t inMax{0}; + + /** + * Outbound connection attempts in flight, not yet active or failed. + */ + std::int64_t connecting{0}; + + /** + * Fixed peers named in the configuration. + */ + std::int64_t fixedConfigured{0}; + + /** + * Fixed peers currently connected, out of fixedConfigured. + */ + std::int64_t fixedActive{0}; + + /** + * Addresses held for bootstrapping, persisted across restarts. + */ + std::int64_t bootcache{0}; + + /** + * Addresses learned from peers this session, held in memory only. + */ + std::int64_t livecache{0}; +}; + /** * Maintains a set of IP addresses used for getting into the network. */ @@ -61,6 +154,26 @@ class Manager : public beast::PropertyStream::Source virtual Config config() = 0; + /** + * Returns one consistent snapshot of slot occupancy and cache depth. + * + * Diagnostics only; nothing in the connection logic reads it. Exposed + * because the counts are otherwise invisible outside the PropertyStream: + * only the two active-peer counts are exported today, so a node that + * cannot dial is indistinguishable from one that has nothing to dial. + * + * Not `const`, and cannot be: the implementation takes the PeerFinder + * lock, which is a plain (non-mutable) member, and every other method on + * this interface is non-const for the same reason. + * + * @return All nine fields, read under a single lock acquire. + * + * @note Cheap: one lock acquire, then integer and container-size reads. + * Intended for a ~10 s telemetry poll, not a per-message path. + */ + virtual SlotCensus + getSlotCensus() = 0; + /** * Add a peer that should always be connected. * This is useful for maintaining a private cluster of peers. diff --git a/include/xrpl/peerfinder/detail/Logic.h b/include/xrpl/peerfinder/detail/Logic.h index c6232638847..84c7e94a3e7 100644 --- a/include/xrpl/peerfinder/detail/Logic.h +++ b/include/xrpl/peerfinder/detail/Logic.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -169,6 +170,43 @@ class Logic return config_; } + /** + * Builds one consistent snapshot of slot occupancy and cache depth. + * + * Lives here rather than on ManagerImp because this is the only scope + * that can see all four sources at once: `counts_` and `fixed_` are + * private, and `lock` is what makes the nine fields describe a single + * instant instead of nine successive ones. + * + * `fixedConfigured` comes from `fixed_.size()` (peers named in the + * config), not `counts_.fixed()` (slots currently present that happen to + * be fixed). The pair that matters to an operator is "how many did I ask + * for" against "how many do I have", which is `fixed_.size()` against + * `counts_.fixedActive()` -- the same comparison autoconnect() makes when + * it decides to redial a fixed peer. + * + * @return All nine fields, read under a single acquire of `lock`. + * + * @note The container sizes are unsigned; each is converted explicitly so + * no value can wrap into a negative reading. + */ + SlotCensus + getSlotCensus() + { + std::scoped_lock const _(lock); + + return SlotCensus{ + .outActive = counts_.outActive(), + .outMax = counts_.outMax(), + .inActive = counts_.inboundActive(), + .inMax = counts_.inMax(), + .connecting = counts_.connectCount(), + .fixedConfigured = static_cast(fixed_.size()), + .fixedActive = static_cast(counts_.fixedActive()), + .bootcache = static_cast(bootcache.size()), + .livecache = static_cast(livecache.size())}; + } + void addFixedPeer(std::string_view name, beast::IP::Endpoint const& ep) { diff --git a/include/xrpl/server/NetworkOPs.h b/include/xrpl/server/NetworkOPs.h index 4839b5f1f31..22e9d00776f 100644 --- a/include/xrpl/server/NetworkOPs.h +++ b/include/xrpl/server/NetworkOPs.h @@ -111,10 +111,37 @@ class NetworkOPs : public InfoSub::Source */ [[nodiscard]] virtual std::chrono::microseconds getServerStateDurationUs() const = 0; + /** + * Microseconds from process start until the node first reached FULL. + * + * Zero while the node has not completed initial sync yet, so a value that + * stays at zero is itself the signal that sync never finished. Once set it + * never changes: this is the one-shot time-to-first-FULL, not the time in + * the current state. Same value as `initial_sync_duration_us` in + * server_info, exposed as a lightweight accessor so metrics can read it + * without building the full server_info JSON on every collection tick. + * + * @return Microseconds to first FULL, or 0 if not yet reached. + */ + [[nodiscard]] virtual std::uint64_t + getInitialSyncDurationUs() const = 0; [[nodiscard]] virtual std::string strOperatingMode(OperatingMode const mode, bool const admin = false) const = 0; [[nodiscard]] virtual std::string strOperatingMode(bool const admin = false) const = 0; + /** + * How far this node's validated ledger trails the network's. + * + * The network target is the highest ledger sequence any connected peer + * reports holding; the result is that target minus our own validated + * sequence, floored at zero. Zero therefore means either "at the tip" or + * "no peer has told us about a newer ledger", which on a node with no + * peers is the same thing. + * + * @return Ledgers behind the network, 0 when at or ahead of the tip. + */ + [[nodiscard]] virtual std::uint32_t + getLedgersBehindNetwork() const = 0; //-------------------------------------------------------------------------- // diff --git a/include/xrpl/shamap/SHAMapAddNode.h b/include/xrpl/shamap/SHAMapAddNode.h index 87222f74d82..38ad3beeef4 100644 --- a/include/xrpl/shamap/SHAMapAddNode.h +++ b/include/xrpl/shamap/SHAMapAddNode.h @@ -24,6 +24,28 @@ class SHAMapAddNode reset(); [[nodiscard]] int getGood() const; + /** + * Nodes rejected as invalid in this tally. + * + * Complements getGood(): isInvalid() only answers "was there at least one", + * which cannot distinguish one bad node from a peer sending nothing but bad + * data. Exposed for the acquire telemetry counters, which need the count. + * + * @return Number of invalid nodes; 0 if none. + */ + [[nodiscard]] int + getBad() const; + /** + * Nodes already held, so re-receiving them was wasted work. + * + * A high duplicate share against a low good share means peers are re-sending + * data the node already has, which looks like healthy traffic but makes no + * acquire progress. + * + * @return Number of duplicate nodes; 0 if none. + */ + [[nodiscard]] int + getDuplicate() const; [[nodiscard]] bool isGood() const; [[nodiscard]] bool @@ -86,6 +108,18 @@ SHAMapAddNode::getGood() const return good_; } +inline int +SHAMapAddNode::getBad() const +{ + return bad_; +} + +inline int +SHAMapAddNode::getDuplicate() const +{ + return duplicate_; +} + inline bool SHAMapAddNode::isInvalid() const { diff --git a/include/xrpl/telemetry/GetObjectMetricNames.h b/include/xrpl/telemetry/GetObjectMetricNames.h index e2951a43d06..6df75182b89 100644 --- a/include/xrpl/telemetry/GetObjectMetricNames.h +++ b/include/xrpl/telemetry/GetObjectMetricNames.h @@ -1,9 +1,5 @@ #pragma once -// cspell:ignore ISTOGRAM -// The all-caps macro name XRPL_METRIC_HISTOGRAM_RECORD trips cspell's -// compound-word splitter, which emits the subword "ISTOGRAM"; ignore it here. - /** * Metric names, label keys, label values, and descriptions for the * `TMGetObjectByHash` request path. diff --git a/src/libxrpl/basics/MallocTrim.cpp b/src/libxrpl/basics/MallocTrim.cpp index 3831c8adae8..e4b982afd99 100644 --- a/src/libxrpl/basics/MallocTrim.cpp +++ b/src/libxrpl/basics/MallocTrim.cpp @@ -15,7 +15,6 @@ #include #include -#include #include #include #include @@ -69,6 +68,73 @@ parseStatmRSSkB(std::string const& statm) return (resident * pageSize) / 1024; } +/** + * Read a whole /proc pseudo-file into a string. + * + * /proc files are frequently not seekable, so the contents are streamed rather + * than sized-then-read. + * + * @param path Absolute path of the pseudo-file. + * @return The file contents, or an empty string if it could not be opened. + */ +std::string +readProcFile(std::string const& path) +{ + std::ifstream ifs(path, std::ios::in | std::ios::binary); + if (!ifs.is_open()) + return {}; + + std::ostringstream oss; + oss << ifs.rdbuf(); + return oss.str(); +} + +/** + * Run malloc_trim and measure what it cost. + * + * Split out of mallocTrim() so the always-on measurement is one testable unit + * and the caller is left with only the logging decision. + * + * Measurement order brackets the trim as tightly as possible: the two RSS + * samples are outermost, the two fault samples inside them, and the clock pair + * innermost, so the reported duration contains the trim and nothing else. + * + * @param padBytes glibc trim padding, passed straight to ::malloc_trim. + * @return A fully populated report. Fields whose source syscall failed keep + * their -1 "not measured" sentinel. + */ +MallocTrimReport +measuredTrim(std::size_t padBytes) +{ + MallocTrimReport report; + report.supported = true; + + std::string const statmPath = "/proc/self/statm"; + + report.rssBeforeKB = detail::parseStatmRSSkB(readProcFile(statmPath)); + + struct rusage ru0{}; + bool const haveRu0 = getRusageThread(ru0); + + auto const t0 = std::chrono::steady_clock::now(); + report.trimResult = detail::mallocTrimWithPad(padBytes); + auto const t1 = std::chrono::steady_clock::now(); + + struct rusage ru1{}; + bool const haveRu1 = getRusageThread(ru1); + + report.rssAfterKB = detail::parseStatmRSSkB(readProcFile(statmPath)); + report.durationUs = std::chrono::duration_cast(t1 - t0); + + if (haveRu0 && haveRu1) + { + report.minfltDelta = ru1.ru_minflt - ru0.ru_minflt; + report.majfltDelta = ru1.ru_majflt - ru0.ru_majflt; + } + + return report; +} + #endif // __GLIBC__ && BOOST_OS_LINUX } // namespace detail @@ -88,70 +154,37 @@ mallocTrim(std::string_view tag, beast::Journal journal) // of RSS reduction and trim-latency stability without adding a tuning surface. static constexpr std::size_t kTrimPad = 0; - report.supported = true; - + // The measurement is unconditional. It used to sit inside + // `if (journal.debug())`, which meant an ordinary node -- which does not run + // at debug level -- measured nothing, so the caller had no duration to + // record and the per-sweep trim cost was invisible in production, the one + // place it matters. + // + // Cost of measuring on every sweep, measured on this platform: two + // /proc/self/statm reads at ~2.8 us each and two getrusage(RUSAGE_THREAD) + // calls at ~0.17 us each, so about 6 us in total. The trim it brackets + // costs milliseconds on a large heap, and the sweep that calls it runs once + // per SizedItem::SweepInterval (10 s at the fastest, tiny-node setting). + // That is a duty cycle under 1e-6 percent, and under 1 percent of the + // measured operation, so nothing here is worth making conditional -- a + // debug-only RSS read would only reintroduce the blind spot it costs + // nothing to remove. + report = detail::measuredTrim(kTrimPad); + + // Only the LOG stays gated: the string formatting is what an ordinary node + // genuinely should not pay for, and the numbers now reach the metrics + // pipeline through the return value instead. if (journal.debug()) { - auto readFile = [](std::string const& path) -> std::string { - std::ifstream ifs(path, std::ios::in | std::ios::binary); - if (!ifs.is_open()) - return {}; - - // /proc files are often not seekable; read as a stream. - std::ostringstream oss; - oss << ifs.rdbuf(); - return oss.str(); - }; - - std::string const tagStr{tag}; - std::string const statmPath = "/proc/self/statm"; - - auto const statmBefore = readFile(statmPath); - long const rssBeforeKB = detail::parseStatmRSSkB(statmBefore); - - struct rusage ru0{}; - bool const haveRu0 = getRusageThread(ru0); - - auto const t0 = std::chrono::steady_clock::now(); - - report.trimResult = detail::mallocTrimWithPad(kTrimPad); - - auto const t1 = std::chrono::steady_clock::now(); - - struct rusage ru1{}; - bool const haveRu1 = getRusageThread(ru1); - - auto const statmAfter = readFile(statmPath); - long const rssAfterKB = detail::parseStatmRSSkB(statmAfter); - - // Populate report fields - report.rssBeforeKB = rssBeforeKB; - report.rssAfterKB = rssAfterKB; - report.durationUs = std::chrono::duration_cast(t1 - t0); - - if (haveRu0 && haveRu1) - { - report.minfltDelta = ru1.ru_minflt - ru0.ru_minflt; - report.majfltDelta = ru1.ru_majflt - ru0.ru_majflt; - } - - std::int64_t const deltaKB = (rssBeforeKB < 0 || rssAfterKB < 0) - ? 0 - : (static_cast(rssAfterKB) - static_cast(rssBeforeKB)); - - JLOG(journal.debug()) << "malloc_trim tag=" << tagStr << " result=" << report.trimResult + JLOG(journal.debug()) << "malloc_trim tag=" << tag << " result=" << report.trimResult << " pad=" << kTrimPad << " bytes" - << " rss_before=" << rssBeforeKB << "kB" - << " rss_after=" << rssAfterKB << "kB" - << " delta=" << deltaKB << "kB" + << " rss_before=" << report.rssBeforeKB << "kB" + << " rss_after=" << report.rssAfterKB << "kB" + << " delta=" << report.deltaKB() << "kB" << " duration_us=" << report.durationUs.count() << " minflt_delta=" << report.minfltDelta << " majflt_delta=" << report.majfltDelta; } - else - { - report.trimResult = detail::mallocTrimWithPad(kTrimPad); - } #endif diff --git a/src/libxrpl/core/detail/JobQueue.cpp b/src/libxrpl/core/detail/JobQueue.cpp index 71051839a96..c235e25f5f4 100644 --- a/src/libxrpl/core/detail/JobQueue.cpp +++ b/src/libxrpl/core/detail/JobQueue.cpp @@ -193,6 +193,25 @@ JobQueue::getJobCountGE(JobType t) const return ret; } +JobQueue::WorkerSaturation +JobQueue::getWorkerSaturation() const +{ + // Read the configured thread count and the in-flight task count before + // taking the mutex: neither is guarded by it (numberOfThreads_ is set at + // construction, runningTaskCount_ is an atomic), and holding the queue + // lock across them would add contention for no benefit. + WorkerSaturation out; + out.runningTasks = workers_.numberOfCurrentlyRunningTasks(); + out.workerThreads = workers_.getNumberOfThreads(); + + std::scoped_lock const lock(mutex_); + + for (auto const& entry : jobData_) + out.totalWaiting += entry.second.waiting; + + return out; +} + std::unique_ptr JobQueue::makeLoadEvent(JobType t, std::string const& name) { diff --git a/src/libxrpl/nodestore/Database.cpp b/src/libxrpl/nodestore/Database.cpp index e2aef00d670..e264336b359 100644 --- a/src/libxrpl/nodestore/Database.cpp +++ b/src/libxrpl/nodestore/Database.cpp @@ -195,6 +195,10 @@ Database::importInternal(Backend& dstBackend, Database& srcDB) Batch batch; batch.reserve(kBatchWritePreallocationSize); auto storeBatch = [&, fname = __func__]() { + // One clock sample per batch, not per object: the loop below walks + // every object in the batch and must stay free of timing work. + auto const begin{std::chrono::steady_clock::now()}; + try { dstBackend.storeBatch(batch); @@ -205,6 +209,10 @@ Database::importInternal(Backend& dstBackend, Database& srcDB) return; } + // Only a batch that actually reached the backend contributes, so a + // failed write does not read as a fast one. + recordStoreDuration(std::chrono::steady_clock::now() - begin); + std::uint64_t sz{0}; for (auto const& nodeObject : batch) sz += nodeObject->getData().size(); diff --git a/src/libxrpl/nodestore/DatabaseNodeImp.cpp b/src/libxrpl/nodestore/DatabaseNodeImp.cpp index 16a3249c277..8b5f4cf1831 100644 --- a/src/libxrpl/nodestore/DatabaseNodeImp.cpp +++ b/src/libxrpl/nodestore/DatabaseNodeImp.cpp @@ -25,14 +25,14 @@ DatabaseNodeImp::store(NodeObjectType type, Blob&& data, uint256 const& hash, st auto obj = NodeObject::createObject(type, std::move(data), hash); - // Time only the backend call. The cache work below is not disk work and - // would blur the write latency signal. + // Time only the backend write, which is the disk work. The cache work below + // is not disk work and would blur the write latency signal. One clock pair + // per stored object, accumulated into an atomic that the metrics gauge reads + // on its own schedule, so nothing is added to the read path or per tree + // node. auto const begin = std::chrono::steady_clock::now(); backend_->store(obj); - storeDurationStats( - static_cast(std::chrono::duration_cast( - std::chrono::steady_clock::now() - begin) - .count())); + recordStoreDuration(std::chrono::steady_clock::now() - begin); if (cache_) { diff --git a/src/libxrpl/nodestore/DatabaseRotatingImp.cpp b/src/libxrpl/nodestore/DatabaseRotatingImp.cpp index 3c1554faacd..bece5b7d9b1 100644 --- a/src/libxrpl/nodestore/DatabaseRotatingImp.cpp +++ b/src/libxrpl/nodestore/DatabaseRotatingImp.cpp @@ -138,14 +138,14 @@ DatabaseRotatingImp::store(NodeObjectType type, Blob&& data, uint256 const& hash return writableBackend_; }(); - // Time only the backend call, matching DatabaseNodeImp, so the two store - // paths feed the same accumulator with comparable numbers. + // Time only the backend write, which is the disk work, matching + // DatabaseNodeImp so the two store paths feed the same accumulator with + // comparable numbers. One clock pair per stored object, accumulated into an + // atomic that the metrics gauge reads on its own schedule, so nothing is + // added to the read path or per tree node. auto const begin = std::chrono::steady_clock::now(); backend->store(nObj); - storeDurationStats( - static_cast(std::chrono::duration_cast( - std::chrono::steady_clock::now() - begin) - .count())); + recordStoreDuration(std::chrono::steady_clock::now() - begin); storeStats(1, nObj->getData().size()); } @@ -223,7 +223,13 @@ DatabaseRotatingImp::fetchNodeObject( if (duplicate || rotationInFlight_.load(std::memory_order_acquire)) { if (!duplicate) + { + // Two counters, one event: the per-rotation tally that + // rotate() resets for its log line, and the monotonic total + // the metrics gauge reads, which must never go backwards. copyForwardCount_.fetch_add(1, std::memory_order_relaxed); + copyForwardTotal_.fetch_add(1, std::memory_order_relaxed); + } writable->store(nodeObject); } } diff --git a/src/libxrpl/peerfinder/PeerfinderManager.cpp b/src/libxrpl/peerfinder/PeerfinderManager.cpp index 2219627f098..aff26b23797 100644 --- a/src/libxrpl/peerfinder/PeerfinderManager.cpp +++ b/src/libxrpl/peerfinder/PeerfinderManager.cpp @@ -97,6 +97,12 @@ class ManagerImp : public Manager return logic_.config(); } + SlotCensus + getSlotCensus() override + { + return logic_.getSlotCensus(); + } + void addFixedPeer(std::string_view name, std::vector const& addresses) override { diff --git a/src/test/app/LedgerReplay_test.cpp b/src/test/app/LedgerReplay_test.cpp index 4cc83608d6b..ab16d5a7e1f 100644 --- a/src/test/app/LedgerReplay_test.cpp +++ b/src/test/app/LedgerReplay_test.cpp @@ -252,6 +252,14 @@ class MagicInboundLedgers : public InboundLedgers return 0; } + // This mock holds no InboundLedger objects, so there is no acquire progress + // to report; the all-zero snapshot is the honest answer. + AcquireProgress + acquireProgress() override + { + return {}; + } + LedgerMaster& ledgerSource; LedgerMaster& ledgerSink; InboundLedgersBehavior bhvr; diff --git a/src/test/core/JobQueue_test.cpp b/src/test/core/JobQueue_test.cpp index 47c0995cdeb..c5ebfb3f82d 100644 --- a/src/test/core/JobQueue_test.cpp +++ b/src/test/core/JobQueue_test.cpp @@ -539,7 +539,9 @@ class JobQueue_test : public beast::unit_test::Suite // Every one of these must defer: the limit is already reached. int const extra = 4; for (int i = 0; i < extra; ++i) + { BEAST_EXPECT(fixture.queue.addJob(JtPack, "GaugeDefer", blockingJob)); + } // Exact values, not bounds: `running` equals the type's limit and // `deferred` equals the number of submissions beyond it. Both are diff --git a/src/test/overlay/TMGetObjectByHash_test.cpp b/src/test/overlay/TMGetObjectByHash_test.cpp index e729ffe00ea..31b3d73eedf 100644 --- a/src/test/overlay/TMGetObjectByHash_test.cpp +++ b/src/test/overlay/TMGetObjectByHash_test.cpp @@ -468,7 +468,9 @@ class TMGetObjectByHash_test : public beast::unit_test::Suite auto reply = parseReply(peer); BEAST_EXPECT(reply.has_value()); if (reply) + { BEAST_EXPECT(reply->objects_size() == 0); + } // Positive counterpart to the oversize test: because the handler did // run, a differential charge was applied, and it is exactly the diff --git a/src/tests/libxrpl/basics/MallocTrim.cpp b/src/tests/libxrpl/basics/MallocTrim.cpp index 6ac8957f0eb..02ab18f1154 100644 --- a/src/tests/libxrpl/basics/MallocTrim.cpp +++ b/src/tests/libxrpl/basics/MallocTrim.cpp @@ -121,7 +121,12 @@ TEST(parseStatmRSSkB, standard_format) } #endif -TEST(mallocTrim, without_debug_logging) +// The measurement must NOT depend on the journal's severity. It used to sit +// inside `if (journal.debug())`, so an ordinary node -- which does not run at +// debug level -- measured nothing and the caller had no duration to record. +// This is the regression test for that: with a null sink (nothing is even +// loggable) every field must still be populated. +TEST(mallocTrim, measures_without_debug_logging) { beast::Journal const journal{beast::Journal::getNullSink()}; @@ -130,10 +135,29 @@ TEST(mallocTrim, without_debug_logging) #if defined(__GLIBC__) && BOOST_OS_LINUX EXPECT_EQ(report.supported, true); EXPECT_GE(report.trimResult, 0); - EXPECT_EQ(report.durationUs, std::chrono::microseconds{-1}); - EXPECT_EQ(report.minfltDelta, -1); - EXPECT_EQ(report.majfltDelta, -1); + + // The three measured fields are populated, NOT left at their -1 + // "not measured" sentinel. Asserting >= 0 rather than == a fixed number + // because these are real timings; the sentinel is what the test excludes. + EXPECT_GE(report.durationUs.count(), 0); + EXPECT_GE(report.minfltDelta, 0); + EXPECT_GE(report.majfltDelta, 0); + + // RSS is read on both sides of the trim, so both are real page counts. + // A live process always has resident pages, so these are strictly > 0. + EXPECT_GT(report.rssBeforeKB, 0); + EXPECT_GT(report.rssAfterKB, 0); + + // deltaKB() is now derived from two real readings rather than from the + // sentinel pair, so it is the genuine change: a trim never grows RSS by + // more than another thread could allocate concurrently, and this test is + // single-threaded, so the reading cannot be positive. + EXPECT_LE(report.deltaKB(), 0); + EXPECT_EQ(report.deltaKB(), report.rssAfterKB - report.rssBeforeKB); #else + // NEGATIVE PLATFORM PATH: not Linux/glibc, so there is no trim at all and + // every field must keep its sentinel. A zero here would falsely claim a + // free trim happened. EXPECT_EQ(report.supported, false); EXPECT_EQ(report.trimResult, -1); EXPECT_EQ(report.rssBeforeKB, -1); @@ -141,6 +165,7 @@ TEST(mallocTrim, without_debug_logging) EXPECT_EQ(report.durationUs, std::chrono::microseconds{-1}); EXPECT_EQ(report.minfltDelta, -1); EXPECT_EQ(report.majfltDelta, -1); + EXPECT_EQ(report.deltaKB(), 0); #endif } @@ -185,6 +210,12 @@ TEST(mallocTrim, with_debug_logging) EXPECT_GE(report.durationUs.count(), 0); EXPECT_GE(report.minfltDelta, 0); EXPECT_GE(report.majfltDelta, 0); + + // Same fields as the null-sink case above: raising the severity adds the + // log line and changes nothing about what is measured. The two tests + // together are what prove the severity no longer gates the measurement. + EXPECT_GT(report.rssBeforeKB, 0); + EXPECT_GT(report.rssAfterKB, 0); #else EXPECT_EQ(report.supported, false); EXPECT_EQ(report.trimResult, -1); diff --git a/src/tests/libxrpl/nodestore/Database.cpp b/src/tests/libxrpl/nodestore/Database.cpp index 2f894428158..837275ad262 100644 --- a/src/tests/libxrpl/nodestore/Database.cpp +++ b/src/tests/libxrpl/nodestore/Database.cpp @@ -468,4 +468,80 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(importBackends()), [](::testing::TestParamInfo const& info) { return info.param; }); +/** + * Verify the import store path feeds the write-duration accumulator. + * + * getStoreDurationUs() backs the only write-side latency signal there is, and + * the telemetry gauge derives the mean write latency by dividing it by + * getStoreCount(). Database::importInternal() writes straight to the backend + * with storeBatch() rather than through the virtual store(), so it is a third + * store path that has to time itself; the per-store coverage in + * DatabaseConfig_test cannot reach it. Asserting the exact zero-before state + * as well as the accumulate-after state means a regression to a never-written + * member fails here rather than showing up as a permanently flat dashboard + * panel. + * + * nudb rather than memory: the import has to do real backend work for the + * elapsed time to round up to a whole microsecond. + */ +TEST(NodeStoreDatabase, import_accumulates_store_duration) +{ + DummyScheduler scheduler; + beast::Journal const journal(TestSink::instance()); + + beast::TempDir const srcDir; + Section srcParams; + srcParams.set("type", "nudb"); + srcParams.set("path", srcDir.path()); + + auto const batch = createPredictableBatch(kNumObjects, kSeedValue); + ASSERT_EQ(batch.size(), static_cast(kNumObjects)); + + auto src = Manager::instance().makeDatabase(megabytes(4), scheduler, 2, srcParams, journal); + ASSERT_NE(src, nullptr); + + storeBatch(*src, batch); + ASSERT_EQ(src->getStoreCount(), batch.size()); + + beast::TempDir const destDir; + Section destParams; + destParams.set("type", "nudb"); + destParams.set("path", destDir.path()); + + auto dest = Manager::instance().makeDatabase(megabytes(4), scheduler, 2, destParams, journal); + ASSERT_NE(dest, nullptr); + + // A second, freshly opened database reads exactly zero even though the + // source has already written. That is what proves the accumulator is + // per-instance state and not a shared global, and it is the state the + // gauge must report as "no mean available" rather than as a + // zero-microsecond latency. + ASSERT_EQ(dest->getStoreCount(), 0u); + ASSERT_EQ(dest->getStoreDurationUs(), 0u); + + dest->importDatabase(*src); + + // Every object crossed exactly once. importInternal reports whole batches + // of kBatchWritePreallocationSize, so the count is the object total and + // not the number of batches. + ASSERT_EQ(dest->getStoreCount(), batch.size()); + + // The core assertion: the import path timed its backend writes. The exact + // microsecond figure is wall-clock dependent, but "still exactly zero + // after real writes" is the dead-member bug being guarded against. + EXPECT_GT(dest->getStoreDurationUs(), 0u); + + // Negative half: the import wrote into dest, not src, so the source's + // store count cannot have moved. Its duration is legitimately non-zero + // from its own storeBatch above, so the count is what isolates the + // import's effect. + EXPECT_EQ(src->getStoreCount(), batch.size()); + + // A mean derived the way the telemetry gauge derives it must be a sane, + // non-zero microsecond figure rather than a division artifact. Strictly + // positive, not >= 0: both operands are unsigned, so >= 0 would be + // vacuously true and gcc rejects it outright. + EXPECT_GT(dest->getStoreDurationUs() / dest->getStoreCount(), 0u); +} + } // namespace xrpl::node_store diff --git a/src/tests/libxrpl/peerfinder/PeerFinder.cpp b/src/tests/libxrpl/peerfinder/PeerFinder.cpp index 31fa59d1cee..5e0d2c33eb2 100644 --- a/src/tests/libxrpl/peerfinder/PeerFinder.cpp +++ b/src/tests/libxrpl/peerfinder/PeerFinder.cpp @@ -1266,5 +1266,299 @@ TEST(PeerFinderConfig, rejects_incomplete_or_out_of_range_peer_limits) } } +/** + * A freshly configured Logic reports its ceilings and nothing else. + * + * This is the baseline reading every other census assertion is measured + * against, and it is a real diagnostic state: a node that has just started + * and connected to nothing must publish its configured capacity so an + * operator can tell "no slots configured" apart from "slots configured but + * empty". Asserting every one of the nine fields, not just the interesting + * ones, is what makes a field added to SlotCensus without a corresponding + * census assignment fail here instead of reading a silent zero forever. + */ +TEST(PeerFinderSlotCensus, reports_configured_ceilings_with_no_peers) +{ + NiceMock store; + allowEmptyStore(store); + NiceMock checker; + TestStopwatch clock; + Logic> logic(clock, store, checker, journal()); + + Config config; + config.autoConnect = false; + config.listeningPort = 1024; + config.ipLimit = 2; + config.outPeers = 3; + config.inPeers = 5; + config.wantIncoming = true; + logic.config(config); + + auto const census = logic.getSlotCensus(); + + // The ceilings come straight from the config, so both are exact. + EXPECT_EQ(census.outMax, 3); + EXPECT_EQ(census.inMax, 5); + + // Nothing is connected, attempted or cached yet. Every occupancy field is + // exactly zero -- not merely "small" -- which is the state the dashboard + // must show as idle rather than as a slot shortage. + EXPECT_EQ(census.outActive, 0); + EXPECT_EQ(census.inActive, 0); + EXPECT_EQ(census.connecting, 0); + EXPECT_EQ(census.fixedConfigured, 0); + EXPECT_EQ(census.fixedActive, 0); + EXPECT_EQ(census.bootcache, 0); + EXPECT_EQ(census.livecache, 0); +} + +/** + * Inbound slots are reported as configured only when inbound is wanted. + * + * Counts::onConfig() assigns inMax_ only under `wantIncoming`, so a node with + * inbound disabled must publish inMax == 0 even though config.inPeers is + * non-zero. This is the negative half of the ceiling reading: without it, a + * census that copied config.inPeers directly would report five free inbound + * slots on a node that can never accept one, and the "inbound saturated" + * panel would read 0/5 forever instead of 0/0. + */ +TEST(PeerFinderSlotCensus, reports_zero_inbound_ceiling_when_inbound_disabled) +{ + NiceMock store; + allowEmptyStore(store); + NiceMock checker; + TestStopwatch clock; + Logic> logic(clock, store, checker, journal()); + + Config config; + config.autoConnect = false; + config.listeningPort = 1024; + config.ipLimit = 2; + config.outPeers = 3; + config.inPeers = 5; + config.wantIncoming = false; + logic.config(config); + + auto const census = logic.getSlotCensus(); + + // inPeers was 5, but inbound is off: the census must report the effective + // ceiling the slot logic will actually enforce, which is 0. + EXPECT_EQ(census.inMax, 0); + + // The outbound ceiling is unaffected, which is what proves the zero above + // is the wantIncoming rule rather than the config being ignored wholesale. + EXPECT_EQ(census.outMax, 3); +} + +/** + * Occupancy fields follow slot state transitions, and each of the three + * outbound-related fields answers a different question. + * + * `connecting`, `outActive` and `fixedActive` are three separate readings of + * the same connection at three different moments, and Counts deliberately + * keeps a fixed peer out of `outActive` (see Counts::adjust, which guards the + * outActive_/inActive_ branch with `!s.fixed() && !s.reserved()`). Walking one + * fixed and one ordinary peer through dial -> connect -> handshake -> close + * and asserting the exact value of every field at each step is what pins that + * accounting down: a census that read the wrong Counts accessor would still + * produce plausible-looking numbers in any single state. + */ +TEST(PeerFinderSlotCensus, tracks_slot_occupancy_through_the_connection_lifecycle) +{ + NiceMock store; + allowEmptyStore(store); + NiceMock checker; + TestStopwatch clock; + Logic> logic(clock, store, checker, journal()); + + Config config; + config.autoConnect = false; + config.listeningPort = 1024; + config.ipLimit = 2; + config.outPeers = 3; + config.inPeers = 5; + config.wantIncoming = true; + logic.config(config); + + auto const fixedEndpoint = endpoint("65.0.0.1:5"); + logic.addFixedPeer("fixed-one", fixedEndpoint); + + // Configured but not yet dialed: the asymmetry the fixed pair exists to + // show. fixedConfigured comes from fixed_.size() and is already 1; + // fixedActive comes from Counts and is still 0. + { + auto const census = logic.getSlotCensus(); + EXPECT_EQ(census.fixedConfigured, 1); + EXPECT_EQ(census.fixedActive, 0); + EXPECT_EQ(census.connecting, 0); + EXPECT_EQ(census.outActive, 0); + } + + // Dial the fixed peer. The attempt is in flight, so it counts as + // `connecting` and as neither `outActive` nor `fixedActive`. A census that + // conflated dialing with connected would hide a node stuck redialing. + auto const [fixedSlot, fixedResult] = logic.newOutboundSlot(fixedEndpoint); + ASSERT_NE(fixedSlot, nullptr); + ASSERT_EQ(fixedResult, Result::Success); + { + auto const census = logic.getSlotCensus(); + EXPECT_EQ(census.connecting, 1); + EXPECT_EQ(census.outActive, 0); + EXPECT_EQ(census.fixedActive, 0); + EXPECT_EQ(census.fixedConfigured, 1); + } + + // TCP is up but the handshake is not done. The slot moves Connect -> + // Connected, both of which Counts still tallies as an attempt, so + // `connecting` must hold at 1 rather than moving early. + ASSERT_TRUE(logic.onConnected(fixedSlot, endpoint("65.0.0.200:1024"))); + { + auto const census = logic.getSlotCensus(); + EXPECT_EQ(census.connecting, 1); + EXPECT_EQ(census.outActive, 0); + EXPECT_EQ(census.fixedActive, 0); + } + + // Handshake done. `connecting` drains and `fixedActive` rises -- but + // `outActive` stays at 0, because Counts excludes fixed peers from the + // ordinary outbound tally. This is the one reading most easily got wrong: + // fixed peers occupy a connection without consuming an outbound slot, so + // an outActive that moved here would overstate slot pressure by exactly + // the number of fixed peers. + PublicKey const fixedKey(randomKeyPair(KeyType::Secp256k1).first); + ASSERT_EQ(logic.activate(fixedSlot, fixedKey, false), Result::Success); + { + auto const census = logic.getSlotCensus(); + EXPECT_EQ(census.connecting, 0); + EXPECT_EQ(census.fixedActive, 1); + EXPECT_EQ(census.outActive, 0); + EXPECT_EQ(census.fixedConfigured, 1); + + // Inbound is untouched by outbound activity, so the two halves of the + // census cannot be cross-contaminated. + EXPECT_EQ(census.inActive, 0); + EXPECT_EQ(census.inMax, 5); + } + + // An ordinary (non-fixed, non-reserved) outbound peer is what does consume + // an outbound slot. Activating it must move outActive to 1 and leave + // fixedActive at 1 -- the positive counterpart of the assertion above. + auto const [outSlot, outResult] = logic.newOutboundSlot(endpoint("65.0.0.3:7")); + ASSERT_NE(outSlot, nullptr); + ASSERT_EQ(outResult, Result::Success); + ASSERT_TRUE(logic.onConnected(outSlot, endpoint("65.0.0.201:1024"))); + PublicKey const outKey(randomKeyPair(KeyType::Secp256k1).first); + ASSERT_EQ(logic.activate(outSlot, outKey, false), Result::Success); + { + auto const census = logic.getSlotCensus(); + EXPECT_EQ(census.outActive, 1); + EXPECT_EQ(census.fixedActive, 1); + EXPECT_EQ(census.connecting, 0); + EXPECT_EQ(census.outMax, 3); + } + + // Accept an inbound peer and activate it. inActive must move while + // outActive and fixedActive both hold, and this peer is not fixed, so + // fixedActive must NOT move -- the negative assertion that fixedActive + // tracks the fixed set rather than total activations. + auto const [inSlot, inResult] = + logic.newInboundSlot(endpoint("65.0.0.200:1024"), endpoint("65.0.0.4:8")); + ASSERT_NE(inSlot, nullptr); + ASSERT_EQ(inResult, Result::Success); + + // Accepted-but-not-handshaked is not yet an active inbound peer either. + EXPECT_EQ(logic.getSlotCensus().inActive, 0); + + PublicKey const inKey(randomKeyPair(KeyType::Secp256k1).first); + ASSERT_EQ(logic.activate(inSlot, inKey, false), Result::Success); + { + auto const census = logic.getSlotCensus(); + EXPECT_EQ(census.inActive, 1); + EXPECT_EQ(census.outActive, 1); + EXPECT_EQ(census.fixedActive, 1); + } + + // Close the fixed peer. fixedActive drains to 0 while fixedConfigured + // stays at 1: the peer is still configured, it is just no longer + // connected. That 1-versus-0 pair is exactly the "configured fixed peer I + // cannot reach" state, and it is the reason both fields are published. + // outActive and inActive are untouched, so closing one slot cannot skew + // the others. + logic.onClosed(fixedSlot); + { + auto const census = logic.getSlotCensus(); + EXPECT_EQ(census.fixedActive, 0); + EXPECT_EQ(census.fixedConfigured, 1); + EXPECT_EQ(census.outActive, 1); + EXPECT_EQ(census.inActive, 1); + } + + // Close the remaining two. Every occupancy field returns to exactly zero, + // which proves the census reads live counters rather than accumulating + // totals that would never come back down. + logic.onClosed(outSlot); + logic.onClosed(inSlot); + { + auto const census = logic.getSlotCensus(); + EXPECT_EQ(census.outActive, 0); + EXPECT_EQ(census.inActive, 0); + EXPECT_EQ(census.connecting, 0); + EXPECT_EQ(census.fixedActive, 0); + + // The ceilings and the configured fixed set are properties of the + // config, not of any connection, so they must survive all of it. + EXPECT_EQ(census.outMax, 3); + EXPECT_EQ(census.inMax, 5); + EXPECT_EQ(census.fixedConfigured, 1); + } +} + +/** + * Cache depths are reported from the caches themselves, independently. + * + * bootcache and livecache are separate stores with different lifetimes + * (persisted versus session-only), and the census reports both so an operator + * can tell a node that has bootstrap addresses but has learned nothing from + * peers apart from one that has neither. Populating exactly one of the two + * and asserting the other stays at zero is what proves they are read from + * distinct sources. + */ +TEST(PeerFinderSlotCensus, reports_bootcache_and_livecache_depths_separately) +{ + NiceMock store; + allowEmptyStore(store); + NiceMock checker; + TestStopwatch clock; + Logic> logic(clock, store, checker, journal()); + + Config config; + config.autoConnect = false; + config.listeningPort = 1024; + config.ipLimit = 2; + logic.config(config); + + ASSERT_EQ(logic.getSlotCensus().bootcache, 0); + ASSERT_EQ(logic.getSlotCensus().livecache, 0); + + // Two distinct bootstrap addresses. + EXPECT_TRUE(logic.bootcache.insert(endpoint("65.0.0.1:10001"))); + EXPECT_TRUE(logic.bootcache.insert(endpoint("65.0.0.2:10002"))); + { + auto const census = logic.getSlotCensus(); + EXPECT_EQ(census.bootcache, 2); + // The session cache is a different store and must still read zero. + EXPECT_EQ(census.livecache, 0); + } + + // One address learned from a peer this session. + logic.livecache.insert(Endpoint{endpoint("65.0.0.3:10003"), 2}); + { + auto const census = logic.getSlotCensus(); + EXPECT_EQ(census.livecache, 1); + // The persisted cache is unchanged by the livecache insert. + EXPECT_EQ(census.bootcache, 2); + } +} + } // namespace } // namespace xrpl::PeerFinder diff --git a/src/tests/libxrpl/telemetry/ConsensusSpanNames.cpp b/src/tests/libxrpl/telemetry/ConsensusSpanNames.cpp new file mode 100644 index 00000000000..cc475560fa1 --- /dev/null +++ b/src/tests/libxrpl/telemetry/ConsensusSpanNames.cpp @@ -0,0 +1,188 @@ +/** + * @file ConsensusSpanNames.cpp + * Unit tests for the consensus.validation.accept span contract in + * ConsensusSpanNames.h (WP-B3). + * + * Two things are pinned here: + * + * 1. The span name and attribute keys, asserted literally. They are a + * cross-component contract with no compile-time link between the sides: + * the collector aggregates the `validation_status` and `accept_gated` + * spanmetrics dimensions on these exact strings, Tempo indexes them as + * searchable tags, and the workload validator names the span in + * expected_spans.json. A silent rename would break every one of those with + * no compile error, so the values are asserted as strings. + * + * 2. `validationStatusValue()`, the rule that turns a `ValStatus` into the + * attribute value. The caller passes `static_cast(status)`, so the + * ENUMERATOR ORDER is part of the contract: asserting the whole domain here + * means inserting a `ValStatus` enumerator without updating the mapping + * fails this test rather than silently relabelling live spans. The rule is a + * pure constexpr function with no dependency on the validation store, so it + * is asserted directly -- no Application, no validations container, and no + * test-only hook added to production code to reach it. + * + * Compiled only when XRPL_ENABLE_TELEMETRY is defined, which is the configuration + * that builds the telemetry test target. The header itself is not + * telemetry-conditional (constants and one constexpr function, no OTel types), + * and since it now lives at a libxrpl test can include it + * directly without reaching into `src/`. + */ + +#ifdef XRPL_ENABLE_TELEMETRY + +#include + +#include + +#include + +#include +#include +#include + +using namespace xrpl::telemetry; + +namespace { + +// The full span name is what hashSpan receives, what the collector keys span +// metrics on, and what expected_spans.json asserts. Check the composed value and +// both halves, so neither the segment nor the suffix can drift unnoticed. +TEST(ConsensusSpanNames, validation_accept_span_name_is_fully_composed) +{ + EXPECT_EQ(std::string_view(consensus::span::validationAccept), "consensus.validation.accept"); + EXPECT_EQ(std::string_view(seg::consensus), "consensus"); + EXPECT_EQ(std::string_view(consensus::span::op::validationAccept), "validation.accept"); +} + +// validation.accept and validation.receive are DIFFERENT events: receive is the +// overlay decoding a validation message on a peer thread; accept is the later, +// per-ledger step where a trusted validation drives the acceptance gate. They +// live in different traces (accept is keyed on the validated ledger hash), so +// collapsing the two names would merge two unrelated signals. +TEST(ConsensusSpanNames, validation_accept_is_distinct_from_validation_receive) +{ + EXPECT_NE( + std::string_view(consensus::span::validationAccept), + std::string_view(consensus::span::validationReceive)); + EXPECT_EQ(std::string_view(consensus::span::validationReceive), "consensus.validation.receive"); + // Nor is it the send-side span, which is this node emitting its own. + EXPECT_NE( + std::string_view(consensus::span::validationAccept), + std::string_view(consensus::span::validationSend)); +} + +// The two attribute keys are bare lower_snake_case, never dotted: the dotted +// xrpl.. form is reserved for resource attributes and fails the +// naming check (Rule A), and a non-snake_case key fails Rule G. +TEST(ConsensusSpanNames, validation_accept_attribute_keys_are_bare_underscore) +{ + std::array const keys{ + consensus::span::attr::validationStatus, consensus::span::attr::acceptGated}; + for (auto const key : keys) + { + EXPECT_EQ(key.find('.'), std::string_view::npos) << "dotted span attr key: " << key; + EXPECT_FALSE(key.empty()); + for (char const c : key) + EXPECT_TRUE((c >= 'a' && c <= 'z') || c == '_') << "bad char in key: " << key; + } + // Exact values: these are the collector dimension and Tempo tag names. + EXPECT_EQ(std::string_view(consensus::span::attr::validationStatus), "validation_status"); + EXPECT_EQ(std::string_view(consensus::span::attr::acceptGated), "accept_gated"); +} + +// The status values are aggregated as a spanmetrics dimension, so each must be +// lower_snake_case too. `bad_seq` is the one that matters: to_string(ValStatus) +// returns camelCase "badSeq", which is why the values are spelled out in the +// header rather than reusing that function. +TEST(ConsensusSpanNames, validation_status_values_are_lower_snake_case) +{ + std::array const values{ + consensus::span::val::statusCurrent, + consensus::span::val::statusStale, + consensus::span::val::statusBadSeq, + consensus::span::val::statusMultiple, + consensus::span::val::statusConflicting, + consensus::span::val::statusUnknown}; + for (auto const value : values) + { + EXPECT_FALSE(value.empty()); + for (char const c : value) + EXPECT_TRUE((c >= 'a' && c <= 'z') || c == '_') << "bad char in value: " << value; + } + EXPECT_EQ(std::string_view(consensus::span::val::statusBadSeq), "bad_seq"); +} + +// Every value distinct: two ValStatus outcomes sharing a string would silently +// merge two series on the aggregated dimension. +TEST(ConsensusSpanNames, validation_status_values_are_mutually_distinct) +{ + std::array const values{ + consensus::span::val::statusCurrent, + consensus::span::val::statusStale, + consensus::span::val::statusBadSeq, + consensus::span::val::statusMultiple, + consensus::span::val::statusConflicting, + consensus::span::val::statusUnknown}; + for (std::size_t i = 0; i < values.size(); ++i) + { + for (std::size_t k = i + 1; k < values.size(); ++k) + EXPECT_NE(values[i], values[k]) << "duplicate status value at " << i << "," << k; + } +} + +// THE RULE, over its whole live domain. The int arguments are +// static_cast(ValStatus), so this also pins the enumerator order: Current, +// Stale, BadSeq, Multiple, Conflicting. +TEST(ConsensusSpanNames, validationStatusValue_maps_every_val_status) +{ + EXPECT_EQ(consensus::span::validationStatusValue(0), consensus::span::val::statusCurrent); + EXPECT_EQ(consensus::span::validationStatusValue(1), consensus::span::val::statusStale); + EXPECT_EQ(consensus::span::validationStatusValue(2), consensus::span::val::statusBadSeq); + EXPECT_EQ(consensus::span::validationStatusValue(3), consensus::span::val::statusMultiple); + EXPECT_EQ(consensus::span::validationStatusValue(4), consensus::span::val::statusConflicting); +} + +// Only `current` continues to the acceptance gate; the other four mean the +// validation was counted for nothing. That split is the diagnostic value of the +// attribute -- a node stuck below quorum receiving validations that are all +// stale or bad-seq looks, from the outside, exactly like one receiving good ones. +TEST(ConsensusSpanNames, validationStatusValue_separates_counted_from_rejected) +{ + EXPECT_EQ(consensus::span::validationStatusValue(0), "current"); + for (int const rejected : {1, 2, 3, 4}) + { + EXPECT_NE(consensus::span::validationStatusValue(rejected), "current") + << "input: " << rejected; + } +} + +// NEGATIVE: an out-of-domain value yields the sentinel, never an empty string. +// An empty attribute value would add a blank series to the aggregated dimension, +// which is worse than a value labelled "unknown". +TEST(ConsensusSpanNames, validationStatusValue_out_of_domain_is_unknown_not_empty) +{ + for (int const bad : {-1, 5, 6, 99}) + { + EXPECT_EQ(consensus::span::validationStatusValue(bad), consensus::span::val::statusUnknown) + << "input: " << bad; + EXPECT_FALSE(consensus::span::validationStatusValue(bad).empty()); + } +} + +// The mapping is a compile-time rule, so a wrong value cannot even be built -- +// the strongest form of the guarantee, checked by the compiler rather than at +// run time. +TEST(ConsensusSpanNames, validationStatusValue_is_a_compile_time_rule) +{ + static_assert(consensus::span::validationStatusValue(0) == "current"); + static_assert(consensus::span::validationStatusValue(2) == "bad_seq"); + static_assert(consensus::span::validationStatusValue(4) == "conflicting"); + static_assert(consensus::span::validationStatusValue(7) == "unknown"); + static_assert(consensus::span::validationStatusValue(-1) == "unknown"); + SUCCEED(); +} + +} // namespace + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/tests/libxrpl/telemetry/LedgerSpanNames.cpp b/src/tests/libxrpl/telemetry/LedgerSpanNames.cpp new file mode 100644 index 00000000000..ce19c7e8f60 --- /dev/null +++ b/src/tests/libxrpl/telemetry/LedgerSpanNames.cpp @@ -0,0 +1,681 @@ +/** + * @file LedgerSpanNames.cpp + * Unit tests for the sync-diagnostic span contracts in LedgerSpanNames.h and + * PeerSpanNames.h: `ledger.acquire` and, in the WP-B2 block at the end of this + * file, `txset.acquire`, the three `ledger.acquire.{header,astree,txtree}` + * phase children, `ledger.serve` and `peer.dial`. + * + * Two things are pinned here: + * + * 1. The literal attribute keys and outcome values. They are a cross-component + * contract: the collector aggregates the spanmetrics `outcome` dimension on + * these exact strings, Tempo indexes `ledger_hash` as a dedicated span + * column, and the workload validator asserts them by name in + * expected_spans.json. A silent rename would break every one of those with + * no compile error, so the values are asserted literally. + * + * 2. `acquireOutcome()`, the rule behind "a ledger.acquire span always carries + * an outcome". InboundLedger has four exits -- done(), the local-store + * shortcut, the "can never be acquired" exit, and the destructor when the + * sweeper drops a fetch -- and every one derives its value from this + * function. Asserting the function over its whole input domain therefore + * asserts the outcome of every exit path, including the destructor path, + * which is the case that previously produced a span with no outcome at all. + * The rule is a pure constexpr function with no dependency on + * InboundLedger, so it is asserted directly here: no Application, no peer + * set, and no test-only hook added to production code to reach it. + * + * The WP-B2 spans follow the same two rules, with three more pure functions + * standing in for their emitters' exits: `phaseOutcome()` for the acquire + * phases and tx-set fetch, and `serveObjectType()` / `serveOutcome()` for the + * eight exits of PeerImp::processLedgerRequest. Every one is asserted over its + * whole input domain, which is what proves no exit can leave a span without an + * outcome -- the property the emitters rely on and that no compiler enforces. + * + * Compiled only when XRPL_ENABLE_TELEMETRY is defined, because that is the + * configuration in which this test target has `src/` on its include path and + * can therefore reach and + * . Neither header is telemetry-conditional + * (constants and constexpr functions, no OTel types); only this file's ability + * to include them is. + */ + +#ifdef XRPL_ENABLE_TELEMETRY + +#include + +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include + +using namespace xrpl::telemetry; + +namespace { + +TEST(LedgerSpanNames, acquire_span_name_is_dot_qualified) +{ + // InboundLedger::init builds the span name as seg::ledger + "." + this + // suffix, which is the "ledger.acquire" every dashboard and the validator + // query by. Assert both halves so the composed name cannot drift. + EXPECT_EQ(std::string_view(seg::ledger), "ledger"); + EXPECT_EQ(std::string_view(ledger_span::op::acquire), "acquire"); +} + +TEST(LedgerSpanNames, acquire_attribute_keys_match_collector_and_tempo) +{ + // `outcome` and `acquire_reason` are the two spanmetrics dimensions listed + // under "# ledger.acquire dimensions" in BOTH collector configs. + EXPECT_EQ(std::string_view(ledger_span::attr::outcome), "outcome"); + EXPECT_EQ(std::string_view(ledger_span::attr::acquireReason), "acquire_reason"); + // Span-only attributes: asserted by expected_spans.json, and ledger_hash is + // the dedicated Parquet span column in tempo.yaml. + EXPECT_EQ(std::string_view(ledger_span::attr::ledgerHash), "ledger_hash"); + EXPECT_EQ(std::string_view(ledger_span::attr::ledgerSeq), "ledger_seq"); + EXPECT_EQ(std::string_view(ledger_span::attr::timeouts), "timeouts"); + EXPECT_EQ(std::string_view(ledger_span::attr::peerCount), "peer_count"); +} + +TEST(LedgerSpanNames, attribute_keys_are_bare_underscore_never_dotted) +{ + // The naming spec reserves dotted keys for resource attributes; a dotted + // span-attribute key fails the CI naming check. Assert the property, not + // just the spelling, so a future key added here is covered too. + for (std::string_view const key : + {std::string_view(ledger_span::attr::outcome), + std::string_view(ledger_span::attr::acquireReason), + std::string_view(ledger_span::attr::ledgerHash), + std::string_view(ledger_span::attr::ledgerSeq), + std::string_view(ledger_span::attr::timeouts), + std::string_view(ledger_span::attr::peerCount)}) + { + EXPECT_EQ(key.find('.'), std::string_view::npos) << "dotted span-attr key: " << key; + EXPECT_FALSE(key.empty()); + } +} + +TEST(LedgerSpanNames, outcome_values_are_the_three_terminal_states) +{ + // These become the `outcome` dimension's value set (cardinality 3), which + // is what makes it safe as a metric dimension. + EXPECT_EQ(std::string_view(ledger_span::val::complete), "complete"); + EXPECT_EQ(std::string_view(ledger_span::val::failed), "failed"); + EXPECT_EQ(std::string_view(ledger_span::val::abandoned), "abandoned"); +} + +TEST(LedgerSpanNames, outcome_values_are_mutually_distinct) +{ + // Cause, not just state: the panel splits acquisitions by this attribute, + // so two outcomes sharing a value would silently merge two different + // failure modes into one line. + EXPECT_NE( + std::string_view(ledger_span::val::abandoned), + std::string_view(ledger_span::val::complete)); + EXPECT_NE( + std::string_view(ledger_span::val::abandoned), std::string_view(ledger_span::val::failed)); + EXPECT_NE( + std::string_view(ledger_span::val::complete), std::string_view(ledger_span::val::failed)); +} + +TEST(LedgerSpanNames, acquire_reason_values_mirror_the_reason_enum) +{ + // One value per InboundLedger::Reason, mapped by the switch in init(). + EXPECT_EQ(std::string_view(ledger_span::val::history), "history"); + EXPECT_EQ(std::string_view(ledger_span::val::consensus), "consensus"); + EXPECT_EQ(std::string_view(ledger_span::val::generic), "generic"); +} + +TEST(LedgerSpanNames, acquireOutcome_normal_done_path_is_complete) +{ + // done() after all data was assembled: complete_ set, failed_ clear. + EXPECT_EQ(ledger_span::acquireOutcome(/*failed=*/false, /*complete=*/true), "complete"); +} + +TEST(LedgerSpanNames, acquireOutcome_local_complete_path_is_complete) +{ + // The tryDB local-store shortcut in init() reaches the same flag state as + // done(), so it must produce the same outcome -- this is the exit that used + // to end the span with no outcome at all. + EXPECT_EQ(ledger_span::acquireOutcome(/*failed=*/false, /*complete=*/true), "complete"); +} + +TEST(LedgerSpanNames, acquireOutcome_failed_path_is_failed) +{ + // Terminal error: bad data, a zero account hash, or the retry budget ran + // out. Reached from done() and from the early-return in init(). + EXPECT_EQ(ledger_span::acquireOutcome(/*failed=*/true, /*complete=*/false), "failed"); +} + +TEST(LedgerSpanNames, acquireOutcome_abort_path_is_abandoned) +{ + // The destructor / sweep path: neither flag set, because the fetch never + // reached a result. This is the assertion the whole change exists for -- an + // acquire swept while stuck is still counted, with `abandoned` naming why. + EXPECT_EQ(ledger_span::acquireOutcome(/*failed=*/false, /*complete=*/false), "abandoned"); +} + +TEST(LedgerSpanNames, acquireOutcome_failure_wins_over_completion) +{ + // Edge case: both flags set. A fetch that hit a terminal error is not a + // success regardless of what was assembled, so `failed` must win. Pinned + // because flipping the precedence would quietly relabel real failures as + // successes and hide them from the outcome rate. + EXPECT_EQ(ledger_span::acquireOutcome(/*failed=*/true, /*complete=*/true), "failed"); +} + +TEST(LedgerSpanNames, acquireOutcome_covers_its_whole_input_domain) +{ + // No input combination yields an empty or unknown value, which is the + // property that guarantees an exit path can never end up with a blank + // outcome. Also asserts every result is one of the three declared values, + // so the spanmetrics dimension can never gain an unexpected fourth. + for (bool const failed : {false, true}) + { + for (bool const complete : {false, true}) + { + auto const outcome = ledger_span::acquireOutcome(failed, complete); + EXPECT_FALSE(outcome.empty()) << "failed=" << failed << " complete=" << complete; + EXPECT_TRUE( + outcome == std::string_view(ledger_span::val::complete) || + outcome == std::string_view(ledger_span::val::failed) || + outcome == std::string_view(ledger_span::val::abandoned)) + << "undeclared outcome '" << outcome << "' for failed=" << failed + << " complete=" << complete; + } + } +} + +TEST(LedgerSpanNames, acquireOutcome_is_a_compile_time_rule) +{ + // constexpr, so the rule costs nothing at the four call sites and can be + // asserted by the compiler itself. static_assert here is the strongest + // available statement that the mapping is fixed. + static_assert(ledger_span::acquireOutcome(false, true) == std::string_view("complete")); + static_assert(ledger_span::acquireOutcome(true, false) == std::string_view("failed")); + static_assert(ledger_span::acquireOutcome(false, false) == std::string_view("abandoned")); + static_assert(ledger_span::acquireOutcome(true, true) == std::string_view("failed")); + SUCCEED(); +} + +TEST(LedgerSpanNames, inactive_guard_finalize_sequence_is_a_no_op) +{ + // Negative / disabled path. A default-constructed SpanGuard is the exact + // state InboundLedger::acquireSpan_ holds when telemetry is off or the + // ledger trace category is disabled: `operator bool()` is false and every + // setter is inert. Drive the whole finalize sequence against it -- the same + // calls, in the same order, that finalizeAcquireSpan() makes -- and assert + // the guard stays inactive and nothing crashes. This is what proves the + // added abort-path finalization emits nothing on a node with telemetry + // disabled, including from the destructor. + SpanGuard guard; + ASSERT_FALSE(static_cast(guard)); + + guard.setAttribute(ledger_span::attr::ledgerHash, "0123456789ABCDEF"); + guard.setAttribute(ledger_span::attr::ledgerSeq, static_cast(12345)); + guard.setAttribute(ledger_span::attr::acquireReason, ledger_span::val::history); + guard.setAttribute( + ledger_span::attr::outcome, + ledger_span::acquireOutcome(/*failed=*/false, /*complete=*/false)); + guard.setAttribute(ledger_span::attr::timeouts, static_cast(6)); + guard.setAttribute(ledger_span::attr::peerCount, static_cast(0)); + + // Still inactive: no span was created, so none can be exported. + EXPECT_FALSE(static_cast(guard)); + // activateIfLive on an empty handle yields a null activation, which is the + // path both the destructor's abort log and done()'s outcome log take when + // telemetry is disabled. + std::optional const empty; + { + auto const activation = activateIfLive(empty); + } + EXPECT_FALSE(empty.has_value()); +} + +// =========================================================================== +// WP-B2 — the new sync-diagnostic spans +// +// Same contract as the ledger.acquire block above and asserted the same way: +// the wire names and attribute keys are pinned literally because they are a +// cross-component contract (the collector aggregates on them, Tempo indexes +// them, expected_spans.json asserts them by name, and a dashboard PromQL +// selector matches them -- a rename would break all four with no compile +// error), and the outcome rules are pinned over their whole input domain +// because they are what guarantees every exit path of every new span records +// an outcome. +// =========================================================================== + +TEST(LedgerSpanNames, txset_acquire_span_name_is_dot_qualified) +{ + // TransactionAcquire::init builds the name as prefix::txset + "." + + // op::acquire. A tx set is not a ledger, so it gets its own root segment + // rather than hiding under `ledger.` -- assert both halves so the composed + // "txset.acquire" cannot drift from what the dashboard queries. + EXPECT_EQ(std::string_view(ledger_span::prefix::txset), "txset"); + EXPECT_EQ(std::string_view(ledger_span::op::acquire), "acquire"); +} + +TEST(LedgerSpanNames, phase_child_span_names_are_fully_composed) +{ + // These are used with childSpan(name, ctx), which takes ONE complete name, + // so unlike the parent they are pre-joined here. Assert the exact composed + // strings: they are what the phase-duration panel selects on and what + // expected_spans.json lists. + EXPECT_EQ(std::string_view(ledger_span::acquireHeader), "ledger.acquire.header"); + EXPECT_EQ(std::string_view(ledger_span::acquireAsTree), "ledger.acquire.astree"); + EXPECT_EQ(std::string_view(ledger_span::acquireTxTree), "ledger.acquire.txtree"); +} + +TEST(LedgerSpanNames, phase_child_span_names_are_children_of_the_acquire_name) +{ + // The naming property, not just the spelling: each phase name must extend + // the parent's "ledger.acquire" exactly, because that shared prefix is what + // the panel's span_name=~"ledger.acquire..*" selector relies on to pick up + // all three phases and no other span. + auto const parent = std::string_view("ledger.acquire"); + for (std::string_view const phase : + {std::string_view(ledger_span::acquireHeader), + std::string_view(ledger_span::acquireAsTree), + std::string_view(ledger_span::acquireTxTree)}) + { + EXPECT_TRUE(phase.starts_with(parent)) << "phase not under the parent name: " << phase; + // A '.' immediately after the parent, and a non-empty leaf after that. + ASSERT_GT(phase.size(), parent.size() + 1); + EXPECT_EQ(phase[parent.size()], '.'); + EXPECT_FALSE(phase.substr(parent.size() + 1).empty()); + // The leaf must be one segment: a further dot would make the panel's + // selector pick up a grandchild that does not exist. + EXPECT_EQ(phase.substr(parent.size() + 1).find('.'), std::string_view::npos); + } +} + +TEST(LedgerSpanNames, phase_child_span_names_are_mutually_distinct) +{ + // Cause, not just state: the duration panel plots one series per phase, so + // two phases sharing a name would silently merge the account-state tree + // (nearly all of a fresh sync) into another phase's line. + EXPECT_NE( + std::string_view(ledger_span::acquireHeader), std::string_view(ledger_span::acquireAsTree)); + EXPECT_NE( + std::string_view(ledger_span::acquireHeader), std::string_view(ledger_span::acquireTxTree)); + EXPECT_NE( + std::string_view(ledger_span::acquireAsTree), std::string_view(ledger_span::acquireTxTree)); +} + +TEST(LedgerSpanNames, serve_span_name_is_dot_qualified) +{ + EXPECT_EQ(std::string_view(seg::ledger), "ledger"); + EXPECT_EQ(std::string_view(ledger_span::op::serve), "serve"); +} + +TEST(LedgerSpanNames, b2_attribute_keys_match_collector_and_tempo) +{ + // `timed_out` and `object_type` are the two NEW spanmetrics dimensions + // listed in BOTH collector configs; the rest are span-only. + EXPECT_EQ(std::string_view(ledger_span::attr::timedOut), "timed_out"); + EXPECT_EQ(std::string_view(ledger_span::attr::objectType), "object_type"); + // Span-only, asserted by expected_spans.json. txset_hash is additionally a + // dedicated Parquet span column in tempo.yaml, for the same per-object + // reason ledger_hash is: it identifies WHICH set stalled, and as a metric + // dimension it would mint one series per consensus round. + EXPECT_EQ(std::string_view(ledger_span::attr::txSetHash), "txset_hash"); + EXPECT_EQ(std::string_view(ledger_span::attr::missingNodes), "missing_nodes"); + EXPECT_EQ(std::string_view(ledger_span::attr::servedNodes), "served_nodes"); + EXPECT_EQ(std::string_view(ledger_span::attr::durationMs), "duration_ms"); +} + +TEST(LedgerSpanNames, b2_attribute_keys_are_bare_underscore_never_dotted) +{ + // The naming spec reserves dotted keys for resource attributes; a dotted + // span-attribute key fails the CI naming check. Assert the property so a + // key added to this group later is covered too. + for (std::string_view const key : + {std::string_view(ledger_span::attr::timedOut), + std::string_view(ledger_span::attr::objectType), + std::string_view(ledger_span::attr::txSetHash), + std::string_view(ledger_span::attr::missingNodes), + std::string_view(ledger_span::attr::servedNodes), + std::string_view(ledger_span::attr::durationMs)}) + { + EXPECT_EQ(key.find('.'), std::string_view::npos) << "dotted span-attr key: " << key; + EXPECT_FALSE(key.empty()); + } +} + +TEST(LedgerSpanNames, timeout_outcome_value_is_distinct_from_the_other_three) +{ + // `timeout` is the value WP-B2 adds to the outcome set the collector + // aggregates. It must be distinct from all three existing values, because + // the whole point is separating "peers never supplied the data" from + // "the data was bad" (failed) and "we stopped waiting" (abandoned). + EXPECT_EQ(std::string_view(ledger_span::val::timeout), "timeout"); + EXPECT_NE( + std::string_view(ledger_span::val::timeout), std::string_view(ledger_span::val::failed)); + EXPECT_NE( + std::string_view(ledger_span::val::timeout), std::string_view(ledger_span::val::complete)); + EXPECT_NE( + std::string_view(ledger_span::val::timeout), std::string_view(ledger_span::val::abandoned)); +} + +TEST(LedgerSpanNames, serve_object_type_values_are_the_four_request_kinds) +{ + // These become the `object_type` dimension's value set (cardinality 4), + // which is what makes it safe as a metric dimension. + EXPECT_EQ(std::string_view(ledger_span::val::header), "header"); + EXPECT_EQ(std::string_view(ledger_span::val::txTree), "tx"); + EXPECT_EQ(std::string_view(ledger_span::val::asTree), "as"); + EXPECT_EQ(std::string_view(ledger_span::val::txSet), "txset"); +} + +TEST(LedgerSpanNames, serve_outcome_values_are_the_three_terminal_states) +{ + // `complete` is shared with the acquire outcomes (same concept, told apart + // by span name); `partial` and `refused` are serve-specific. + EXPECT_EQ(std::string_view(ledger_span::val::partial), "partial"); + EXPECT_EQ(std::string_view(ledger_span::val::refused), "refused"); + EXPECT_NE( + std::string_view(ledger_span::val::partial), std::string_view(ledger_span::val::refused)); + EXPECT_NE( + std::string_view(ledger_span::val::partial), std::string_view(ledger_span::val::complete)); + EXPECT_NE( + std::string_view(ledger_span::val::refused), std::string_view(ledger_span::val::complete)); +} + +TEST(LedgerSpanNames, phaseOutcome_normal_completion_is_complete) +{ + // A phase whose tree assembled, or a tx set that arrived: complete_ set, + // nothing else. Reached from receiveNode()/trigger() for a phase and from + // done() for a tx set. + EXPECT_EQ( + ledger_span::phaseOutcome(/*failed=*/false, /*complete=*/true, /*timedOut=*/false), + "complete"); +} + +TEST(LedgerSpanNames, phaseOutcome_bad_data_is_failed) +{ + // A terminal data fault with no timeout: a peer served a tree or set that + // would not build. This is the case `timeout` must NOT absorb. + EXPECT_EQ( + ledger_span::phaseOutcome(/*failed=*/true, /*complete=*/false, /*timedOut=*/false), + "failed"); +} + +TEST(LedgerSpanNames, phaseOutcome_exhausted_budget_reports_timeout_not_failed) +{ + // THE assertion this rule exists for, and the one that would regress + // silently. Both emitters' exhausted-budget path sets timedOut_ AND + // failed_ -- failed_ is how the TimeoutCounter base stops its timer loop -- + // so if `failed` were checked first, every timeout would be relabelled as a + // data fault and the "peers are not serving this" signal would vanish + // exactly when a node is stuck. + EXPECT_EQ( + ledger_span::phaseOutcome(/*failed=*/true, /*complete=*/false, /*timedOut=*/true), + "timeout"); +} + +TEST(LedgerSpanNames, phaseOutcome_timeout_outranks_a_late_completion) +{ + // Edge case: the budget expired and the data then arrived. It still reports + // `timeout`, because the retry budget was really spent -- counting it as a + // success would hide the cost. + EXPECT_EQ( + ledger_span::phaseOutcome(/*failed=*/false, /*complete=*/true, /*timedOut=*/true), + "timeout"); +} + +TEST(LedgerSpanNames, phaseOutcome_dropped_mid_fetch_is_abandoned) +{ + // No flag at all: the object was destroyed while still fetching (the + // InboundLedger sweep, or InboundTransactions::newRound dropping a set). + // Reporting a value here is what keeps a stuck-then-swept unit in the + // outcome rate instead of vanishing from it. + EXPECT_EQ( + ledger_span::phaseOutcome(/*failed=*/false, /*complete=*/false, /*timedOut=*/false), + "abandoned"); +} + +TEST(LedgerSpanNames, phaseOutcome_covers_its_whole_input_domain) +{ + // No input combination yields an empty or undeclared value, which is the + // property that guarantees no exit can end up with a blank outcome and that + // the spanmetrics dimension can never gain an unexpected fifth value. + for (bool const failed : {false, true}) + { + for (bool const complete : {false, true}) + { + for (bool const timedOut : {false, true}) + { + auto const outcome = ledger_span::phaseOutcome(failed, complete, timedOut); + EXPECT_FALSE(outcome.empty()) + << "failed=" << failed << " complete=" << complete << " timedOut=" << timedOut; + EXPECT_TRUE( + outcome == std::string_view(ledger_span::val::complete) || + outcome == std::string_view(ledger_span::val::failed) || + outcome == std::string_view(ledger_span::val::timeout) || + outcome == std::string_view(ledger_span::val::abandoned)) + << "undeclared outcome '" << outcome << "' for failed=" << failed + << " complete=" << complete << " timedOut=" << timedOut; + // Whenever the budget expired, the answer is `timeout` + // regardless of the other two -- the precedence property, not + // just the four sampled points above. + // Braced deliberately: EXPECT_EQ expands to an if/else, so an + // unbraced if around it is a dangling else, which gcc rejects. + if (timedOut) + { + EXPECT_EQ(outcome, std::string_view(ledger_span::val::timeout)); + } + } + } + } +} + +TEST(LedgerSpanNames, phaseOutcome_is_a_compile_time_rule) +{ + // constexpr, so the rule costs nothing at its call sites and the mapping is + // fixed by the compiler itself. + static_assert(ledger_span::phaseOutcome(false, true, false) == std::string_view("complete")); + static_assert(ledger_span::phaseOutcome(true, false, false) == std::string_view("failed")); + static_assert(ledger_span::phaseOutcome(true, false, true) == std::string_view("timeout")); + static_assert(ledger_span::phaseOutcome(false, false, false) == std::string_view("abandoned")); + SUCCEED(); +} + +TEST(LedgerSpanNames, serveObjectType_maps_every_protobuf_itype) +{ + // The exact protobuf TMLedgerInfoType values, which are fixed by the wire + // protocol: liBASE=0, liTX_NODE=1, liAS_NODE=2, liTS_CANDIDATE=3. Passed as + // an int so this rule stays free of protobuf headers and assertable here. + EXPECT_EQ(ledger_span::serveObjectType(0), "header"); + EXPECT_EQ(ledger_span::serveObjectType(1), "tx"); + EXPECT_EQ(ledger_span::serveObjectType(2), "as"); + EXPECT_EQ(ledger_span::serveObjectType(3), "txset"); +} + +TEST(LedgerSpanNames, serveObjectType_never_yields_an_undeclared_value) +{ + // Edge case: an out-of-range itype cannot occur -- PeerImp::onMessage + // rejects the request before the worker runs -- but the rule must still + // produce a declared value rather than an empty attribute, so the + // object_type dimension's value set stays closed at four. + for (int const itype : {-1, 4, 99}) + { + auto const value = ledger_span::serveObjectType(itype); + EXPECT_EQ(value, std::string_view(ledger_span::val::header)) + << "unexpected fallback for itype=" << itype; + } +} + +TEST(LedgerSpanNames, serveOutcome_empty_reply_is_refused) +{ + // Seven of the eight exits of processLedgerRequest send nothing, and all of + // them reach this through a zero node count. Deriving the value from the + // reply is what makes those seven impossible to mislabel. + EXPECT_EQ(ledger_span::serveOutcome(/*servedNodes=*/0, /*softCap=*/128), "refused"); +} + +TEST(LedgerSpanNames, serveOutcome_partial_reply_below_cap_is_complete) +{ + EXPECT_EQ(ledger_span::serveOutcome(/*servedNodes=*/12, /*softCap=*/128), "complete"); + EXPECT_EQ(ledger_span::serveOutcome(/*servedNodes=*/127, /*softCap=*/128), "complete"); +} + +TEST(LedgerSpanNames, serveOutcome_reply_at_the_cap_is_partial) +{ + // Edge case at the exact boundary: the assembly loop stops here, so the + // requester must come back for the rest. Counting it as a success would + // hide the extra round trips a large tree really costs. + EXPECT_EQ(ledger_span::serveOutcome(/*servedNodes=*/128, /*softCap=*/128), "partial"); + EXPECT_EQ(ledger_span::serveOutcome(/*servedNodes=*/256, /*softCap=*/128), "partial"); +} + +TEST(LedgerSpanNames, serveOutcome_never_yields_an_undeclared_value) +{ + // Negative counts cannot occur (nodes_size() is non-negative) but must + // still map to a declared value rather than an empty attribute. + for (int const served : {-5, 0, 1, 64, 128, 4096}) + { + auto const outcome = ledger_span::serveOutcome(served, 128); + EXPECT_TRUE( + outcome == std::string_view(ledger_span::val::complete) || + outcome == std::string_view(ledger_span::val::partial) || + outcome == std::string_view(ledger_span::val::refused)) + << "undeclared serve outcome '" << outcome << "' for servedNodes=" << served; + } +} + +TEST(LedgerSpanNames, serveOutcome_is_a_compile_time_rule) +{ + static_assert(ledger_span::serveOutcome(0, 128) == std::string_view("refused")); + static_assert(ledger_span::serveOutcome(1, 128) == std::string_view("complete")); + static_assert(ledger_span::serveOutcome(128, 128) == std::string_view("partial")); + SUCCEED(); +} + +TEST(LedgerSpanNames, peer_dial_span_name_is_dot_qualified) +{ + // ConnectAttempt::run builds the name as seg::peer + "." + op::dial. + EXPECT_EQ(std::string_view(seg::peer), "peer"); + EXPECT_EQ(std::string_view(peer_span::op::dial), "dial"); +} + +TEST(LedgerSpanNames, peer_dial_attribute_keys_are_bare_underscore) +{ + // remote_endpoint is the dedicated Parquet span column in tempo.yaml and + // is deliberately NOT a spanmetrics dimension: one series per peer address + // would be unbounded cardinality. + EXPECT_EQ(std::string_view(peer_span::attr::remoteEndpoint), "remote_endpoint"); + EXPECT_EQ(std::string_view(peer_span::attr::durationMs), "duration_ms"); + EXPECT_EQ(std::string_view(peer_span::attr::outcome), "outcome"); + for (std::string_view const key : + {std::string_view(peer_span::attr::remoteEndpoint), + std::string_view(peer_span::attr::durationMs), + std::string_view(peer_span::attr::outcome)}) + { + EXPECT_EQ(key.find('.'), std::string_view::npos) << "dotted span-attr key: " << key; + } +} + +TEST(LedgerSpanNames, peer_dial_outcome_values_match_the_counter_label_set) +{ + // These six ARE the values ConnectAttempt::reportOutcome passes to the + // overlay_connect_total counter -- the span and the counter read the same + // constants from the same funnel, which is what stops them drifting apart. + // Pinned literally because the Bootstrap-row dial panel and the runbook + // both name them. + EXPECT_EQ(std::string_view(peer_span::val::connected), "connected"); + EXPECT_EQ(std::string_view(peer_span::val::tcpFail), "tcp_fail"); + EXPECT_EQ(std::string_view(peer_span::val::tlsFail), "tls_fail"); + EXPECT_EQ(std::string_view(peer_span::val::upgradeFail), "upgrade_fail"); + EXPECT_EQ(std::string_view(peer_span::val::timeout), "timeout"); + + // Reuses the slug handshake_negotiation_fail_total already publishes for the + // same fault, so one misconfiguration reads identically on both signals. + EXPECT_EQ(std::string_view(peer_span::val::selfConnection), "self_connection"); +} + +TEST(LedgerSpanNames, peer_dial_outcome_values_are_mutually_distinct) +{ + // The dial panel splits by this attribute, so two outcomes sharing a value + // would merge two different failure stages into one line -- and the stage + // is the whole diagnostic content of the dial signal. + std::array const values{ + peer_span::val::connected, + peer_span::val::tcpFail, + peer_span::val::tlsFail, + peer_span::val::selfConnection, + peer_span::val::upgradeFail, + peer_span::val::timeout}; + for (std::size_t i = 0; i < values.size(); ++i) + { + EXPECT_FALSE(values[i].empty()); + for (std::size_t j = i + 1; j < values.size(); ++j) + EXPECT_NE(values[i], values[j]) << "duplicate dial outcome at " << i << "," << j; + } +} + +TEST(LedgerSpanNames, b2_inactive_guard_finalize_sequences_are_no_ops) +{ + // Negative / disabled path for all four new spans. A default-constructed + // SpanGuard is the exact state TransactionAcquire::acquireSpan_, + // InboundLedger's three phase handles and ConnectAttempt::dialSpan_ hold + // when telemetry is off or the category is disabled: operator bool() is + // false and every setter is inert. Drive the full finalize sequence of each + // emitter -- the same calls, in the same order -- and assert the guard + // stays inactive and nothing crashes. This is what proves the new spans + // emit nothing on a node with telemetry disabled, including from a + // destructor. + SpanGuard guard; + ASSERT_FALSE(static_cast(guard)); + + // TransactionAcquire::finalizeAcquireSpan() + guard.setAttribute(ledger_span::attr::txSetHash, "0123456789ABCDEF"); + guard.setAttribute( + ledger_span::attr::outcome, + ledger_span::phaseOutcome(/*failed=*/false, /*complete=*/false, /*timedOut=*/false)); + guard.setAttribute(ledger_span::attr::timeouts, static_cast(21)); + guard.setAttribute(ledger_span::attr::durationMs, static_cast(5250)); + guard.setAttribute(ledger_span::attr::peerCount, static_cast(0)); + + // InboundLedger::beginPhaseSpan() / endPhaseSpan() + guard.setAttribute(ledger_span::attr::ledgerHash, "FEDCBA9876543210"); + guard.setAttribute(ledger_span::attr::ledgerSeq, static_cast(9000)); + guard.setAttribute(ledger_span::attr::timedOut, true); + guard.setAttribute(ledger_span::attr::missingNodes, static_cast(256)); + + // PeerImp::processLedgerRequest()'s scope-exit finalizer + guard.setAttribute(ledger_span::attr::objectType, ledger_span::serveObjectType(/*itype=*/2)); + guard.setAttribute(ledger_span::attr::servedNodes, static_cast(0)); + guard.setAttribute( + ledger_span::attr::outcome, ledger_span::serveOutcome(/*servedNodes=*/0, /*softCap=*/128)); + + // ConnectAttempt::reportOutcome() + guard.setAttribute(peer_span::attr::remoteEndpoint, "10.0.0.5:51235"); + guard.setAttribute(peer_span::attr::outcome, peer_span::val::timeout); + guard.setAttribute(peer_span::attr::durationMs, static_cast(15000)); + + // Still inactive: no span was created, so none can be exported. + EXPECT_FALSE(static_cast(guard)); + + // childSpan on an inactive guard yields another inactive guard, which is + // what makes InboundLedger::beginPhaseSpan() a no-op when telemetry is off: + // it never creates a phase span at all, so the whole per-phase feature + // costs one branch on the disabled path. + auto const child = guard.childSpan(ledger_span::acquireAsTree); + EXPECT_FALSE(static_cast(child)); + // Same via the explicit-parent overload, the one beginPhaseSpan() actually + // calls: an invalid parent context yields an inactive child. + auto const childOfCtx = SpanGuard::childSpan(ledger_span::acquireTxTree, guard.spanContext()); + EXPECT_FALSE(static_cast(childOfCtx)); + EXPECT_FALSE(guard.spanContext().isValid()); +} + +} // namespace + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/tests/libxrpl/telemetry/MetricMacros.cpp b/src/tests/libxrpl/telemetry/MetricMacros.cpp index ddcd2af33c9..7c93ed82c6b 100644 --- a/src/tests/libxrpl/telemetry/MetricMacros.cpp +++ b/src/tests/libxrpl/telemetry/MetricMacros.cpp @@ -17,25 +17,40 @@ * MetricsRegistry.cpp is only compiled into this binary on the no-op path. */ -// cspell:ignore ISTOGRAM -// The all-caps macro name XRPL_METRIC_HISTOGRAM_RECORD trips cspell's -// compound-word splitter, which emits the subword "ISTOGRAM"; ignore it here. - #ifdef XRPL_ENABLE_TELEMETRY #include +#include +#include +#include + +#include +#include +#include + #include #include #include +#include #include #include +#include +#include +#include +#include +#include #include #include +#include +#include #include +#include #include +#include #include +#include #include #include @@ -167,6 +182,286 @@ class ScopedBareProvider opentelemetry::nostd::shared_ptr previous_; }; +namespace otel_sdk = opentelemetry::sdk::metrics; + +/** + * One collected time series set for a single metric name: the exact points the + * SDK produced, keyed by their full attribute (label) set. + * + * `PointAttributes` is an ordered map of label name -> owned value, so two + * different label sets are two different keys -- which is precisely what the + * "distinct outcomes must not collapse into one series" assertions check. + */ +using PointsByAttrs = std::map; + +/** + * Everything one Collect() pulled out of the SDK, keyed by metric name. + * A metric name that was never recorded is simply absent from the map, which + * is how the disabled/no-op tests prove nothing was emitted. + */ +using CollectedMetrics = std::map; + +/** + * A MetricReader that collects synchronously, on demand, in-process. + * + * The OTel SDK only hands aggregated points to a MetricReader, and the only + * pull entry point is the base class's `Collect(callback)`. This subclass adds + * nothing but a convenience wrapper that flattens one collection into a + * name -> (labels -> point) map a test can assert exact values against. + * + * Why not the shipped InMemoryMetricExporter? Its symbols live in a separate + * Conan archive (libopentelemetry_exporter_in_memory_metric.a) which is NOT on + * the xrpl_tests link line -- only libopentelemetry_exporter_in_memory.a (the + * SPAN exporter, used by SpanGuardScope.cpp) is. Subclassing MetricReader needs + * only libopentelemetry_metrics.a, which is already linked via the umbrella + * target, so this keeps the tests hermetic with no build-system change. + * + * Inheritance: + * + * +--------------+ + * | MetricReader | (SDK abstract pull interface) + * +------+-------+ + * | + * +------+-----------------+ + * | CollectOnDemandReader | synchronous, in-process collection + * +------------------------+ + * + * Example usage -- assert an exact counter value: + * @code + * CollectingProvider provider; + * FakeApp app; + * wire(app, true, provider.meter()); + * XRPL_METRIC_COUNTER_INC(app, "my_total", "desc"); + * auto data = provider.collect(); + * EXPECT_EQ(counterValue(data, "my_total", {}), 1); + * @endcode + * + * Example usage -- edge case: prove a metric was NOT emitted at all: + * @code + * wire(app, false, provider.meter()); // registry disabled + * XRPL_METRIC_COUNTER_INC(app, "my_total", "desc"); + * EXPECT_EQ(provider.collect().count("my_total"), 0u); + * @endcode + * + * @note Reports kCumulative temporality, so counter totals accumulate across + * repeated collect() calls rather than resetting -- the assertions below + * therefore collect exactly once per test unless stated otherwise. + * @note Test-only and not thread-safe by itself: collect() must not run + * concurrently with the macro calls it measures. Every test here is + * single-threaded, which satisfies that. + */ +class CollectOnDemandReader final : public otel_sdk::MetricReader +{ +public: + /** + * Pull one collection and flatten it to metric name -> labels -> point. + * @return The points produced by this collection. Metrics that were never + * recorded are absent from the returned map. + */ + [[nodiscard]] CollectedMetrics + collect() + { + CollectedMetrics out; + Collect([&out](otel_sdk::ResourceMetrics& resourceMetrics) { + for (auto const& scope : resourceMetrics.scope_metric_data_) + { + for (auto const& metric : scope.metric_data_) + { + for (auto const& point : metric.point_data_attr_) + { + out[metric.instrument_descriptor.name_][point.attributes] = + point.point_data; + } + } + } + return true; + }); + return out; + } + + /** + * Cumulative so counter totals are absolute, not per-interval deltas. + */ + [[nodiscard]] otel_sdk::AggregationTemporality + GetAggregationTemporality(otel_sdk::InstrumentType) const noexcept override + { + return otel_sdk::AggregationTemporality::kCumulative; + } + +private: + /** + * Nothing is buffered outside the SDK, so a flush always succeeds. + */ + bool + OnForceFlush(std::chrono::microseconds) noexcept override + { + return true; + } + + /** + * No exporter thread or socket to tear down. + */ + bool + OnShutDown(std::chrono::microseconds) noexcept override + { + return true; + } +}; + +/** + * Installs a bare SDK MeterProvider that has a CollectOnDemandReader attached, + * and restores the previous global provider on scope exit. + * + * This is ScopedBareProvider plus a reader: the plain ScopedBareProvider has no + * reader, so the SDK creates no storage and every recorded value is discarded. + * Attaching a reader is what makes the recorded values observable, which is the + * whole point of the value-asserting tests. + * + * @note The reader must be attached BEFORE any instrument is created, because + * Meter::RegisterSyncMetricStorage() binds storage to the collectors that exist + * at instrument-creation time. The constructor guarantees that ordering. + */ +class CollectingProvider +{ +public: + CollectingProvider() + { + reader_ = std::make_shared(); + // Attach the reader first, then publish the provider globally, so any + // instrument created afterwards gets storage wired to this reader. + sdkProvider_ = opentelemetry::sdk::metrics::MeterProviderFactory::Create(); + sdkProvider_->AddMetricReader(reader_); + previous_ = opentelemetry::metrics::Provider::GetMeterProvider(); + opentelemetry::metrics::Provider::SetMeterProvider( + opentelemetry::nostd::shared_ptr(sdkProvider_)); + } + + ~CollectingProvider() + { + opentelemetry::metrics::Provider::SetMeterProvider(previous_); + } + + CollectingProvider(CollectingProvider const&) = delete; + CollectingProvider& + operator=(CollectingProvider const&) = delete; + + /** + * @return A real meter whose instruments feed this provider's reader. + */ + [[nodiscard]] opentelemetry::nostd::shared_ptr + meter() const + { + return sdkProvider_->GetMeter("xrpld_test", "1.0.0"); + } + + /** + * @return The points from one synchronous collection. + */ + [[nodiscard]] CollectedMetrics + collect() const + { + return reader_->collect(); + } + +private: + /** + * The on-demand reader; owned jointly with the provider's collector. + */ + std::shared_ptr reader_; + + /** + * SDK provider that owns the metric storage the reader collects from. + */ + std::shared_ptr sdkProvider_; + + /** + * Previous global provider, restored on scope exit. + */ + opentelemetry::nostd::shared_ptr previous_; +}; + +/** + * Builds the attribute key for a single-label series, e.g. {"outcome","empty"}. + * @return A PointAttributes usable as a lookup key into PointsByAttrs. + */ +[[nodiscard]] otel_sdk::PointAttributes +attrs(std::string const& key, std::string const& value) +{ + otel_sdk::PointAttributes out; + out.SetAttribute(key, value); + return out; +} + +/** + * Builds the attribute key for a two-label series, e.g. site + outcome. + * @return A PointAttributes carrying exactly the two given labels. + */ +[[nodiscard]] otel_sdk::PointAttributes +attrs( + std::string const& key1, + std::string const& value1, + std::string const& key2, + std::string const& value2) +{ + otel_sdk::PointAttributes out; + out.SetAttribute(key1, value1); + out.SetAttribute(key2, value2); + return out; +} + +/** + * Reads the exact accumulated value of one counter time series. + * + * A uint64 Counter aggregates into a SumPointData holding an int64_t, so the + * value is unwrapped through both variants. Fails the calling test (via + * std::map::at) if the metric or the exact label set is missing -- absence is + * itself a defect for these assertions. + * + * @return The counter total for that exact label set. + */ +[[nodiscard]] std::int64_t +counterValue( + CollectedMetrics const& data, + std::string const& metric, + otel_sdk::PointAttributes const& labels) +{ + auto const& point = data.at(metric).at(labels); + auto const& sum = opentelemetry::nostd::get(point); + return opentelemetry::nostd::get(sum.value_); +} + +/** + * Reads the exact last observed value of one Int64ObservableGauge series. + * An observable gauge aggregates as last-value, holding an int64_t. + * @return The observed value for that exact label set (may be negative). + */ +[[nodiscard]] std::int64_t +gaugeValue( + CollectedMetrics const& data, + std::string const& metric, + otel_sdk::PointAttributes const& labels) +{ + auto const& point = data.at(metric).at(labels); + auto const& last = opentelemetry::nostd::get(point); + return opentelemetry::nostd::get(last.value_); +} + +/** + * Reads the sample count and sum of an unlabeled double Histogram. + * + * The macros record histograms with no labels, so the metric has exactly one + * series whose key is the empty attribute set. + * + * @return {count, sum} for the single unlabeled series. + */ +[[nodiscard]] std::pair +histogramCountAndSum(CollectedMetrics const& data, std::string const& metric) +{ + auto const& point = data.at(metric).at(otel_sdk::PointAttributes{}); + auto const& hist = opentelemetry::nostd::get(point); + return {hist.count_, opentelemetry::nostd::get(hist.sum_)}; +} + /** * Fetches a real (non-noop) meter from whatever provider is globally * installed -- inside a test this is the ScopedBareProvider's SDK provider. @@ -186,6 +481,23 @@ wire(FakeApp& app, bool enabled) app.registry().configure(enabled, bareMeter()); } +/** + * Builds a FakeApp wired to an explicitly supplied meter -- used with a + * CollectingProvider so the recorded values are collectable. + * + * @param app The duck-typed app the macros will be driven through. + * @param enabled What the macros' isEnabled() gate will observe. + * @param meter The meter the macros will create their instruments on. + */ +void +wire( + FakeApp& app, + bool enabled, + opentelemetry::nostd::shared_ptr meter) +{ + app.registry().configure(enabled, std::move(meter)); +} + } // namespace TEST(MetricMacros, counter_inc_creates_once_and_does_not_crash) @@ -217,7 +529,7 @@ TEST(MetricMacros, counter_inc_labeled_does_not_crash) app, "test_macro_labeled_counter_total", "Test labeled counter for macro unit test", - {{"reason", std::string("unit_test")}}); + {{telemetry::label::reason, std::string("unit_test")}}); // Instrument was created exactly once at this single call site. EXPECT_EQ(app.registry().meterCalls(), 1); @@ -266,7 +578,7 @@ TEST(MetricMacros, updown_add_labeled_does_not_crash) "test_macro_updown_labeled_total", "Test labeled updown for macro unit test", -1, - {{"reason", std::string("unit_test")}}); + {{telemetry::label::reason, std::string("unit_test")}}); EXPECT_EQ(app.registry().meterCalls(), 1); } @@ -335,4 +647,2670 @@ TEST(MetricMacros, disabled_registry_is_noop) EXPECT_EQ(app.registry().meterCalls(), 0); } +// ----------------------------------------------------------------- +// Fresh-node sync diagnostics metrics. +// +// The tests below assert the EXACT recorded values and the EXACT label shape of +// the eight sync-diagnostics metrics, using a real SDK MeterProvider with a +// CollectOnDemandReader attached. They drive the same macros production uses, +// through the existing FakeApp, so what is proved is the real emit path: +// dns_resolve_total{outcome} OverlayImpl::reportDnsResolve +// dns_resolve_latency_ms (same) +// overlay_connect_total{outcome} ConnectAttempt::reportOutcome +// overlay_dial_latency_ms (same) +// handshake_negotiation_fail_total{reason} Handshake throwNegotiationFailure +// unl_fetch_total{site,outcome} ValidatorSite::reportFetchOutcome +// unl_quorum{metric} MetricsRegistry::registerUnlQuorumGauge +// clock_close_offset_seconds{metric} MetricsRegistry::registerClockSkewGauge +// +// The two observable gauges are registered directly on the SDK meter, mirroring +// the production callback shape, because the real MetricsRegistry's enabled path +// cannot be linked into this standalone binary (see the file header). +// ----------------------------------------------------------------- + +// dns_resolve_total must tally each outcome separately and dns_resolve_latency_ms +// must record every sample: one "resolved", two "empty", and two known latency +// samples whose count and sum are checked exactly. +TEST(MetricMacros, dns_resolve_records_exact_counts_and_latency) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // outcome=resolved once; outcome=empty twice. Both go through the single + // production call site, so the label value is the only difference -- exactly + // how OverlayImpl::reportDnsResolve() emits it. + for (bool const resolved : {true, false, false}) + { + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::dnsResolveTotal, + "Peer hostname resolutions, by outcome", + {{telemetry::label::outcome, std::string(resolved ? "resolved" : "empty")}}); + } + + // Two latency samples with known values: 1.5 ms + 2.5 ms = 4.0 ms. + for (double const ms : {1.5, 2.5}) + { + XRPL_METRIC_HISTOGRAM_RECORD( + app, + telemetry::metric::dnsResolveLatencyMs, + "Time taken to resolve a configured peer hostname, in milliseconds", + ms); + } + + auto const data = provider.collect(); + + // Exactly two series, one per outcome value -- the labeled counter must not + // collapse "resolved" and "empty" into a single series. + ASSERT_EQ(data.at("dns_resolve_total").size(), 2u); + EXPECT_EQ(counterValue(data, "dns_resolve_total", attrs("outcome", "resolved")), 1); + EXPECT_EQ(counterValue(data, "dns_resolve_total", attrs("outcome", "empty")), 2); + + // The label key is exactly "outcome" and it is the ONLY label present. + auto const& firstKey = data.at("dns_resolve_total").begin()->first; + ASSERT_EQ(firstKey.size(), 1u); + EXPECT_EQ(firstKey.begin()->first, "outcome"); + + // Histogram: exactly the two samples recorded, summing to exactly 4.0 ms. + auto const [count, sum] = histogramCountAndSum(data, "dns_resolve_latency_ms"); + EXPECT_EQ(count, 2u); + EXPECT_DOUBLE_EQ(sum, 4.0); + + // Unlabeled histogram: exactly one series, with an empty label set. + ASSERT_EQ(data.at("dns_resolve_latency_ms").size(), 1u); + EXPECT_TRUE(data.at("dns_resolve_latency_ms").begin()->first.empty()); +} + +// overlay_connect_total must keep every terminal outcome in its own series (a +// "timeout" must never be folded into "tcp_fail"), and overlay_dial_latency_ms +// must record each dial's duration. +TEST(MetricMacros, overlay_connect_records_exact_counts_per_outcome) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // Three distinct outcomes with distinct multiplicities: connected x1, + // tcp_fail x3, timeout x2. + auto const bump = [&app](char const* outcome) { + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::overlayConnectTotal, + "Outbound peer connection attempts, by terminal outcome", + {{telemetry::label::outcome, std::string(outcome)}}); + }; + bump("connected"); + bump("tcp_fail"); + bump("tcp_fail"); + bump("tcp_fail"); + bump("timeout"); + bump("timeout"); + + // Dial latencies: 10.0 + 20.0 + 30.5 = 60.5 ms across 3 samples. + for (double const ms : {10.0, 20.0, 30.5}) + { + XRPL_METRIC_HISTOGRAM_RECORD( + app, + telemetry::metric::overlayDialLatencyMs, + "Time from starting an outbound peer dial to its terminal outcome, in milliseconds", + ms); + } + + auto const data = provider.collect(); + + // Three distinct outcomes stay three distinct series. + ASSERT_EQ(data.at("overlay_connect_total").size(), 3u); + EXPECT_EQ(counterValue(data, "overlay_connect_total", attrs("outcome", "connected")), 1); + EXPECT_EQ(counterValue(data, "overlay_connect_total", attrs("outcome", "tcp_fail")), 3); + EXPECT_EQ(counterValue(data, "overlay_connect_total", attrs("outcome", "timeout")), 2); + + // NEGATIVE: an outcome that was never emitted has no series at all, so the + // counts above are not an artifact of some catch-all series. + EXPECT_EQ(data.at("overlay_connect_total").count(attrs("outcome", "tls_fail")), 0u); + + auto const [count, sum] = histogramCountAndSum(data, "overlay_dial_latency_ms"); + EXPECT_EQ(count, 3u); + EXPECT_DOUBLE_EQ(sum, 60.5); +} + +// handshake_negotiation_fail_total must keep each rejection reason distinct; +// throwNegotiationFailure() routes every branch through one call site, so the +// `reason` label is the only thing separating them. +TEST(MetricMacros, handshake_negotiation_fail_keeps_reasons_distinct) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // wrong_network twice, self_connection once, clock_skew once. + for (char const* reason : {"wrong_network", "wrong_network", "self_connection", "clock_skew"}) + { + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::handshakeNegotiationFailTotal, + "Peer handshake negotiations rejected, by reason", + {{telemetry::label::reason, std::string(reason)}}); + } + + auto const data = provider.collect(); + + ASSERT_EQ(data.at("handshake_negotiation_fail_total").size(), 3u); + EXPECT_EQ( + counterValue(data, "handshake_negotiation_fail_total", attrs("reason", "wrong_network")), + 2); + EXPECT_EQ( + counterValue(data, "handshake_negotiation_fail_total", attrs("reason", "self_connection")), + 1); + EXPECT_EQ( + counterValue(data, "handshake_negotiation_fail_total", attrs("reason", "clock_skew")), 1); + + // The label key is exactly "reason", and nothing else rides along. + auto const& firstKey = data.at("handshake_negotiation_fail_total").begin()->first; + ASSERT_EQ(firstKey.size(), 1u); + EXPECT_EQ(firstKey.begin()->first, "reason"); + + // NEGATIVE: a reason from the production set that was not emitted here has + // no series, proving reasons are not being merged. + EXPECT_EQ( + data.at("handshake_negotiation_fail_total").count(attrs("reason", "bad_public_key")), 0u); +} + +// unl_fetch_total carries TWO labels, so the series identity is the (site, +// outcome) PAIR. This is the highest-value signal of the eight: if the pair did +// not form the key, one failing list site would be masked by a healthy one. +TEST(MetricMacros, unl_fetch_total_keys_series_on_site_and_outcome_pair) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + auto const bump = [&app](char const* site, char const* outcome) { + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::unlFetchTotal, + "Validator list fetch attempts, by site and outcome", + {{telemetry::label::site, std::string(site)}, + {telemetry::label::outcome, std::string(outcome)}}); + }; + + constexpr char const* kSiteA = "https://a.example.com/vl.json"; + constexpr char const* kSiteB = "https://b.example.com/vl.json"; + + // Same outcome, two DIFFERENT sites -> must be two series. + bump(kSiteA, "accepted"); + bump(kSiteB, "accepted"); + bump(kSiteB, "accepted"); + // Same site, a SECOND outcome -> must be its own series. + bump(kSiteA, "fetch_error"); + bump(kSiteA, "fetch_error"); + bump(kSiteA, "fetch_error"); + + auto const data = provider.collect(); + + // Three distinct (site, outcome) pairs -> exactly three series. + ASSERT_EQ(data.at("unl_fetch_total").size(), 3u); + + // Two different sites with the SAME outcome are distinct, with their own + // exact counts -- site B's two successes do not inflate site A's one. + EXPECT_EQ( + counterValue(data, "unl_fetch_total", attrs("site", kSiteA, "outcome", "accepted")), 1); + EXPECT_EQ( + counterValue(data, "unl_fetch_total", attrs("site", kSiteB, "outcome", "accepted")), 2); + + // The SAME site with two outcomes is also distinct: A's 3 fetch_errors do + // not merge into A's 1 accepted. + EXPECT_EQ( + counterValue(data, "unl_fetch_total", attrs("site", kSiteA, "outcome", "fetch_error")), 3); + + // NEGATIVE: a pair never emitted (site B erroring) has no series, so the + // per-site failure signal is genuinely per-site. + EXPECT_EQ( + data.at("unl_fetch_total").count(attrs("site", kSiteB, "outcome", "fetch_error")), 0u); + + // Every series key carries exactly the two expected label names. + for (auto const& [labels, point] : data.at("unl_fetch_total")) + { + ASSERT_EQ(labels.size(), 2u); + EXPECT_EQ(labels.count("site"), 1u); + EXPECT_EQ(labels.count("outcome"), 1u); + } +} + +// unl_quorum observes two series from ONE callback, mirroring +// MetricsRegistry::registerUnlQuorumGauge(): trusted_keys and quorum under the +// `metric` label. Registered on the SDK meter directly because that production +// method cannot be linked here (see the file header). +TEST(MetricMacros, unl_quorum_gauge_observes_exact_trusted_keys_and_quorum) +{ + CollectingProvider const provider; + + // Values the callback will report, owned by the test exactly as the real + // registry reads them live from ValidatorList on each collection tick. + struct Observed + { + std::int64_t trustedKeys{0}; + std::int64_t quorum{0}; + }; + Observed observed{.trustedKeys = 5, .quorum = 4}; + + // Keep the instrument alive for the whole test: destroying the handle + // deregisters the callback (ObservableInstrument's destructor calls + // CleanupCallback), which is why the real registry holds it in a member. + auto gauge = provider.meter()->CreateInt64ObservableGauge( + telemetry::metric::unlQuorum, "Trusted UNL key count vs required quorum"); + gauge->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto const* self = static_cast(state); + // Same Observe() form the production callback uses. + auto observe = [&](char const* name, std::int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{telemetry::label::metric, name}}); + }; + observe("trusted_keys", self->trustedKeys); + observe("quorum", self->quorum); + }, + &observed); + + auto const data = provider.collect(); + + // Exactly two series, one per `metric` value, with the exact values. + ASSERT_EQ(data.at("unl_quorum").size(), 2u); + EXPECT_EQ(gaugeValue(data, "unl_quorum", attrs("metric", "trusted_keys")), 5); + EXPECT_EQ(gaugeValue(data, "unl_quorum", attrs("metric", "quorum")), 4); + + // The label key is exactly "metric". + auto const& firstKey = data.at("unl_quorum").begin()->first; + ASSERT_EQ(firstKey.size(), 1u); + EXPECT_EQ(firstKey.begin()->first, "metric"); +} + +// clock_close_offset_seconds must carry a NEGATIVE offset through unchanged -- +// that is the real-world case (local clock ahead of the network) and the reason +// the production gauge is an Int64ObservableGauge rather than an unsigned +// counter. Mirrors MetricsRegistry::registerClockSkewGauge(). +TEST(MetricMacros, clock_skew_gauge_observes_exact_negative_offset) +{ + CollectingProvider const provider; + + // -3 s: this node's clock runs 3 seconds ahead of network close time. + std::int64_t offsetSeconds = -3; + + auto gauge = provider.meter()->CreateInt64ObservableGauge( + telemetry::metric::clockCloseOffsetSeconds, + "Network close time offset from the local clock, in seconds"); + gauge->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto const* value = static_cast(state); + auto observe = [&](char const* name, std::int64_t v) { + opentelemetry::nostd::get>>(result) + ->Observe(v, {{telemetry::label::metric, name}}); + }; + observe("offset", *value); + }, + &offsetSeconds); + + auto const data = provider.collect(); + + // Exactly one series, and the negative value survived the round trip: not + // clamped to 0, not reinterpreted as a large unsigned value. + ASSERT_EQ(data.at("clock_close_offset_seconds").size(), 1u); + EXPECT_EQ(gaugeValue(data, "clock_close_offset_seconds", attrs("metric", "offset")), -3); + + auto const& firstKey = data.at("clock_close_offset_seconds").begin()->first; + ASSERT_EQ(firstKey.size(), 1u); + EXPECT_EQ(firstKey.begin()->first, "metric"); +} + +// RUNTIME-DISABLED no-op proof: with the registry disabled, every one of the +// four sync-diagnostics counter/histogram families emits NOTHING -- no series +// exists for any of those metric names, and meter() is never consulted, so not +// even an instrument was created. This is the runtime counterpart to the +// compile-time no-op proof in MetricsRegistry.cpp. +TEST(MetricMacros, sync_diagnostics_metrics_emit_nothing_when_registry_disabled) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/false, provider.meter()); + + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::dnsResolveTotal, + "Peer hostname resolutions, by outcome", + {{telemetry::label::outcome, std::string("resolved")}}); + XRPL_METRIC_HISTOGRAM_RECORD( + app, + telemetry::metric::dnsResolveLatencyMs, + "Time taken to resolve a configured peer hostname, in milliseconds", + 1.5); + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::overlayConnectTotal, + "Outbound peer connection attempts, by terminal outcome", + {{telemetry::label::outcome, std::string("connected")}}); + XRPL_METRIC_HISTOGRAM_RECORD( + app, + telemetry::metric::overlayDialLatencyMs, + "Time from starting an outbound peer dial to its terminal outcome, in milliseconds", + 10.0); + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::handshakeNegotiationFailTotal, + "Peer handshake negotiations rejected, by reason", + {{telemetry::label::reason, std::string("wrong_network")}}); + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::unlFetchTotal, + "Validator list fetch attempts, by site and outcome", + {{telemetry::label::site, std::string("https://a.example.com/vl.json")}, + {telemetry::label::outcome, std::string("accepted")}}); + + auto const data = provider.collect(); + + // No series at all for any of the six metric names -- not a zero-valued + // series, but total absence: the instruments were never even created. + EXPECT_EQ(data.count("dns_resolve_total"), 0u); + EXPECT_EQ(data.count("dns_resolve_latency_ms"), 0u); + EXPECT_EQ(data.count("overlay_connect_total"), 0u); + EXPECT_EQ(data.count("overlay_dial_latency_ms"), 0u); + EXPECT_EQ(data.count("handshake_negotiation_fail_total"), 0u); + EXPECT_EQ(data.count("unl_fetch_total"), 0u); + + // Nothing else leaked in either: the collection is completely empty. + EXPECT_EQ(data.size(), 0u); + + // Cause, not just state: the isEnabled() gate short-circuited before the + // macro ever asked for a meter. + EXPECT_EQ(app.registry().meterCalls(), 0); +} + +// ----------------------------------------------------------------- +// Sync-state diagnostics (WP-A2). +// +// Asserts the EXACT values and label shapes of the five sync-state signals: +// state_changes_total{from,to} NetworkOPsImp::setMode +// sync_state{metric} MetricsRegistry::registerSyncStateGauge +// initial_full_duration_us +// network_ledger_gate +// server_stall_seconds +// ledgers_behind +// server_stall_events_total MetricsRegistry::registerStallEventsCounter +// +// The counter goes through the same macro production uses. The two observable +// instruments are registered directly on the SDK meter, mirroring the +// production callback shape, because the real MetricsRegistry's enabled path +// cannot be linked into this standalone binary (see the file header). +// ----------------------------------------------------------------- + +// state_changes_total is keyed on the (from, to) PAIR, so a transition edge is +// its own series. This is the whole point of the label: an unlabelled total +// cannot tell a clean tracking->connected->full climb from full->connected +// flapping, because both produce the same count. +TEST(MetricMacros, state_changes_total_keys_series_on_from_to_pair) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // Mirrors the production call site: one macro invocation, the label values + // supplied by strOperatingMode() on the previous and new mode. + auto const transition = [&app](char const* from, char const* to) { + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::stateChangesTotal, + "Total operating mode changes", + {{telemetry::label::from, std::string(from)}, {telemetry::label::to, std::string(to)}}); + }; + + // A clean climb: disconnected -> connected -> syncing -> full, once each. + transition("disconnected", "connected"); + transition("connected", "syncing"); + transition("syncing", "full"); + // Then flapping: full -> connected twice more, and connected -> full twice. + transition("full", "connected"); + transition("full", "connected"); + transition("connected", "full"); + transition("connected", "full"); + + auto const data = provider.collect(); + + // Five distinct (from, to) pairs -> exactly five series. Seven transitions + // were made, but full->connected and connected->full each happened twice, + // which is the point: a repeated edge adds to its own series rather than + // minting a new one, so flapping shows up as a rising count on one edge. + ASSERT_EQ(data.at("state_changes_total").size(), 5u); + + // The climb edges, each traversed exactly once. + EXPECT_EQ( + counterValue(data, "state_changes_total", attrs("from", "disconnected", "to", "connected")), + 1); + EXPECT_EQ( + counterValue(data, "state_changes_total", attrs("from", "connected", "to", "syncing")), 1); + EXPECT_EQ(counterValue(data, "state_changes_total", attrs("from", "syncing", "to", "full")), 1); + + // The flap edges carry their own exact counts and do not merge into the + // climb edges above: full->connected is 2, not folded into connected->full. + EXPECT_EQ( + counterValue(data, "state_changes_total", attrs("from", "full", "to", "connected")), 2); + EXPECT_EQ( + counterValue(data, "state_changes_total", attrs("from", "connected", "to", "full")), 2); + + // Direction matters: connected->syncing exists, syncing->connected does not, + // proving the pair is ordered rather than an unordered edge set. + EXPECT_EQ( + data.at("state_changes_total").count(attrs("from", "syncing", "to", "connected")), 0u); + + // NEGATIVE: a mode pair never emitted has no series at all. + EXPECT_EQ(data.at("state_changes_total").count(attrs("from", "tracking", "to", "full")), 0u); + + // Every series key carries exactly the two expected label names and nothing + // else -- no stray dimension inflating the cardinality. + for (auto const& [labels, point] : data.at("state_changes_total")) + { + ASSERT_EQ(labels.size(), 2u); + EXPECT_EQ(labels.count("from"), 1u); + EXPECT_EQ(labels.count("to"), 1u); + } +} + +// sync_state fans four independent signals out of ONE callback under the +// `metric` label, mirroring MetricsRegistry::registerSyncStateGauge(). The +// values chosen are the diagnostically interesting combination: never reached +// FULL (0 duration) while the gate is still closed, the loop is stalled, and +// the node trails the network. +TEST(MetricMacros, sync_state_gauge_observes_exact_stuck_node_values) +{ + CollectingProvider const provider; + + // The live values the callback reports, owned by the test exactly as the + // real registry reads them from NetworkOPs/LoadManager on each tick. + struct Observed + { + std::int64_t initialFullDurationUs{0}; + std::int64_t networkLedgerGate{0}; + std::int64_t serverStallSeconds{0}; + std::int64_t ledgersBehind{0}; + }; + // A node that never synced: no FULL yet, gate closed, 42 s stalled, 150 + // ledgers behind (network tip 250 vs our validated 100). + Observed observed{ + .initialFullDurationUs = 0, + .networkLedgerGate = 1, + .serverStallSeconds = 42, + .ledgersBehind = 150}; + + // Keep the instrument alive for the whole test: destroying the handle + // deregisters the callback, which is why the real registry holds a member. + auto gauge = provider.meter()->CreateInt64ObservableGauge( + telemetry::metric::syncState, "Sync-pipeline health signals"); + gauge->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto const* self = static_cast(state); + // Same Observe() form the production callback uses. + auto observe = [&](char const* name, std::int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{telemetry::label::metric, name}}); + }; + observe("initial_full_duration_us", self->initialFullDurationUs); + observe("network_ledger_gate", self->networkLedgerGate); + observe("server_stall_seconds", self->serverStallSeconds); + observe("ledgers_behind", self->ledgersBehind); + }, + &observed); + + auto const data = provider.collect(); + + // Exactly four series, one per `metric` value -- the four signals must not + // collapse into a single series. + ASSERT_EQ(data.at("sync_state").size(), 4u); + + // Zero is a REAL observed value here, not a missing series: it is the + // "never reached FULL" signal, so the series must exist and read 0. + ASSERT_EQ(data.at("sync_state").count(attrs("metric", "initial_full_duration_us")), 1u); + EXPECT_EQ(gaugeValue(data, "sync_state", attrs("metric", "initial_full_duration_us")), 0); + + EXPECT_EQ(gaugeValue(data, "sync_state", attrs("metric", "network_ledger_gate")), 1); + EXPECT_EQ(gaugeValue(data, "sync_state", attrs("metric", "server_stall_seconds")), 42); + EXPECT_EQ(gaugeValue(data, "sync_state", attrs("metric", "ledgers_behind")), 150); + + // The label key is exactly "metric" and it is the only label present. + auto const& firstKey = data.at("sync_state").begin()->first; + ASSERT_EQ(firstKey.size(), 1u); + EXPECT_EQ(firstKey.begin()->first, "metric"); + + // NEGATIVE: the stall EPISODE count is deliberately NOT a sync_state + // series -- it is a separate cumulative instrument, so querying it here + // must find nothing. + EXPECT_EQ(data.at("sync_state").count(attrs("metric", "server_stall_events")), 0u); + + // A healthy node reports the complementary values through the same + // callback: synced in 12.5 s, gate open, no stall, at the tip. + observed = Observed{ + .initialFullDurationUs = 12'500'000, + .networkLedgerGate = 0, + .serverStallSeconds = 0, + .ledgersBehind = 0}; + auto const healthy = provider.collect(); + EXPECT_EQ( + gaugeValue(healthy, "sync_state", attrs("metric", "initial_full_duration_us")), 12'500'000); + EXPECT_EQ(gaugeValue(healthy, "sync_state", attrs("metric", "network_ledger_gate")), 0); + EXPECT_EQ(gaugeValue(healthy, "sync_state", attrs("metric", "server_stall_seconds")), 0); + EXPECT_EQ(gaugeValue(healthy, "sync_state", attrs("metric", "ledgers_behind")), 0); +} + +// server_stall_events_total is a cumulative ObservableCounter, not a gauge +// series. It must aggregate as a Sum (so rate() is meaningful) and be +// unlabelled, mirroring MetricsRegistry::registerStallEventsCounter(). +TEST(MetricMacros, stall_events_counter_observes_exact_cumulative_count) +{ + CollectingProvider const provider; + + // Three stall episodes reported so far by the load-monitor thread. + std::int64_t stallEpisodes = 3; + + auto counter = provider.meter()->CreateInt64ObservableCounter( + telemetry::metric::serverStallEventsTotal, "Total server main-loop stall episodes"); + counter->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto const* value = static_cast(state); + // Production observes this with NO labels. + opentelemetry::nostd::get>>(result) + ->Observe(*value); + }, + &stallEpisodes); + + auto const data = provider.collect(); + + // Exactly one unlabelled series, read through counterValue() -- which + // unwraps a SumPointData, so this also proves the instrument aggregates as + // a counter and not as a last-value gauge. + ASSERT_EQ(data.at("server_stall_events_total").size(), 1u); + EXPECT_TRUE(data.at("server_stall_events_total").begin()->first.empty()); + EXPECT_EQ(counterValue(data, "server_stall_events_total", otel_sdk::PointAttributes{}), 3); + + // Monotonic: a later collection sees the higher total, not a delta. + stallEpisodes = 5; + EXPECT_EQ( + counterValue(provider.collect(), "server_stall_events_total", otel_sdk::PointAttributes{}), + 5); +} + +// RUNTIME-DISABLED no-op proof for the counter half of WP-A2: with the registry +// disabled, the setMode call site emits NOTHING -- no series for +// state_changes_total, and meter() is never consulted, so not even an +// instrument was created. +TEST(MetricMacros, state_changes_total_emits_nothing_when_registry_disabled) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/false, provider.meter()); + + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::stateChangesTotal, + "Total operating mode changes", + {{telemetry::label::from, std::string("connected")}, + {telemetry::label::to, std::string("full")}}); + + auto const data = provider.collect(); + + // Total absence, not a zero-valued series. + EXPECT_EQ(data.count("state_changes_total"), 0u); + EXPECT_EQ(data.size(), 0u); + + // Cause, not just state: the isEnabled() gate short-circuited before the + // macro asked for a meter. + EXPECT_EQ(app.registry().meterCalls(), 0); +} + +// ----------------------------------------------------------------- +// Acquire + SHAMap sync diagnostics (WP-A3). +// +// Asserts the EXACT values and label shapes of the five acquire signals: +// sync_acquire_source_total{source} InboundLedger::init +// sync_acquire_no_progress_total InboundLedger::onTimer +// sync_addnode_total{outcome} InboundLedger::recordBatchOutcome +// sync_acquire{metric} MetricsRegistry::registerSyncAcquireGauge +// missing_state_nodes_max +// missing_tx_nodes_max +// received_data_depth +// in_flight +// shamap_cache_hit_rate{metric} MetricsRegistry::registerCacheHitRateDetailGauge +// +// The counters go through the same macros production uses. The two observable +// instruments are registered directly on the SDK meter, mirroring the production +// callback shape, because the real MetricsRegistry's enabled path cannot be +// linked into this standalone binary (see the file header). +// ----------------------------------------------------------------- + +// sync_acquire_source_total splits acquires by whether the local node store +// already held the ledger. This is the disk-bound vs peer-bound distinction, so +// the two sources must never collapse into one series. +TEST(MetricMacros, acquire_source_splits_local_and_network) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // Mirrors the production call site in InboundLedger::init(): one macro + // invocation whose label is derived from complete_ after the first tryDB(). + auto const acquire = [&app](bool localComplete) { + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::syncAcquireSourceTotal, + "Ledger acquires by where the data came from", + {{telemetry::label::source, std::string(localComplete ? "local" : "network")}}); + }; + // One satisfied locally, two needing the network. + acquire(true); + acquire(false); + acquire(false); + + auto const data = provider.collect(); + + ASSERT_EQ(data.at("sync_acquire_source_total").size(), 2u); + EXPECT_EQ(counterValue(data, "sync_acquire_source_total", attrs("source", "local")), 1); + EXPECT_EQ(counterValue(data, "sync_acquire_source_total", attrs("source", "network")), 2); + + // The label key is exactly "source" and nothing rides along with it. + auto const& firstKey = data.at("sync_acquire_source_total").begin()->first; + ASSERT_EQ(firstKey.size(), 1u); + EXPECT_EQ(firstKey.begin()->first, "source"); + + // NEGATIVE: a source value that was never emitted has no series, so the + // counts above are not an artifact of a catch-all series. + EXPECT_EQ(data.at("sync_acquire_source_total").count(attrs("source", "fetch_pack")), 0u); +} + +// sync_acquire_no_progress_total counts ONLY timeouts where no node arrived. +// InboundLedger::onTimer reaches the macro exclusively on its !wasProgress +// branch, so a tick that made progress must leave the total unchanged -- that +// is the whole difference between "slow" and "stuck". +TEST(MetricMacros, acquire_no_progress_counts_only_stalled_timeouts) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // Stands in for onTimer(): the guard lives at the call site in production, + // so the harness reproduces the guard rather than the macro alone. + auto const onTimer = [&app](bool wasProgress) { + if (!wasProgress) + { + XRPL_METRIC_COUNTER_INC( + app, + telemetry::metric::syncAcquireNoProgressTotal, + "Ledger-acquire timeouts where no new node arrived"); + } + }; + + onTimer(/*wasProgress=*/false); + onTimer(/*wasProgress=*/false); + + // Exactly two stalled timeouts so far, on one unlabelled series. + auto const stalled = provider.collect(); + ASSERT_EQ(stalled.at("sync_acquire_no_progress_total").size(), 1u); + EXPECT_TRUE(stalled.at("sync_acquire_no_progress_total").begin()->first.empty()); + EXPECT_EQ( + counterValue(stalled, "sync_acquire_no_progress_total", otel_sdk::PointAttributes{}), 2); + + // A tick that DID make progress must not advance the counter: still 2. + onTimer(/*wasProgress=*/true); + EXPECT_EQ( + counterValue( + provider.collect(), "sync_acquire_no_progress_total", otel_sdk::PointAttributes{}), + 2); + + // A further stalled tick does advance it, proving the counter is live and + // the unchanged reading above was the guard working, not a dead instrument. + onTimer(/*wasProgress=*/false); + EXPECT_EQ( + counterValue( + provider.collect(), "sync_acquire_no_progress_total", otel_sdk::PointAttributes{}), + 3); +} + +// sync_addnode_total separates useful progress from wasted work. All three +// outcomes come from ONE aggregated batch tally, added after the per-node loop +// has finished, so each outcome must land on its own series with its exact count. +TEST(MetricMacros, addnode_outcomes_record_exact_batch_tallies) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // Mirrors InboundLedger::recordBatchOutcome(): three _ADD calls per batch, + // each skipped when its tally is zero (a zero Add would create a series that + // says "we saw invalid nodes", which would be false). + auto const emitBatch = [&app](int good, int duplicate, int invalid) { + auto const emit = [&app](char const* outcome, int count) { + if (count <= 0) + return; + XRPL_METRIC_COUNTER_ADD_LABELED( + app, + telemetry::metric::syncAddnodeTotal, + "SHAMap nodes received during ledger acquire, by outcome", + static_cast(count), + {{telemetry::label::outcome, std::string(outcome)}}); + }; + emit("good", good); + emit("duplicate", duplicate); + emit("invalid", invalid); + }; + + // One batch: 5 good, 2 duplicate, 1 invalid. + emitBatch(5, 2, 1); + + auto const oneBatch = provider.collect(); + ASSERT_EQ(oneBatch.at("sync_addnode_total").size(), 3u); + EXPECT_EQ(counterValue(oneBatch, "sync_addnode_total", attrs("outcome", "good")), 5); + EXPECT_EQ(counterValue(oneBatch, "sync_addnode_total", attrs("outcome", "duplicate")), 2); + EXPECT_EQ(counterValue(oneBatch, "sync_addnode_total", attrs("outcome", "invalid")), 1); + + // Every series key carries exactly the one expected label name. + for (auto const& [labels, point] : oneBatch.at("sync_addnode_total")) + { + ASSERT_EQ(labels.size(), 1u); + EXPECT_EQ(labels.count("outcome"), 1u); + } + + // A second batch accumulates per outcome rather than replacing: 3 more good + // and 4 more duplicates, no invalid this time. + emitBatch(3, 4, 0); + auto const twoBatches = provider.collect(); + EXPECT_EQ(counterValue(twoBatches, "sync_addnode_total", attrs("outcome", "good")), 8); + EXPECT_EQ(counterValue(twoBatches, "sync_addnode_total", attrs("outcome", "duplicate")), 6); + // The zero-tally outcome did NOT advance: still exactly 1 from the first + // batch, so an all-good batch cannot inflate the invalid series. + EXPECT_EQ(counterValue(twoBatches, "sync_addnode_total", attrs("outcome", "invalid")), 1); + + // Still exactly three series after two batches: the zero-tally guard means a + // batch never invents a series for an outcome it did not observe. + EXPECT_EQ(twoBatches.at("sync_addnode_total").size(), 3u); + + // NEGATIVE: an outcome value outside the production set has no series, so + // the three counts above are not an artifact of a catch-all series. + EXPECT_EQ(twoBatches.at("sync_addnode_total").count(attrs("outcome", "stale")), 0u); +} + +// sync_acquire fans four values out of ONE aggregated snapshot, mirroring +// MetricsRegistry::registerSyncAcquireGauge(). The values chosen are the +// headline stuck-sync reading: two acquires in flight, the state tree still +// missing nodes, a backed-up stash. +TEST(MetricMacros, sync_acquire_gauge_observes_exact_stuck_acquire_values) +{ + CollectingProvider const provider; + + // The snapshot the callback reports, owned by the test exactly as the real + // registry reads it from InboundLedgers on each collection tick. + struct Observed + { + std::int64_t maxMissingStateNodes{0}; + std::int64_t maxMissingTxNodes{0}; + std::int64_t receivedDataDepth{0}; + std::int64_t inFlight{0}; + }; + // A stuck acquire: 256 state nodes still outstanding (the sweep cap), the tx + // tree already done, 4 packets stashed, 2 acquires running. + Observed observed{ + .maxMissingStateNodes = 256, .maxMissingTxNodes = 0, .receivedDataDepth = 4, .inFlight = 2}; + + // Keep the instrument alive for the whole test: destroying the handle + // deregisters the callback, which is why the real registry holds a member. + auto gauge = provider.meter()->CreateInt64ObservableGauge( + telemetry::metric::syncAcquire, + "Aggregate ledger-acquire progress across in-flight acquires"); + gauge->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto const* self = static_cast(state); + // Same Observe() form the production callback uses. + auto observe = [&](char const* name, std::int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{telemetry::label::metric, name}}); + }; + observe("missing_state_nodes_max", self->maxMissingStateNodes); + observe("missing_tx_nodes_max", self->maxMissingTxNodes); + observe("received_data_depth", self->receivedDataDepth); + observe("in_flight", self->inFlight); + }, + &observed); + + auto const stuck = provider.collect(); + + // Exactly four series, one per `metric` value. + ASSERT_EQ(stuck.at("sync_acquire").size(), 4u); + EXPECT_EQ(gaugeValue(stuck, "sync_acquire", attrs("metric", "missing_state_nodes_max")), 256); + EXPECT_EQ(gaugeValue(stuck, "sync_acquire", attrs("metric", "received_data_depth")), 4); + EXPECT_EQ(gaugeValue(stuck, "sync_acquire", attrs("metric", "in_flight")), 2); + + // Zero is a REAL reading here, not a missing series: it says the tx tree + // needs nothing while the state tree is still stuck, which is exactly the + // per-map split this signal exists to provide. + ASSERT_EQ(stuck.at("sync_acquire").count(attrs("metric", "missing_tx_nodes_max")), 1u); + EXPECT_EQ(gaugeValue(stuck, "sync_acquire", attrs("metric", "missing_tx_nodes_max")), 0); + + // The label key is exactly "metric" and it is the only label present. This + // is the cardinality guard: a ledger_seq label here would mint a new series + // per ledger acquired. + auto const& firstKey = stuck.at("sync_acquire").begin()->first; + ASSERT_EQ(firstKey.size(), 1u); + EXPECT_EQ(firstKey.begin()->first, "metric"); + EXPECT_EQ(stuck.at("sync_acquire").count(attrs("metric", "ledger_seq")), 0u); + + // A shrinking count is the "slow but alive" reading, and an idle node + // reports all zeros with in_flight=0 -- distinguishable from a stuck node + // only because in_flight is exported alongside. + observed = Observed{ + .maxMissingStateNodes = 128, .maxMissingTxNodes = 0, .receivedDataDepth = 1, .inFlight = 2}; + EXPECT_EQ( + gaugeValue(provider.collect(), "sync_acquire", attrs("metric", "missing_state_nodes_max")), + 128); + + observed = Observed{}; + auto const idle = provider.collect(); + EXPECT_EQ(gaugeValue(idle, "sync_acquire", attrs("metric", "missing_state_nodes_max")), 0); + EXPECT_EQ(gaugeValue(idle, "sync_acquire", attrs("metric", "in_flight")), 0); +} + +// shamap_cache_hit_rate reports the tree-node cache rate normalized to 0.0-1.0, +// mirroring MetricsRegistry::registerCacheHitRateDetailGauge(). The scaling is +// the part worth pinning: TaggedCache::getHitRate() returns 0-100, and the +// dashboard panel uses percentunit, so an unnormalized value would render as +// 9000% instead of 90%. +TEST(MetricMacros, shamap_cache_hit_rate_gauge_normalizes_to_unit_fraction) +{ + CollectingProvider const provider; + + // What TaggedCache::getHitRate() would return: 90 means 90%. + float rawHitRatePercent = 90.0F; + + auto gauge = provider.meter()->CreateDoubleObservableGauge( + telemetry::metric::shamapCacheHitRate, + "SHAMap tree-node cache hit rate (0.0-1.0), by cache"); + gauge->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto const* raw = static_cast(state); + // Same normalization the production callback performs. + opentelemetry::nostd::get< + opentelemetry::nostd::shared_ptr>>( + result) + ->Observe( + static_cast(*raw / 100.0F), {{telemetry::label::metric, "treenode"}}); + }, + &rawHitRatePercent); + + auto const warm = provider.collect(); + + // Exactly one series: the full-below cache is deliberately not reported + // (its TaggedCache hit accounting writes members getHitRate() never reads, + // so it would be a hard-wired zero). + ASSERT_EQ(warm.at("shamap_cache_hit_rate").size(), 1u); + EXPECT_EQ(warm.at("shamap_cache_hit_rate").count(attrs("metric", "full_below")), 0u); + + // 90 percent arrives as exactly 0.9, not 90 and not 9000. + auto const& warmPoint = warm.at("shamap_cache_hit_rate").at(attrs("metric", "treenode")); + auto const& warmLast = opentelemetry::nostd::get(warmPoint); + // NEAR, not DOUBLE_EQ: TaggedCache::getHitRate() returns float, so the + // percent-to-fraction division carries float precision (0.9 arrives as + // 0.89999997...). The tolerance is far tighter than any threshold a + // dashboard reads, and asserting exact equality would only be asserting + // the float representation. + EXPECT_NEAR(opentelemetry::nostd::get(warmLast.value_), 0.9, 1e-6); + + // A cold cache reads exactly 0.0 -- the fresh-sync case, where every lookup + // goes to the node store. + rawHitRatePercent = 0.0F; + auto const cold = provider.collect(); + auto const& coldPoint = cold.at("shamap_cache_hit_rate").at(attrs("metric", "treenode")); + auto const& coldLast = opentelemetry::nostd::get(coldPoint); + EXPECT_NEAR(opentelemetry::nostd::get(coldLast.value_), 0.0, 1e-6); + + // A fully warm cache reads exactly 1.0, pinning the upper bound of the + // normalized range. + rawHitRatePercent = 100.0F; + auto const full = provider.collect(); + auto const& fullPoint = full.at("shamap_cache_hit_rate").at(attrs("metric", "treenode")); + auto const& fullLast = opentelemetry::nostd::get(fullPoint); + EXPECT_NEAR(opentelemetry::nostd::get(fullLast.value_), 1.0, 1e-6); +} + +// RUNTIME-DISABLED no-op proof for the counter half of WP-A3: with the registry +// disabled, all three acquire counters emit NOTHING -- no series at all, and +// meter() is never consulted, so not even an instrument was created. +TEST(MetricMacros, acquire_counters_emit_nothing_when_registry_disabled) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/false, provider.meter()); + + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::syncAcquireSourceTotal, + "Ledger acquires by where the data came from", + {{telemetry::label::source, std::string("network")}}); + XRPL_METRIC_COUNTER_INC( + app, + telemetry::metric::syncAcquireNoProgressTotal, + "Ledger-acquire timeouts where no new node arrived"); + XRPL_METRIC_COUNTER_ADD_LABELED( + app, + telemetry::metric::syncAddnodeTotal, + "SHAMap nodes received during ledger acquire, by outcome", + static_cast(5), + {{telemetry::label::outcome, std::string("good")}}); + + auto const data = provider.collect(); + + // Total absence, not zero-valued series: the instruments never existed. + EXPECT_EQ(data.count("sync_acquire_source_total"), 0u); + EXPECT_EQ(data.count("sync_acquire_no_progress_total"), 0u); + EXPECT_EQ(data.count("sync_addnode_total"), 0u); + EXPECT_EQ(data.size(), 0u); + + // Cause, not just state: the isEnabled() gate short-circuited before the + // macros asked for a meter. + EXPECT_EQ(app.registry().meterCalls(), 0); +} + +// ----------------------------------------------------------------- +// JobQueue saturation diagnostics (WP-A4). +// +// Asserts the EXACT values and label shapes of the pool-wide gauge: +// jobq_saturation{metric} MetricsRegistry::registerJobQueueSaturationGauge +// running_tasks / worker_threads / total_waiting +// +// It is an observable instrument registered directly on the SDK meter, +// mirroring the production callback shape, because the real MetricsRegistry's +// enabled path cannot be linked into this standalone binary (see the file +// header). The reading type is the REAL JobQueue::WorkerSaturation, so a +// rename or reorder on either side breaks this test instead of silently +// drifting from production. +// +// The per-job-type waiting/running/deferred counts are a separate mechanism: +// JobQueue::collect() publishes them as the beast::insight gauges +// jobq__waiting / _running / _deferred, which JobQueue_test covers. +// ----------------------------------------------------------------- + +// jobq_saturation exports the worker-thread count alongside the in-flight +// count so a dashboard can form the ratio without hardcoding a denominator +// that is derived at startup. The values chosen are a fully exhausted pool. +TEST(MetricMacros, jobq_saturation_gauge_observes_exact_pool_exhaustion_values) +{ + CollectingProvider const provider; + + // The real reading type the production callback consumes: every one of 6 + // workers busy, with 12 jobs queued behind them. + JobQueue::WorkerSaturation observed{.runningTasks = 6, .workerThreads = 6, .totalWaiting = 12}; + + auto gauge = provider.meter()->CreateInt64ObservableGauge( + telemetry::metric::jobqSaturation, + "Worker-pool saturation: tasks in flight, worker threads, jobs queued"); + gauge->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto const* self = static_cast(state); + auto observe = [&](char const* field, std::int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{telemetry::label::metric, field}}); + }; + observe("running_tasks", self->runningTasks); + observe("worker_threads", self->workerThreads); + observe("total_waiting", self->totalWaiting); + }, + &observed); + + auto const exhausted = provider.collect(); + + // Exactly three series, one per `metric` value. + ASSERT_EQ(exhausted.at("jobq_saturation").size(), 3u); + EXPECT_EQ(gaugeValue(exhausted, "jobq_saturation", attrs("metric", "running_tasks")), 6); + EXPECT_EQ(gaugeValue(exhausted, "jobq_saturation", attrs("metric", "worker_threads")), 6); + EXPECT_EQ(gaugeValue(exhausted, "jobq_saturation", attrs("metric", "total_waiting")), 12); + + // The label key is exactly "metric" and it is the only label present, so + // this gauge stays a single fixed-cardinality group. + auto const& firstKey = exhausted.at("jobq_saturation").begin()->first; + ASSERT_EQ(firstKey.size(), 1u); + EXPECT_EQ(firstKey.begin()->first, "metric"); + + // A busy-but-not-exhausted pool: same 1.0 ratio, but nothing is queued. + // These two readings are what the ratio alone cannot separate, which is + // why total_waiting is exported next to it. + observed = JobQueue::WorkerSaturation{.runningTasks = 6, .workerThreads = 6, .totalWaiting = 0}; + auto const busy = provider.collect(); + EXPECT_EQ(gaugeValue(busy, "jobq_saturation", attrs("metric", "running_tasks")), 6); + EXPECT_EQ(gaugeValue(busy, "jobq_saturation", attrs("metric", "total_waiting")), 0); + + // An idle pool reads zero in flight with the thread count still reported. + // A zero denominator would make the dashboard ratio undefined, so the + // thread count must never drop out with the load. + observed = JobQueue::WorkerSaturation{.runningTasks = 0, .workerThreads = 6, .totalWaiting = 0}; + auto const idle = provider.collect(); + EXPECT_EQ(gaugeValue(idle, "jobq_saturation", attrs("metric", "running_tasks")), 0); + EXPECT_EQ(gaugeValue(idle, "jobq_saturation", attrs("metric", "worker_threads")), 6); + + // Standalone mode runs a single worker: the denominator is genuinely + // node-specific, which is exactly why it is exported and not hardcoded. + observed = JobQueue::WorkerSaturation{.runningTasks = 1, .workerThreads = 1, .totalWaiting = 3}; + auto const standalone = provider.collect(); + EXPECT_EQ(gaugeValue(standalone, "jobq_saturation", attrs("metric", "worker_threads")), 1); + EXPECT_EQ(gaugeValue(standalone, "jobq_saturation", attrs("metric", "total_waiting")), 3); +} + +// ----------------------------------------------------------------- +// Peer-supply, slot-census and amendment-countdown diagnostics (WP-A7). +// +// Asserts the EXACT values and label shapes of the three gauges and the four +// counters: +// peer_ledger_supply{metric} MetricsRegistry::registerPeerLedgerSupplyGauge +// peerfinder_slot_census{metric} MetricsRegistry::registerSlotCensusGauge +// amendment_block{metric} MetricsRegistry::registerAmendmentBlockGauge +// peer_disconnect_total{reason,direction} PeerImp::close +// peer_accept_total{outcome} OverlayImpl::reportAcceptOutcome +// serve_refused_total{request,reason} PeerImp::reportServeRefusal +// ledger_jump_total (unlabelled) NetworkOPsImp::switchLastClosedLedger +// +// The three gauges are observable instruments registered directly on the SDK +// meter, mirroring the production callback shape, because the real +// MetricsRegistry's enabled path cannot be linked into this standalone binary +// (see the file header). The snapshot types are the REAL xrpl::PeerLedgerSupply +// and xrpl::PeerFinder::SlotCensus aggregates, so a field rename or a reorder on +// either side breaks these tests instead of silently drifting from production. +// Both are plain header-only aggregates with no out-of-line members, so using +// them here adds no xrpld link dependency. +// +// The four counters are driven through XRPL_METRIC_COUNTER_INC_LABELED / +// XRPL_METRIC_COUNTER_INC, which is exactly what the production call sites use. +// ----------------------------------------------------------------- + +// peer_ledger_supply must keep the two "who can serve me" counts on separate +// series from the "who is even talking" denominator. The values chosen are the +// headline supply gap: three peers connected and advertising a range, all three +// covering the validated sequence, and NOT ONE covering the next one needed. +TEST(MetricMacros, peer_ledger_supply_gauge_names_a_gap_no_peer_can_fill) +{ + CollectingProvider const provider; + + // The real aggregate the production callback reports, filled here as + // OverlayImpl::getPeerLedgerSupply() would fill it. Peer set holds + // [1000, 4000]; this node's validated sequence is 4000, so the next needed + // is 4001 -- past every peer's tip. + PeerLedgerSupply observed{ + .peersReporting = 3, + .peersServingValidated = 3, + .peersServingNext = 0, + .supplyMinSeq = 1000, + .supplyMaxSeq = 4000}; + + // Keep the instrument alive for the whole test: destroying the handle + // deregisters the callback, which is why the real registry holds a member. + auto gauge = provider.meter()->CreateInt64ObservableGauge( + telemetry::metric::peerLedgerSupply, + "Peer coverage of the ledger sequence this node needs"); + gauge->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto const* self = static_cast(state); + // Same single-label Observe() form the production callback uses. + auto observe = [&](char const* field, std::int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{telemetry::label::metric, field}}); + }; + observe("peers_reporting", self->peersReporting); + observe("peers_serving_validated", self->peersServingValidated); + observe("peers_serving_next", self->peersServingNext); + observe("supply_min_seq", self->supplyMinSeq); + observe("supply_max_seq", self->supplyMaxSeq); + }, + &observed); + + auto const gap = provider.collect(); + + // Exactly five series, one per `metric` value: no field collapses into + // another, so the denominator and the verdict stay separately readable. + ASSERT_EQ(gap.at("peer_ledger_supply").size(), 5u); + + // THE verdict this signal exists for: zero peers can serve the next needed + // ledger while three are connected and reporting. That pair is the whole + // point -- "the network cannot supply what I need" is otherwise + // indistinguishable from "my peers are slow", and the two faults have + // completely different fixes (change the peer set vs. wait). + EXPECT_EQ(gaugeValue(gap, "peer_ledger_supply", attrs("metric", "peers_serving_next")), 0); + EXPECT_EQ(gaugeValue(gap, "peer_ledger_supply", attrs("metric", "peers_reporting")), 3); + + // Serving the validated sequence is NOT the same question, and reads 3 here: + // the peers can serve where this node already is, just not where it must go + // next. Without both counts the gap would look like a total peer failure. + EXPECT_EQ(gaugeValue(gap, "peer_ledger_supply", attrs("metric", "peers_serving_validated")), 3); + + // The window, so an operator can see whether the wanted sequence is below + // the peer set's floor (discarded history) or above its tip (unreached). + EXPECT_EQ(gaugeValue(gap, "peer_ledger_supply", attrs("metric", "supply_min_seq")), 1000); + EXPECT_EQ(gaugeValue(gap, "peer_ledger_supply", attrs("metric", "supply_max_seq")), 4000); + + // Exactly one label key, and it is "metric". This is the cardinality guard: + // a peer_id label here would mint a new series per connection. + for (auto const& [labels, point] : gap.at("peer_ledger_supply")) + { + ASSERT_EQ(labels.size(), 1u); + EXPECT_EQ(labels.begin()->first, "metric"); + } + + // NEGATIVE: a `metric` value outside the production set of five has no + // series, so the readings above are not an artifact of a catch-all series. + EXPECT_EQ(gap.at("peer_ledger_supply").count(attrs("metric", "peers_serving")), 0u); +} + +// The complementary reading to the gap above: a healthy peer set where every +// reporting peer covers the next needed sequence, so waiting WILL finish the +// sync. Split from the gap test to keep each under the length limit. +TEST(MetricMacros, peer_ledger_supply_gauge_reads_zero_window_as_unknown) +{ + CollectingProvider const provider; + + // Healthy: four peers, all covering both the validated sequence and the + // next one, window [1000, 5000]. + PeerLedgerSupply observed{ + .peersReporting = 4, + .peersServingValidated = 4, + .peersServingNext = 4, + .supplyMinSeq = 1000, + .supplyMaxSeq = 5000}; + + auto gauge = provider.meter()->CreateInt64ObservableGauge( + telemetry::metric::peerLedgerSupply, + "Peer coverage of the ledger sequence this node needs"); + gauge->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto const* self = static_cast(state); + auto observe = [&](char const* field, std::int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{telemetry::label::metric, field}}); + }; + observe("peers_reporting", self->peersReporting); + observe("peers_serving_validated", self->peersServingValidated); + observe("peers_serving_next", self->peersServingNext); + observe("supply_min_seq", self->supplyMinSeq); + observe("supply_max_seq", self->supplyMaxSeq); + }, + &observed); + + auto const healthy = provider.collect(); + EXPECT_EQ(gaugeValue(healthy, "peer_ledger_supply", attrs("metric", "peers_serving_next")), 4); + EXPECT_EQ(gaugeValue(healthy, "peer_ledger_supply", attrs("metric", "peers_reporting")), 4); + + // NEGATIVE/edge: nothing has advertised a range yet. Peers that have not + // sent mtSTATUS_CHANGE report [0, 0] and are excluded from every field, so + // all five read 0. + observed = PeerLedgerSupply{}; + auto const silent = provider.collect(); + + // A 0 window means "unknown", NOT "the peer set serves from genesis". The + // only thing that separates the two is peers_reporting, which is why it must + // be read alongside: 0 out of 0 reporting is silence, 0 out of many would be + // a real supply gap. Asserting the pair together pins that contract. + EXPECT_EQ(gaugeValue(silent, "peer_ledger_supply", attrs("metric", "supply_min_seq")), 0); + EXPECT_EQ(gaugeValue(silent, "peer_ledger_supply", attrs("metric", "peers_reporting")), 0); + EXPECT_EQ(gaugeValue(silent, "peer_ledger_supply", attrs("metric", "supply_max_seq")), 0); + + // Every field is still a present series at 0, never absent: a dropped + // series would be indistinguishable from a dead exporter. + ASSERT_EQ(silent.at("peer_ledger_supply").size(), 5u); + EXPECT_EQ(gaugeValue(silent, "peer_ledger_supply", attrs("metric", "peers_serving_next")), 0); + EXPECT_EQ( + gaugeValue(silent, "peer_ledger_supply", attrs("metric", "peers_serving_validated")), 0); + + // A single reporting peer at the network tip: min and max collapse to the + // same sequence, which is a legitimate reading, not a defect. + observed = PeerLedgerSupply{ + .peersReporting = 1, + .peersServingValidated = 1, + .peersServingNext = 0, + .supplyMinSeq = 5000, + .supplyMaxSeq = 5000}; + auto const single = provider.collect(); + EXPECT_EQ(gaugeValue(single, "peer_ledger_supply", attrs("metric", "supply_min_seq")), 5000); + EXPECT_EQ(gaugeValue(single, "peer_ledger_supply", attrs("metric", "supply_max_seq")), 5000); +} + +// peerfinder_slot_census must export all nine numbers, because each of the three +// common bootstrap failures is named by a DIFFERENT pair of them and today only +// the two active counts exist. The values chosen are the "dialling but never +// completing" case, which the two legacy gauges cannot express at all. +TEST(MetricMacros, slot_census_gauge_names_each_bootstrap_fault_exactly) +{ + CollectingProvider const provider; + + // The real snapshot type the production callback consumes. Outbound is 2 of + // 10 with 6 dials in flight; one configured fixed peer is missing. + PeerFinder::SlotCensus observed{ + .outActive = 2, + .outMax = 10, + .inActive = 0, + .inMax = 0, + .connecting = 6, + .fixedConfigured = 2, + .fixedActive = 1, + .bootcache = 40, + .livecache = 12}; + + auto gauge = provider.meter()->CreateInt64ObservableGauge( + telemetry::metric::peerfinderSlotCensus, + "PeerFinder slots, connection attempts and address caches"); + gauge->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto const* self = static_cast(state); + // Same single-label Observe() form the production callback uses. + auto observe = [&](char const* field, std::int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{telemetry::label::metric, field}}); + }; + observe("out_active", self->outActive); + observe("out_max", self->outMax); + observe("in_active", self->inActive); + observe("in_max", self->inMax); + observe("connecting", self->connecting); + observe("fixed_configured", self->fixedConfigured); + observe("fixed_active", self->fixedActive); + observe("bootcache", self->bootcache); + observe("livecache", self->livecache); + }, + &observed); + + auto const dialling = provider.collect(); + + // Exactly nine series, one per `metric` value: all nine fields reach the + // exporter, not just the two the legacy insight gauges carried. + ASSERT_EQ(dialling.at("peerfinder_slot_census").size(), 9u); + // FAULT (a) "dialling but never completing": out_active below out_max WITH + // connecting non-zero. Without the attempt count this is indistinguishable + // from a node that is not dialling at all -- the capacity term alone says + // only "under-connected", never why. + EXPECT_EQ(gaugeValue(dialling, "peerfinder_slot_census", attrs("metric", "out_active")), 2); + EXPECT_EQ(gaugeValue(dialling, "peerfinder_slot_census", attrs("metric", "out_max")), 10); + EXPECT_EQ(gaugeValue(dialling, "peerfinder_slot_census", attrs("metric", "connecting")), 6); + + // FAULT (c) "configured fixed peer unreachable": fixed_active strictly below + // fixed_configured. 1 of 2 asked-for peers is connected. + EXPECT_EQ( + gaugeValue(dialling, "peerfinder_slot_census", attrs("metric", "fixed_configured")), 2); + EXPECT_EQ(gaugeValue(dialling, "peerfinder_slot_census", attrs("metric", "fixed_active")), 1); + + // Inbound disabled reads in_max=0, which makes in_active=0 a configuration + // fact rather than a fault -- the pair is what separates them. + EXPECT_EQ(gaugeValue(dialling, "peerfinder_slot_census", attrs("metric", "in_max")), 0); + EXPECT_EQ(gaugeValue(dialling, "peerfinder_slot_census", attrs("metric", "in_active")), 0); + + // Addresses ARE available here, so fault (b) is ruled out on this reading: + // the node has somewhere to dial and is still not completing. + EXPECT_EQ(gaugeValue(dialling, "peerfinder_slot_census", attrs("metric", "bootcache")), 40); + EXPECT_EQ(gaugeValue(dialling, "peerfinder_slot_census", attrs("metric", "livecache")), 12); + + // Exactly one label key on every series, and it is "metric": the nine + // fields form one fixed-cardinality group, not nine label dimensions. + for (auto const& [labels, point] : dialling.at("peerfinder_slot_census")) + { + ASSERT_EQ(labels.size(), 1u); + EXPECT_EQ(labels.begin()->first, "metric"); + } + // NEGATIVE: a `metric` value outside the production set of nine has no + // series, so the readings above are not an artifact of a catch-all series. + EXPECT_EQ(dialling.at("peerfinder_slot_census").count(attrs("metric", "out_count")), 0u); +} + +// FAULT (b) "nothing to dial", plus the idle-but-healthy reading. Separated from +// the fault-(a)/(c) test above to keep each function under the length limit. +TEST(MetricMacros, slot_census_gauge_reports_every_field_even_when_idle) +{ + CollectingProvider const provider; + + // A fresh node with no seed addresses at all: nothing dialled because there + // is nothing to dial. Distinct from fault (a), where dials are attempted. + PeerFinder::SlotCensus observed{ + .outActive = 0, + .outMax = 10, + .inActive = 0, + .inMax = 20, + .connecting = 0, + .fixedConfigured = 0, + .fixedActive = 0, + .bootcache = 0, + .livecache = 0}; + + auto gauge = provider.meter()->CreateInt64ObservableGauge( + telemetry::metric::peerfinderSlotCensus, + "PeerFinder slots, connection attempts and address caches"); + gauge->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto const* self = static_cast(state); + auto observe = [&](char const* field, std::int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{telemetry::label::metric, field}}); + }; + observe("out_active", self->outActive); + observe("out_max", self->outMax); + observe("in_active", self->inActive); + observe("in_max", self->inMax); + observe("connecting", self->connecting); + observe("fixed_configured", self->fixedConfigured); + observe("fixed_active", self->fixedActive); + observe("bootcache", self->bootcache); + observe("livecache", self->livecache); + }, + &observed); + + auto const nothingToDial = provider.collect(); + + // FAULT (b) "nothing to dial": both caches at exactly 0 while outbound + // capacity is available. The fix is [ips] / DNS, not the network. + EXPECT_EQ(gaugeValue(nothingToDial, "peerfinder_slot_census", attrs("metric", "bootcache")), 0); + EXPECT_EQ(gaugeValue(nothingToDial, "peerfinder_slot_census", attrs("metric", "livecache")), 0); + // connecting=0 here is what separates fault (b) from fault (a): no dial was + // even attempted, because there was no address to attempt. + EXPECT_EQ( + gaugeValue(nothingToDial, "peerfinder_slot_census", attrs("metric", "connecting")), 0); + EXPECT_EQ(gaugeValue(nothingToDial, "peerfinder_slot_census", attrs("metric", "out_max")), 10); + + // NEGATIVE: an idle-but-healthy node still reports EVERY field. A zero-valued + // field must be a present series, never an absent one -- absence would be + // indistinguishable from a dead exporter or a crashed callback. + observed = PeerFinder::SlotCensus{ + .outActive = 10, + .outMax = 10, + .inActive = 5, + .inMax = 20, + .connecting = 0, + .fixedConfigured = 0, + .fixedActive = 0, + .bootcache = 55, + .livecache = 30}; + auto const idle = provider.collect(); + + ASSERT_EQ(idle.at("peerfinder_slot_census").size(), 9u); + // The four fields that are legitimately 0 on a healthy node are each PRESENT + // with value 0, asserted one by one so a dropped series fails the test. + for (char const* field : {"connecting", "fixed_configured", "fixed_active"}) + { + ASSERT_EQ(idle.at("peerfinder_slot_census").count(attrs("metric", field)), 1u); + EXPECT_EQ(gaugeValue(idle, "peerfinder_slot_census", attrs("metric", field)), 0); + } + // Slots full: out_active has reached out_max, so nothing is dialling because + // nothing needs to be. Same connecting=0 as fault (b), opposite meaning -- + // only the capacity pair tells them apart. + EXPECT_EQ(gaugeValue(idle, "peerfinder_slot_census", attrs("metric", "out_active")), 10); + EXPECT_EQ(gaugeValue(idle, "peerfinder_slot_census", attrs("metric", "out_max")), 10); +} + +// amendment_block's whole value is the countdown, so the arithmetic is what this +// pins: the production callback computes max(expected - now, 0) in std::int64_t +// and observes -1 when firstUnsupportedExpected() is nullopt. All four states are +// asserted to EXACT values, including the two that a naive implementation gets +// wrong (the healthy sentinel, and the unsigned-subtraction wrap). +TEST(MetricMacros, amendment_block_gauge_observes_exact_countdown_and_sentinel) +{ + CollectingProvider const provider; + + // Two explicit epoch-second constants, so the expected difference is exact + // rather than approximate -- a clock read would make 7200 unassertable. + constexpr std::int64_t kNowEpochSeconds = 800'000'000; + constexpr std::int64_t kTwoHours = 7200; + + // Mirrors what the production callback reads: the warned flag from + // NetworkOPs::isAmendmentWarned(), and the optional activation time from + // AmendmentTable::firstUnsupportedExpected(). + struct Observed + { + bool warned{false}; + std::optional expectedEpochSeconds; + std::int64_t nowEpochSeconds{0}; + }; + // State (a): nothing pending at all -- the healthy case. + Observed observed{ + .warned = false, .expectedEpochSeconds = {}, .nowEpochSeconds = kNowEpochSeconds}; + + auto gauge = provider.meter()->CreateInt64ObservableGauge( + telemetry::metric::amendmentBlock, + "Amendment-block warning and seconds until the node stops validating"); + gauge->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto const* self = static_cast(state); + auto observe = [&](char const* field, std::int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{telemetry::label::metric, field}}); + }; + observe("warned", self->warned ? 1 : 0); + + // The production arithmetic, reproduced exactly: -1 sentinel when + // nothing is pending, otherwise the difference taken in int64_t and + // clamped at 0. + std::int64_t secondsToBlock = -1; + if (self->expectedEpochSeconds) + { + secondsToBlock = + std::max(*self->expectedEpochSeconds - self->nowEpochSeconds, 0); + } + observe("seconds_to_block", secondsToBlock); + }, + &observed); + + auto const healthy = provider.collect(); + + // Exactly two series, one per `metric` value. + ASSERT_EQ(healthy.at("amendment_block").size(), 2u); + + // STATE (a) nothing pending: warned is exactly 0, and the countdown is + // exactly -1. Not 0 (which would read as "blocking right now", the most + // alarming possible value) and not absent (which a dashboard cannot tell + // from a stopped exporter). A distinct in-band sentinel is the only encoding + // that makes "healthy" assertable, matching unl_expiry_days' -1. + EXPECT_EQ(gaugeValue(healthy, "amendment_block", attrs("metric", "warned")), 0); + EXPECT_EQ(gaugeValue(healthy, "amendment_block", attrs("metric", "seconds_to_block")), -1); + ASSERT_EQ(healthy.at("amendment_block").count(attrs("metric", "seconds_to_block")), 1u); + + // STATE (b) pending two hours out: warned flips to exactly 1 and the + // countdown is exactly 7200, computed from the two constants above. + observed = Observed{ + .warned = true, + .expectedEpochSeconds = kNowEpochSeconds + kTwoHours, + .nowEpochSeconds = kNowEpochSeconds}; + auto const pending = provider.collect(); + EXPECT_EQ(gaugeValue(pending, "amendment_block", attrs("metric", "warned")), 1); + EXPECT_EQ(gaugeValue(pending, "amendment_block", attrs("metric", "seconds_to_block")), 7200); + + // Both series are present in both states, so the countdown never drops out + // when the warning flips. + ASSERT_EQ(pending.at("amendment_block").size(), 2u); +} + +// The clamp at the bottom of the countdown, and the deliberate absence of an +// amendment-id label. Split out of the test above to keep each function inside +// the length limit; it re-establishes the same callback shape. +TEST(MetricMacros, amendment_block_gauge_clamps_past_due_and_carries_no_amendment_id) +{ + CollectingProvider const provider; + + constexpr std::int64_t kNowEpochSeconds = 800'000'000; + constexpr std::int64_t kOneHour = 3600; + + struct Observed + { + bool warned{false}; + std::optional expectedEpochSeconds; + std::int64_t nowEpochSeconds{0}; + }; + // State (c): the activation time passed an hour ago. + Observed observed{ + .warned = true, + .expectedEpochSeconds = kNowEpochSeconds - kOneHour, + .nowEpochSeconds = kNowEpochSeconds}; + + auto gauge = provider.meter()->CreateInt64ObservableGauge( + telemetry::metric::amendmentBlock, + "Amendment-block warning and seconds until the node stops validating"); + gauge->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto const* self = static_cast(state); + auto observe = [&](char const* field, std::int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{telemetry::label::metric, field}}); + }; + observe("warned", self->warned ? 1 : 0); + std::int64_t secondsToBlock = -1; + if (self->expectedEpochSeconds) + { + secondsToBlock = + std::max(*self->expectedEpochSeconds - self->nowEpochSeconds, 0); + } + observe("seconds_to_block", secondsToBlock); + }, + &observed); + + // STATE (c) past due by an hour: clamped to exactly 0, never negative. This + // is the assertion that proves the difference is taken in std::int64_t and + // NOT in NetClock's unsigned representation -- subtracting the time_points + // directly would wrap to roughly 4.29 billion and plot as 136 years away, + // turning the most urgent reading into the least. + auto const pastDue = provider.collect(); + EXPECT_EQ(gaugeValue(pastDue, "amendment_block", attrs("metric", "seconds_to_block")), 0); + EXPECT_EQ(gaugeValue(pastDue, "amendment_block", attrs("metric", "warned")), 1); + + // STATE (d) exactly at the boundary (expected == now): exactly 0. Pins the + // clamp's inclusive edge, so the transition into state (c) cannot skip a + // value or briefly emit -1. + observed = Observed{ + .warned = true, + .expectedEpochSeconds = kNowEpochSeconds, + .nowEpochSeconds = kNowEpochSeconds}; + auto const boundary = provider.collect(); + EXPECT_EQ(gaugeValue(boundary, "amendment_block", attrs("metric", "seconds_to_block")), 0); + + // NEGATIVE: there is NO label carrying the blocking amendment's id or hash. + // Exactly two series, and "metric" is the only label key. The identity is + // deliberately excluded: the network can vote on an arbitrary 256-bit + // amendment id, not just this build's known features, so an id label would + // be unbounded cardinality and would mint a permanent new series per + // amendment. The id is already in the log line, correlated by node and time. + ASSERT_EQ(boundary.at("amendment_block").size(), 2u); + for (auto const& [labels, point] : boundary.at("amendment_block")) + { + ASSERT_EQ(labels.size(), 1u); + EXPECT_EQ(labels.begin()->first, "metric"); + } + // An id-shaped label value has no series, so the two above are the whole set. + EXPECT_EQ(boundary.at("amendment_block").count(attrs("metric", "amendment_id")), 0u); +} + +// peer_disconnect_total is the signal that splits today's single unlabelled +// disconnect tally by cause AND direction. The series identity is the (reason, +// direction) PAIR: if it were not, a wave of our-fault backpressure on outbound +// links would be masked by ordinary inbound peer churn. +TEST(MetricMacros, peer_disconnect_total_keys_series_on_reason_and_direction_pair) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // The exact call PeerImp::close() makes, once per teardown. + auto const bump = [&app](char const* reason, char const* direction) { + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::peerDisconnectTotal, + "Peer disconnects, by cause and connection direction", + {{telemetry::label::reason, std::string(reason)}, + {telemetry::label::direction, std::string(direction)}}); + }; + + // Two OUR-FAULT reasons: this node could not keep up with what it owed the + // peer, or charged it off under its own resource pressure. + bump("large_sendq", "outbound"); + bump("charge_resources", "inbound"); + // Three NETWORK/TOPOLOGY reasons: the peer is on another chain, stopped + // answering, or the socket failed. + bump("not_useful", "outbound"); + bump("ping_timeout", "outbound"); + bump("read_error", "inbound"); + + auto const data = provider.collect(); + + // Five distinct (reason, direction) pairs -> exactly five series. Today all + // five collapse into one number, which is the defect this fixes. + ASSERT_EQ(data.at("peer_disconnect_total").size(), 5u); + + // Exact value 1 per distinct labelset. These two are also the proof that the + // labelsets do NOT collapse: each holds exactly 1 rather than one of them + // holding 2, so an our-fault outbound teardown and a network-fault inbound + // one stay two separate stories with two separate fixes. (counterValue() + // looks the key up with std::map::at, so a merged series fails here.) + EXPECT_EQ( + counterValue( + data, "peer_disconnect_total", attrs("reason", "large_sendq", "direction", "outbound")), + 1); + EXPECT_EQ( + counterValue( + data, "peer_disconnect_total", attrs("reason", "read_error", "direction", "inbound")), + 1); + + // Our-fault reasons are individually addressable, so "the node is shedding + // its own peers" is readable without reading logs. + EXPECT_EQ( + counterValue( + data, + "peer_disconnect_total", + attrs("reason", "charge_resources", "direction", "inbound")), + 1); + // Network/topology reasons stay distinct from each other too: a chain split + // ("not_useful") is not a dead link ("ping_timeout"). + EXPECT_EQ( + counterValue( + data, "peer_disconnect_total", attrs("reason", "not_useful", "direction", "outbound")), + 1); + EXPECT_EQ( + counterValue( + data, + "peer_disconnect_total", + attrs("reason", "ping_timeout", "direction", "outbound")), + 1); + + // Exactly two label keys on every series, and exactly these two: a third + // (a peer address, say) would be unbounded cardinality. + for (auto const& [labels, point] : data.at("peer_disconnect_total")) + { + ASSERT_EQ(labels.size(), 2u); + EXPECT_EQ(labels.count("reason"), 1u); + EXPECT_EQ(labels.count("direction"), 1u); + } +} + +// The `direction` label must genuinely participate in the series key, not just +// ride along: the SAME reason seen on both directions has to produce two series +// of 1 rather than one series of 2. Split out of the test above to keep each +// function inside the length limit. +TEST(MetricMacros, peer_disconnect_total_does_not_merge_directions_for_one_reason) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + auto const bump = [&app](char const* reason, char const* direction) { + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::peerDisconnectTotal, + "Peer disconnects, by cause and connection direction", + {{telemetry::label::reason, std::string(reason)}, + {telemetry::label::direction, std::string(direction)}}); + }; + + // One read_error each way. If direction did not key the series, this would + // collapse to a single series holding 2. + bump("read_error", "inbound"); + bump("read_error", "outbound"); + + auto const data = provider.collect(); + + // Two series, not one: the same cause on opposite directions stays split. + ASSERT_EQ(data.at("peer_disconnect_total").size(), 2u); + EXPECT_EQ( + counterValue( + data, "peer_disconnect_total", attrs("reason", "read_error", "direction", "inbound")), + 1); + EXPECT_EQ( + counterValue( + data, "peer_disconnect_total", attrs("reason", "read_error", "direction", "outbound")), + 1); + + // Bumping ONE direction advances only that series. This is the assertion that + // would fail if the labels were merged: inbound must stay at exactly 1. + bump("read_error", "outbound"); + auto const second = provider.collect(); + EXPECT_EQ( + counterValue( + second, + "peer_disconnect_total", + attrs("reason", "read_error", "direction", "outbound")), + 2); + EXPECT_EQ( + counterValue( + second, "peer_disconnect_total", attrs("reason", "read_error", "direction", "inbound")), + 1); + EXPECT_EQ(second.at("peer_disconnect_total").size(), 2u); + + // NEGATIVE: a reason outside the fixed literal set in PeerImp.cpp has no + // series, so the counts above are not an artifact of a catch-all series. + EXPECT_EQ( + second.at("peer_disconnect_total") + .count(attrs("reason", "disconnected", "direction", "inbound")), + 0u); + // NEGATIVE: a direction value outside {inbound, outbound} has no series. + EXPECT_EQ( + second.at("peer_disconnect_total") + .count(attrs("reason", "read_error", "direction", "unknown")), + 0u); +} + +// peer_accept_total is the inbound twin of the existing +// overlay_connect_total{outcome}: that one counts OUTBOUND dials this node +// makes, this one counts INBOUND attempts it receives. Together they give the +// full in/out split, which neither provides alone -- a node whose outbound dials +// all succeed while every inbound attempt is refused looks perfectly healthy on +// overlay_connect_total by itself. +TEST(MetricMacros, peer_accept_total_keys_series_on_outcome) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // The exact call OverlayImpl::reportAcceptOutcome() makes, once per + // terminal outcome of one inbound attempt. + auto const bump = [&app](char const* outcome) { + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::peerAcceptTotal, + "Inbound peer connection attempts, by terminal outcome", + {{telemetry::label::outcome, std::string(outcome)}}); + }; + + // accepted x2, no_slot x3, handshake_error x1 -- three distinct + // multiplicities so no two series can be confused with each other. + bump("accepted"); + bump("accepted"); + bump("no_slot"); + bump("no_slot"); + bump("no_slot"); + bump("handshake_error"); + + auto const data = provider.collect(); + + // Three distinct outcomes stay three distinct series with exact values. + ASSERT_EQ(data.at("peer_accept_total").size(), 3u); + EXPECT_EQ(counterValue(data, "peer_accept_total", attrs("outcome", "accepted")), 2); + // "no_slot" is capacity, "handshake_error" is a protocol or crypto failure. + // Collapsed into one number they would be indistinguishable, yet the first + // is fixed by configuration and the second by investigation. + EXPECT_EQ(counterValue(data, "peer_accept_total", attrs("outcome", "no_slot")), 3); + EXPECT_EQ(counterValue(data, "peer_accept_total", attrs("outcome", "handshake_error")), 1); + + // Exactly one label key, and it is "outcome". + auto const& firstKey = data.at("peer_accept_total").begin()->first; + ASSERT_EQ(firstKey.size(), 1u); + EXPECT_EQ(firstKey.begin()->first, "outcome"); + + // NEGATIVE: an outcome from the production set that was not emitted here has + // no series, proving outcomes are not being merged into a catch-all. + EXPECT_EQ(data.at("peer_accept_total").count(attrs("outcome", "resource_limit")), 0u); + + // Accumulates rather than replaces: the reader is cumulative, so a second + // acceptance advances the existing series to exactly 3. + bump("accepted"); + EXPECT_EQ( + counterValue(provider.collect(), "peer_accept_total", attrs("outcome", "accepted")), 3); +} + +// serve_refused_total measures the SUPPLY side: what this node refuses to serve +// its peers. Nothing measured it before, so a node shedding every ledger request +// looked identical to one being asked for nothing. The series identity is the +// (request, reason) PAIR, because the same reason means different things on +// different request kinds. +TEST(MetricMacros, serve_refused_total_keys_series_on_request_and_reason_pair) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // The exact call PeerImp::reportServeRefusal() makes, once per refused + // request. + auto const bump = [&app](char const* request, char const* reason) { + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::serveRefusedTotal, + "Peer data requests this node declined to serve, by request kind and cause", + {{telemetry::label::request, std::string(request)}, + {telemetry::label::reason, std::string(reason)}}); + }; + + // Same request kind, two different reasons -> two series. + bump("ledger", "sendq_full"); + bump("ledger", "sendq_full"); + bump("ledger", "not_found"); + // Different request kinds -> their own series, even sharing a reason. + bump("fetchpack", "load_shed"); + bump("fetchpack", "load_shed"); + bump("fetchpack", "load_shed"); + bump("txset", "not_found"); + + auto const data = provider.collect(); + + // Four distinct (request, reason) pairs -> exactly four series. + ASSERT_EQ(data.at("serve_refused_total").size(), 4u); + + // SELF-INFLICTED BACKPRESSURE: "sendq_full" and "load_shed" both mean this + // node chose not to answer because it was already behind. The fix is local + // (capacity, tuning), and the refusal directly slows the asking peer's sync. + EXPECT_EQ( + counterValue( + data, "serve_refused_total", attrs("request", "ledger", "reason", "sendq_full")), + 2); + EXPECT_EQ( + counterValue( + data, "serve_refused_total", attrs("request", "fetchpack", "reason", "load_shed")), + 3); + + // HISTORY GAP: "not_found" is not backpressure at all -- the node simply + // does not hold what was asked for. Same counter, completely different + // meaning and a completely different fix (history configuration), which is + // why it must not share a series with the two above. + EXPECT_EQ( + counterValue( + data, "serve_refused_total", attrs("request", "ledger", "reason", "not_found")), + 1); + EXPECT_EQ( + counterValue(data, "serve_refused_total", attrs("request", "txset", "reason", "not_found")), + 1); + + // The same reason on two different request kinds stays two series -- both + // present, each holding its own 1: a txset miss (consensus proposal data) + // and a ledger miss (history) are unrelated faults despite the shared slug. + EXPECT_EQ( + data.at("serve_refused_total").count(attrs("request", "ledger", "reason", "not_found")), + 1u); + EXPECT_EQ( + data.at("serve_refused_total").count(attrs("request", "txset", "reason", "not_found")), 1u); + + // Exactly two label keys on every series, and exactly these two. + for (auto const& [labels, point] : data.at("serve_refused_total")) + { + ASSERT_EQ(labels.size(), 2u); + EXPECT_EQ(labels.count("request"), 1u); + EXPECT_EQ(labels.count("reason"), 1u); + } + + // NEGATIVE: a pair that was never emitted has no series, so the four counts + // above are not an artifact of a catch-all series. + EXPECT_EQ( + data.at("serve_refused_total").count(attrs("request", "object", "reason", "sendq_full")), + 0u); +} + +// ledger_jump_total counts a node discarding its own chain tip to follow the +// network. It is deliberately UNLABELLED, so the single series' key is the empty +// attribute set -- and repeated jumps must accumulate on it, because a node +// thrashing between chains is the pattern worth alerting on and a single jump is +// not. +TEST(MetricMacros, ledger_jump_total_accumulates_on_one_unlabelled_series) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // The exact call NetworkOPsImp::switchLastClosedLedger() makes. Three jumps + // from the same call site, as a thrashing node would produce. + for (int i = 0; i < 3; ++i) + { + XRPL_METRIC_COUNTER_INC( + app, + telemetry::metric::ledgerJumpTotal, + "Forced jumps of the last closed ledger to a divergent chain"); + } + + auto const data = provider.collect(); + + // Exactly ONE series, and its key is the empty attribute set: the metric + // carries no labels at all. + ASSERT_EQ(data.at("ledger_jump_total").size(), 1u); + ASSERT_EQ(data.at("ledger_jump_total").count(otel_sdk::PointAttributes{}), 1u); + + // Three jumps accumulate to exactly 3 on that one series -- the reader is + // cumulative, so this is a running total, not a per-interval delta. + EXPECT_EQ(counterValue(data, "ledger_jump_total", otel_sdk::PointAttributes{}), 3); + + // ZERO labels, asserted on the series key itself. Both candidate labels were + // deliberately rejected as unbounded: the divergent ledger's hash is a + // 256-bit value and its sequence grows without limit, so either would mint a + // permanent new series per jump. The JLOG error line immediately above the + // emit already carries both, correlated to this series by node and time. + EXPECT_TRUE(data.at("ledger_jump_total").begin()->first.empty()); + EXPECT_EQ(data.at("ledger_jump_total").begin()->first.size(), 0u); + + // A fourth jump advances the SAME series to exactly 4 rather than creating a + // second one, which is what "no labels" has to mean over time. + XRPL_METRIC_COUNTER_INC( + app, + telemetry::metric::ledgerJumpTotal, + "Forced jumps of the last closed ledger to a divergent chain"); + auto const fourth = provider.collect(); + ASSERT_EQ(fourth.at("ledger_jump_total").size(), 1u); + EXPECT_EQ(counterValue(fourth, "ledger_jump_total", otel_sdk::PointAttributes{}), 4); +} + +// RUNTIME-DISABLED no-op proof for all four WP-A7 counters: with the registry +// disabled, every one emits NOTHING -- no series at all, and meter() is never +// consulted, so not even an instrument was created. Proves the isEnabled() gate +// short-circuits BEFORE any SDK work, which is what makes these emits free on a +// node with telemetry turned off. +TEST(MetricMacros, sync_supply_counters_emit_nothing_when_registry_disabled) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/false, provider.meter()); + + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::peerDisconnectTotal, + "Peer disconnects, by cause and connection direction", + {{telemetry::label::reason, std::string("large_sendq")}, + {telemetry::label::direction, std::string("outbound")}}); + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::peerAcceptTotal, + "Inbound peer connection attempts, by terminal outcome", + {{telemetry::label::outcome, std::string("accepted")}}); + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::serveRefusedTotal, + "Peer data requests this node declined to serve, by request kind and cause", + {{telemetry::label::request, std::string("ledger")}, + {telemetry::label::reason, std::string("sendq_full")}}); + XRPL_METRIC_COUNTER_INC( + app, + telemetry::metric::ledgerJumpTotal, + "Forced jumps of the last closed ledger to a divergent chain"); + + auto const data = provider.collect(); + + // Total absence, not zero-valued series: the instruments never existed. + EXPECT_EQ(data.count("peer_disconnect_total"), 0u); + EXPECT_EQ(data.count("peer_accept_total"), 0u); + EXPECT_EQ(data.count("serve_refused_total"), 0u); + EXPECT_EQ(data.count("ledger_jump_total"), 0u); + EXPECT_EQ(data.size(), 0u); + + // Cause, not just state: the isEnabled() gate short-circuited before the + // macros asked for a meter, so no instrument was created either. + EXPECT_EQ(app.registry().meterCalls(), 0); +} + +// ----------------------------------------------------------------- +// WP-A6: back-fill and persistence sync diagnostics +// ----------------------------------------------------------------- + +// Replay falling back to a full ledger acquire silently defeats the replay +// optimisation. The `stage` label must keep the skip-list and delta stages +// apart, because they fail for different reasons. +TEST(MetricMacros, ledger_replay_fallback_counter_separates_stages_by_exact_count) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // The skip-list stage falls back twice, the delta stage once: distinct + // counts so a collapsed label set cannot coincidentally look correct. + for (int i = 0; i < 2; ++i) + { + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::ledgerReplayFallbackTotal, + "Replay sub-acquires that fell back to a full ledger acquire", + {{telemetry::label::stage, std::string("skiplist")}}); + } + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::ledgerReplayFallbackTotal, + "Replay sub-acquires that fell back to a full ledger acquire", + {{telemetry::label::stage, std::string("delta")}}); + + auto const data = provider.collect(); + + // Exactly two series, and each carries its own exact total. + ASSERT_EQ(data.at("ledger_replay_fallback_total").size(), 2u); + EXPECT_EQ(counterValue(data, "ledger_replay_fallback_total", attrs("stage", "skiplist")), 2); + EXPECT_EQ(counterValue(data, "ledger_replay_fallback_total", attrs("stage", "delta")), 1); + + // NEGATIVE: a stage that never fell back has no series at all. An absent + // series, not a zero, is what a healthy replay path looks like. + EXPECT_EQ(data.at("ledger_replay_fallback_total").count(attrs("stage", "txset")), 0u); + + // The label key is exactly `stage` and is the only label, so this stays a + // bounded two-value group. + auto const& firstKey = data.at("ledger_replay_fallback_total").begin()->first; + ASSERT_EQ(firstKey.size(), 1u); + EXPECT_EQ(firstKey.begin()->first, "stage"); +} + +// The replay task's four terminal states must be four distinct series: a +// timeout and a failed build are diagnosed differently, and a success total +// with no failures is the only healthy reading. +TEST(MetricMacros, ledger_replay_outcome_counter_records_each_terminal_state_exactly) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // Distinct counts per outcome, matching the four terminal sites in + // LedgerReplayTask: success, timeout, build_failed, parameter_failed. + auto const record = [&app](char const* outcome, int times) { + for (int i = 0; i < times; ++i) + { + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::ledgerReplayOutcomeTotal, + "Ledger replay tasks by terminal outcome", + {{telemetry::label::outcome, std::string(outcome)}}); + } + }; + record("success", 3); + record("timeout", 2); + record("build_failed", 1); + record("parameter_failed", 4); + + auto const data = provider.collect(); + + ASSERT_EQ(data.at("ledger_replay_outcome_total").size(), 4u); + EXPECT_EQ(counterValue(data, "ledger_replay_outcome_total", attrs("outcome", "success")), 3); + EXPECT_EQ(counterValue(data, "ledger_replay_outcome_total", attrs("outcome", "timeout")), 2); + EXPECT_EQ( + counterValue(data, "ledger_replay_outcome_total", attrs("outcome", "build_failed")), 1); + EXPECT_EQ( + counterValue(data, "ledger_replay_outcome_total", attrs("outcome", "parameter_failed")), 4); + + // NEGATIVE: an outcome value the code never emits has no series, proving + // the four above are real label values and not a catch-all. + EXPECT_EQ(data.at("ledger_replay_outcome_total").count(attrs("outcome", "cancelled")), 0u); + + // The two WP-A6 replay counters are separate instruments, so a fallback + // never inflates an outcome total. + EXPECT_EQ(data.count("ledger_replay_fallback_total"), 0u); +} + +// Telemetry disabled at runtime: both replay counters must be complete no-ops. +// Absence of the instrument, not a zero-valued series. +TEST(MetricMacros, ledger_replay_counters_emit_nothing_when_disabled) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/false, provider.meter()); + + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::ledgerReplayFallbackTotal, + "Replay sub-acquires that fell back to a full ledger acquire", + {{telemetry::label::stage, std::string("skiplist")}}); + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::ledgerReplayOutcomeTotal, + "Ledger replay tasks by terminal outcome", + {{telemetry::label::outcome, std::string("timeout")}}); + + auto const data = provider.collect(); + + EXPECT_EQ(data.count("ledger_replay_fallback_total"), 0u); + EXPECT_EQ(data.count("ledger_replay_outcome_total"), 0u); + EXPECT_EQ(data.size(), 0u); + + // Cause, not just state: the isEnabled() gate short-circuited before the + // macros ever asked for a meter, so no instrument was created. + EXPECT_EQ(app.registry().meterCalls(), 0); +} + +// The nodestore_state gauge derives its two mean latencies from cumulative +// totals the node store already keeps. This mirrors the production callback in +// MetricsRegistry::observeNodeStoreTotals, whose enabled path cannot be linked +// into this binary (MetricsRegistry.cpp is excluded from xrpl_tests when +// telemetry is ON -- see src/tests/libxrpl/CMakeLists.txt), so the derivation +// is asserted here against the same inputs. The division itself is the real +// MetricsRegistry::scaledMean, a public constexpr inline that IS linkable, so +// this test exercises production arithmetic rather than a copy of it. +TEST(MetricMacros, nodestore_state_gauge_observes_exact_derived_means) +{ + // Each scenario gets a FRESH provider. The reader reports cumulative + // temporality (see CollectOnDemandReader), so a series observed by one + // scenario would still be present in the next collect() -- which would + // defeat the assertions below that a mean is ABSENT when its + // denominator is zero. + + // The four totals the production callback reads, chosen so each mean + // divides exactly and the two means differ: writes are 4x slower per + // operation than reads, which is the "existing DB back-fills slowly" + // shape this signal exists to show. + struct NodeStoreTotals + { + std::uint64_t storeCount; + std::uint64_t storeDurationUs; + std::uint64_t fetchCount; + std::uint64_t fetchDurationUs; + }; + NodeStoreTotals totals{ + .storeCount = 500, + .storeDurationUs = 2'000'000, // 2 s over 500 stores -> 4000 us + .fetchCount = 1000, + .fetchDurationUs = 1'000'000}; // 1 s over 1000 fetches -> 1000 us + + // Registers the production callback against one fresh provider and returns + // its single collection, so every scenario is measured in isolation. + auto collectWith = [](NodeStoreTotals& state) { + CollectingProvider const provider; + auto gauge = provider.meter()->CreateInt64ObservableGauge( + telemetry::metric::nodestoreState, + "NodeStore I/O counters, latencies, write-queue depth and acquisition stalls"); + gauge->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto const* self = static_cast(state); + auto observe = [&](char const* field, std::int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{telemetry::label::metric, field}}); + }; + // The four cumulative totals, observed unconditionally: for a + // total, zero is the meaningful "nothing yet" reading. + observe("node_writes", static_cast(self->storeCount)); + observe("node_reads_total", static_cast(self->fetchCount)); + observe( + "node_writes_duration_us", static_cast(self->storeDurationUs)); + observe("node_reads_duration_us", static_cast(self->fetchDurationUs)); + + // The two derived means, through the production helper. It + // returns nullopt when the denominator is 0, and the series is + // then omitted rather than observed as 0: a reported 0 us would + // claim the operation is instantaneous, which is worse than a + // visible gap. + using Registry = telemetry::MetricsRegistry; + if (auto const mean = Registry::scaledMean(self->fetchDurationUs, self->fetchCount)) + { + observe("read_mean_us", *mean); + } + if (auto const mean = Registry::scaledMean(self->storeDurationUs, self->storeCount)) + { + observe("write_mean_us", *mean); + } + }, + &state); + return provider.collect(); + }; + + auto const busy = collectWith(totals); + + // Exactly six series: the two means and the four cumulative totals that let + // a dashboard recover interval latency from them. + ASSERT_EQ(busy.at("nodestore_state").size(), 6u); + EXPECT_EQ(gaugeValue(busy, "nodestore_state", attrs("metric", "write_mean_us")), 4000); + EXPECT_EQ(gaugeValue(busy, "nodestore_state", attrs("metric", "read_mean_us")), 1000); + EXPECT_EQ(gaugeValue(busy, "nodestore_state", attrs("metric", "node_writes")), 500); + EXPECT_EQ(gaugeValue(busy, "nodestore_state", attrs("metric", "node_reads_total")), 1000); + EXPECT_EQ( + gaugeValue(busy, "nodestore_state", attrs("metric", "node_writes_duration_us")), 2'000'000); + EXPECT_EQ( + gaugeValue(busy, "nodestore_state", attrs("metric", "node_reads_duration_us")), 1'000'000); + + // The write mean is the signal this pair exists for, and it must be legible + // next to the read mean rather than merely present. + EXPECT_GT( + gaugeValue(busy, "nodestore_state", attrs("metric", "write_mean_us")), + gaugeValue(busy, "nodestore_state", attrs("metric", "read_mean_us"))); + + // Single fixed-cardinality label group, keyed exactly `metric`. + auto const& firstKey = busy.at("nodestore_state").begin()->first; + ASSERT_EQ(firstKey.size(), 1u); + EXPECT_EQ(firstKey.begin()->first, "metric"); + + // EDGE CASE: a node that has never written. The zero denominator must skip + // the mean rather than divide by zero, while the total is still reported -- + // that is what distinguishes "nothing written yet" from "writes are + // instant". The read side is unaffected and still reports both. + totals = NodeStoreTotals{ + .storeCount = 0, .storeDurationUs = 0, .fetchCount = 4, .fetchDurationUs = 800}; + auto const idle = collectWith(totals); + + EXPECT_EQ(idle.at("nodestore_state").count(attrs("metric", "write_mean_us")), 0u); + EXPECT_EQ(gaugeValue(idle, "nodestore_state", attrs("metric", "node_writes")), 0); + EXPECT_EQ(gaugeValue(idle, "nodestore_state", attrs("metric", "read_mean_us")), 200); + EXPECT_EQ(gaugeValue(idle, "nodestore_state", attrs("metric", "node_reads_total")), 4); + // Five series, not six: every total plus the one mean that is derivable. + EXPECT_EQ(idle.at("nodestore_state").size(), 5u); + + // EDGE CASE: integer division truncates rather than rounding. 7 stores + // over 100 us is 14.28 us, reported as 14 -- asserted so a future change + // to floating point is a deliberate, visible decision. + totals = NodeStoreTotals{ + .storeCount = 7, .storeDurationUs = 100, .fetchCount = 0, .fetchDurationUs = 0}; + auto const truncating = collectWith(totals); + + EXPECT_EQ(gaugeValue(truncating, "nodestore_state", attrs("metric", "write_mean_us")), 14); + // The read side now has the zero denominator, so its mean drops out too. + EXPECT_EQ(truncating.at("nodestore_state").count(attrs("metric", "read_mean_us")), 0u); + EXPECT_EQ(gaugeValue(truncating, "nodestore_state", attrs("metric", "node_reads_total")), 0); + + // EDGE CASE: stores counted, but every one measured below a microsecond. + // recordStoreDuration() only adds when the cast to microseconds is > 0, so + // sub-microsecond stores leave the duration total at 0 while the count + // climbs. scaledMean divides on the COUNT alone, so it reports a genuine + // mean of 0 here rather than omitting the series. That is the correct + // reading: the stores really did complete + // in under a microsecond each, and the total beside it proves they + // happened. Absence is reserved for "no samples at all". + totals = NodeStoreTotals{ + .storeCount = 9000, .storeDurationUs = 0, .fetchCount = 10, .fetchDurationUs = 50}; + auto const subMicrosecond = collectWith(totals); + + EXPECT_EQ(subMicrosecond.at("nodestore_state").count(attrs("metric", "write_mean_us")), 1u); + EXPECT_EQ(gaugeValue(subMicrosecond, "nodestore_state", attrs("metric", "write_mean_us")), 0); + // The count is published beside it, so the reading is interpretable: real + // write throughput with a sub-microsecond per-operation cost. + EXPECT_EQ(gaugeValue(subMicrosecond, "nodestore_state", attrs("metric", "node_writes")), 9000); + EXPECT_EQ( + gaugeValue(subMicrosecond, "nodestore_state", attrs("metric", "node_writes_duration_us")), + 0); + // The read side is independent and unaffected by the write-side reading. + EXPECT_EQ(gaugeValue(subMicrosecond, "nodestore_state", attrs("metric", "read_mean_us")), 5); + // All six series present: both means are derivable here. + EXPECT_EQ(subMicrosecond.at("nodestore_state").size(), 6u); +} + +// consensus_round_duration_ms: exact recorded values, not "greater than zero". +// Three rounds of known length must give a count of exactly 3 and a sum of +// exactly their total. Mirrors the one record site in +// RCLConsensus::Adaptor::makeAcceptSpan, which runs once per round. +// +// The 15200 sample is above the SDK's 10,000 default top boundary on purpose: +// that limit is why MetricsRegistry registers explicit buckets for this +// instrument (addRoundDurationHistogramView), and the sum must carry the real +// value through no matter how the value is bucketed. +TEST(MetricMacros, consensus_round_duration_records_exact_values) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // 3100 ms (healthy), 4250 ms (slow), 15200 ms (recovering) = 22550 ms. + for (std::int64_t const ms : {3100, 4250, 15200}) + { + XRPL_METRIC_HISTOGRAM_RECORD( + app, + telemetry::metric::consensusRoundDurationMs, + "Wall-clock duration of a completed consensus round in milliseconds", + ms); + } + + auto const data = provider.collect(); + + auto const [count, sum] = histogramCountAndSum(data, "consensus_round_duration_ms"); + EXPECT_EQ(count, 3u); + EXPECT_DOUBLE_EQ(sum, 22550.0); + + // Exactly ONE series carrying an EMPTY label set. The round histogram is + // deliberately unlabelled: a per-ledger or per-round label would mint a + // series per ledger, the same unbounded-cardinality trap the collector + // config documents for close_time. + ASSERT_EQ(data.at("consensus_round_duration_ms").size(), 1u); + EXPECT_TRUE(data.at("consensus_round_duration_ms").begin()->first.empty()); + + // One call site, three records: created once via std::call_once, then reused. + EXPECT_EQ(app.registry().meterCalls(), 1); +} + +// RUNTIME-DISABLED no-op proof for the round histogram: with the registry +// disabled, nothing is emitted -- not a zero-valued series but total absence -- +// and the macro never even asks for a meter, so no instrument is created. The +// runtime counterpart to the compile-time no-op (telemetry not built at all). +TEST(MetricMacros, consensus_round_duration_emits_nothing_when_registry_disabled) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/false, provider.meter()); + + XRPL_METRIC_HISTOGRAM_RECORD( + app, + telemetry::metric::consensusRoundDurationMs, + "Wall-clock duration of a completed consensus round in milliseconds", + 3100); + + auto const data = provider.collect(); + + // State: no series at all under that name, and nothing else leaked in. + EXPECT_EQ(data.count("consensus_round_duration_ms"), 0u); + EXPECT_EQ(data.size(), 0u); + // Cause, not just state: the isEnabled() gate short-circuited before the + // macro ever asked for a meter. + EXPECT_EQ(app.registry().meterCalls(), 0); +} + +// ----------------------------------------------------------------- +// WP-B5: per-sweep malloc_trim, and online_delete rotation writes +// +// Suspect 3. malloc_trim runs after EVERY cache sweep and its cost scales with +// the resident heap, so a node with a large existing database pays a per-sweep +// penalty a fresh one does not. The MallocTrimReport already carried every +// number; the emit site discarded it. +// +// Suspect 4. An online_delete rotation performs writes an ordinary fetch would +// not -- copy-forward from the doomed archive, plus copyNode re-stores -- which +// compete with sync I/O and only exist on a populated, already-rotated +// database. +// ----------------------------------------------------------------- + +// The three sweep instruments must stay three separate series carrying their own +// exact values, and the guards at the emit site must drop a reading that was +// never measured rather than publishing a zero for it. +// +// The 45000 us sample is above the SDK's 10,000 default top boundary on +// purpose: that is why MetricsRegistry registers the microsecond ladder for +// this instrument, and a 45 ms trim is exactly the large-heap case the signal +// exists to catch. The sum must carry the real value however it is bucketed. +TEST(MetricMacros, sweep_malloc_trim_records_exact_duration_faults_and_reclaim) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // Mirrors ApplicationImp::trimHeapAndRecord for three sweeps: a cheap trim + // on a small heap, an expensive one on a large heap, and one that reclaimed + // nothing. Distinct values so a collapsed instrument cannot look correct. + struct Sweep + { + std::int64_t durationUs; + std::int64_t minfltDelta; + std::int64_t reclaimedKb; + }; + for (auto const& sweep : { + Sweep{.durationUs = 120, .minfltDelta = 3, .reclaimedKb = 512}, + Sweep{.durationUs = 45'000, .minfltDelta = 900, .reclaimedKb = 262'144}, + Sweep{.durationUs = 80, .minfltDelta = 0, .reclaimedKb = 0}, + }) + { + XRPL_METRIC_HISTOGRAM_RECORD( + app, + telemetry::metric::sweepMallocTrimUs, + "Duration of the malloc_trim call ending each cache sweep (microseconds)", + sweep.durationUs); + if (sweep.minfltDelta > 0) + { + XRPL_METRIC_COUNTER_ADD( + app, + telemetry::metric::sweepMallocTrimMinorFaultsTotal, + "Minor page faults taken inside the sweep's malloc_trim call", + static_cast(sweep.minfltDelta)); + } + if (sweep.reclaimedKb > 0) + { + XRPL_METRIC_COUNTER_ADD( + app, + telemetry::metric::sweepMallocTrimReclaimedKbTotal, + "Resident kilobytes returned to the OS by the sweep's malloc_trim", + static_cast(sweep.reclaimedKb)); + } + } + + auto const data = provider.collect(); + + // Histogram: all three sweeps recorded, including the one whose fault and + // reclaim readings were skipped -- a trim always has a duration. + auto const [count, sum] = histogramCountAndSum(data, "sweep_malloc_trim_us"); + EXPECT_EQ(count, 3u); + EXPECT_NEAR(sum, 45'200.0, 1e-9); + + // Counters: cumulative totals, so the panel can rate() them. The zero-value + // third sweep contributed to neither. + EXPECT_EQ( + counterValue(data, "sweep_malloc_trim_minor_faults_total", otel_sdk::PointAttributes{}), + 903); + EXPECT_EQ( + counterValue(data, "sweep_malloc_trim_reclaimed_kb_total", otel_sdk::PointAttributes{}), + 262'656); + + // All three are unlabelled: one series each, per node. A label here would be + // unbounded (there is no bounded dimension a sweep varies over) and the + // dashboard reads one line per node instead. + ASSERT_EQ(data.at("sweep_malloc_trim_us").size(), 1u); + EXPECT_TRUE(data.at("sweep_malloc_trim_us").begin()->first.empty()); + ASSERT_EQ(data.at("sweep_malloc_trim_minor_faults_total").size(), 1u); + EXPECT_TRUE(data.at("sweep_malloc_trim_minor_faults_total").begin()->first.empty()); + ASSERT_EQ(data.at("sweep_malloc_trim_reclaimed_kb_total").size(), 1u); + EXPECT_TRUE(data.at("sweep_malloc_trim_reclaimed_kb_total").begin()->first.empty()); + + // Exactly the three instruments, so neither counter absorbed the other's + // adds and the histogram did not spawn a sibling. + EXPECT_EQ(data.size(), 3u); +} + +// EDGE CASE: nothing was measured. On a non-glibc platform mallocTrim() returns +// a report whose fields are all at their -1 sentinel, and the emit site's +// guards must publish NO series for it. A zero-valued series would claim the +// trim was instantaneous and reclaimed nothing, which is a different (and +// false) statement from "this platform has no trim". +// +// A FRESH provider, because the reader is cumulative: a second collect() on the +// provider used above would still show the earlier series and the absence +// assertions could not fail. +TEST(MetricMacros, sweep_malloc_trim_publishes_nothing_when_unmeasured) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // The sentinel report, exactly as MallocTrimReport default-constructs it. + MallocTrimReport const unsupported; + ASSERT_FALSE(unsupported.supported); + ASSERT_EQ(unsupported.durationUs.count(), -1); + ASSERT_EQ(unsupported.minfltDelta, -1); + ASSERT_EQ(unsupported.deltaKB(), 0); + + // trimHeapAndRecord's first guard: an unsupported report emits nothing at + // all, so the three statements below are never reached on such a platform. + if (unsupported.supported) + { + if (unsupported.durationUs.count() >= 0) + { + XRPL_METRIC_HISTOGRAM_RECORD( + app, + telemetry::metric::sweepMallocTrimUs, + "Duration of the malloc_trim call ending each cache sweep (microseconds)", + unsupported.durationUs.count()); + } + } + + auto const data = provider.collect(); + + // State: not one of the three series exists. + EXPECT_EQ(data.count("sweep_malloc_trim_us"), 0u); + EXPECT_EQ(data.count("sweep_malloc_trim_minor_faults_total"), 0u); + EXPECT_EQ(data.count("sweep_malloc_trim_reclaimed_kb_total"), 0u); + EXPECT_EQ(data.size(), 0u); +} + +// EDGE CASE: RSS GREW across the trim, because another thread allocated faster +// than the trim released. deltaKB() is then positive, and the reclaimed counter +// must skip it -- a counter cannot decrease, and there is no such thing as +// reclaiming a negative number of kilobytes. The duration is still recorded, +// because the trim did happen and did cost time. +TEST(MetricMacros, sweep_malloc_trim_skips_reclaim_when_rss_grew) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // A report whose after-RSS is ABOVE its before-RSS. + MallocTrimReport grew; + grew.supported = true; + grew.trimResult = 1; + grew.rssBeforeKB = 1'000'000; + grew.rssAfterKB = 1'000'800; + grew.durationUs = std::chrono::microseconds{9'000}; + grew.minfltDelta = 11; + + // Cause: the sign convention is after-minus-before, so growth is positive + // and a naive negation would add 800 to a "reclaimed" total. + ASSERT_EQ(grew.deltaKB(), 800); + + XRPL_METRIC_HISTOGRAM_RECORD( + app, + telemetry::metric::sweepMallocTrimUs, + "Duration of the malloc_trim call ending each cache sweep (microseconds)", + grew.durationUs.count()); + XRPL_METRIC_COUNTER_ADD( + app, + telemetry::metric::sweepMallocTrimMinorFaultsTotal, + "Minor page faults taken inside the sweep's malloc_trim call", + static_cast(grew.minfltDelta)); + if (auto const deltaKB = grew.deltaKB(); deltaKB < 0) + { + XRPL_METRIC_COUNTER_ADD( + app, + telemetry::metric::sweepMallocTrimReclaimedKbTotal, + "Resident kilobytes returned to the OS by the sweep's malloc_trim", + static_cast(-deltaKB)); + } + + auto const data = provider.collect(); + + // The duration and the faults are real and are recorded. + auto const [count, sum] = histogramCountAndSum(data, "sweep_malloc_trim_us"); + EXPECT_EQ(count, 1u); + EXPECT_NEAR(sum, 9'000.0, 1e-9); + EXPECT_EQ( + counterValue(data, "sweep_malloc_trim_minor_faults_total", otel_sdk::PointAttributes{}), + 11); + + // NEGATIVE: the reclaim counter has no series at all. Absence, not a zero + // and emphatically not 800. + EXPECT_EQ(data.count("sweep_malloc_trim_reclaimed_kb_total"), 0u); + EXPECT_EQ(data.size(), 2u); +} + +// rotation_state is polled from the node store, so the production callback in +// MetricsRegistry::registerRotationStateGauge cannot be linked into this +// binary. The derivation it performs is asserted here against the same two +// inputs, mirroring how nodestore_state is tested above. +TEST(MetricMacros, rotation_state_gauge_observes_in_flight_window_and_copy_forward_total) +{ + // Each scenario gets a FRESH provider: the reader is cumulative, so a + // series observed once would persist into the next collect() and the + // "no rotating store means no series" assertion below could not fail. + struct RotationState + { + bool isRotating; + std::uint64_t copyForwardTotal; + bool hasRotatingStore; + }; + + auto collectWith = [](RotationState& state) { + CollectingProvider const provider; + auto gauge = provider.meter()->CreateInt64ObservableGauge( + telemetry::metric::rotationState, + "Online-delete rotation state and copy-forward write total"); + gauge->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto const* self = static_cast(state); + // The production dynamic_cast: a node without online_delete has + // a non-rotating store and publishes nothing. + if (!self->hasRotatingStore) + return; + auto observe = [&](char const* field, std::int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{telemetry::label::metric, field}}); + }; + observe( + telemetry::lval::rotation_state::inFlight, + static_cast(self->isRotating ? 1 : 0)); + observe( + telemetry::lval::rotation_state::copyForward, + static_cast(self->copyForwardTotal)); + }, + &state); + return provider.collect(); + }; + + // A rotation is running and has already forced 4096 copy-forward writes: + // the shape an operator should see, and the one that explains sync I/O + // contention on a populated online_delete database. + RotationState state{.isRotating = true, .copyForwardTotal = 4096, .hasRotatingStore = true}; + auto const rotating = collectWith(state); + + ASSERT_EQ(rotating.at("rotation_state").size(), 2u); + EXPECT_EQ(gaugeValue(rotating, "rotation_state", attrs("metric", "in_flight")), 1); + EXPECT_EQ(gaugeValue(rotating, "rotation_state", attrs("metric", "copy_forward")), 4096); + + // Single fixed-cardinality label group, keyed exactly `metric`. + auto const& firstKey = rotating.at("rotation_state").begin()->first; + ASSERT_EQ(firstKey.size(), 1u); + EXPECT_EQ(firstKey.begin()->first, "metric"); + + // Between rotations: in_flight drops to 0 while the total HOLDS its value. + // That combination is the whole point of pairing them -- the extra writes + // are not happening now, but they did, and the total must not reset (a + // counter that drops cannot be rated). + state = RotationState{.isRotating = false, .copyForwardTotal = 4096, .hasRotatingStore = true}; + auto const idle = collectWith(state); + + EXPECT_EQ(gaugeValue(idle, "rotation_state", attrs("metric", "in_flight")), 0); + EXPECT_EQ(gaugeValue(idle, "rotation_state", attrs("metric", "copy_forward")), 4096); + // Both series still exist while idle: a zero in_flight is a real reading, + // and absence would be the regression. + EXPECT_EQ(idle.at("rotation_state").size(), 2u); + + // EDGE CASE: a fresh node that has online_delete configured but has never + // rotated. Both readings are 0 and BOTH series still exist -- 0 copy-forward + // writes is the healthy answer to "what did rotation cost", not missing data. + state = RotationState{.isRotating = false, .copyForwardTotal = 0, .hasRotatingStore = true}; + auto const neverRotated = collectWith(state); + + EXPECT_EQ(gaugeValue(neverRotated, "rotation_state", attrs("metric", "in_flight")), 0); + EXPECT_EQ(gaugeValue(neverRotated, "rotation_state", attrs("metric", "copy_forward")), 0); + EXPECT_EQ(neverRotated.at("rotation_state").size(), 2u); + + // NEGATIVE: no rotating store at all -- online_delete is not configured, so + // the node store is a DatabaseNodeImp and the cast fails. NO series is + // published, deliberately: an absent series means "rotation is not + // configured", which a zero would misreport as "rotation is free". + state = RotationState{.isRotating = false, .copyForwardTotal = 0, .hasRotatingStore = false}; + auto const notConfigured = collectWith(state); + + EXPECT_EQ(notConfigured.count("rotation_state"), 0u); + EXPECT_EQ(notConfigured.size(), 0u); +} + +// The copyNode re-store counter: one increment per node rescued from neither +// backend, on ONE unlabelled series. The node hash must never become a label -- +// it is unbounded runtime data and would mint one series per rescued node. +TEST(MetricMacros, rotation_copy_node_restore_accumulates_on_one_unlabelled_series) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/true, provider.meter()); + + // Seven rescued nodes across one rotation's state-map walk. + for (int i = 0; i < 7; ++i) + { + XRPL_METRIC_COUNTER_INC( + app, + telemetry::metric::rotationCopyNodeRestoreTotal, + "Nodes re-stored during rotation because they were missing from both backends"); + } + + auto const data = provider.collect(); + + ASSERT_EQ(data.at("rotation_copy_node_restore_total").size(), 1u); + EXPECT_EQ( + counterValue(data, "rotation_copy_node_restore_total", otel_sdk::PointAttributes{}), 7); + + // The single series carries NO labels, which is what bounds its cardinality + // to one per node however many distinct hashes were rescued. + EXPECT_TRUE(data.at("rotation_copy_node_restore_total").begin()->first.empty()); + + // The rotation gauge is a separate instrument, so the counter cannot inflate + // the copy-forward total: they measure two different extra writes. + EXPECT_EQ(data.count("rotation_state"), 0u); + EXPECT_EQ(data.size(), 1u); +} + +// RUNTIME-DISABLED no-op proof for all four WP-B5 push instruments. With the +// registry disabled nothing is emitted -- total absence, not zero-valued series +// -- and no macro ever asks for a meter. +TEST(MetricMacros, sweep_and_rotation_metrics_emit_nothing_when_registry_disabled) +{ + CollectingProvider const provider; + FakeApp app; + wire(app, /*enabled=*/false, provider.meter()); + + XRPL_METRIC_HISTOGRAM_RECORD( + app, + telemetry::metric::sweepMallocTrimUs, + "Duration of the malloc_trim call ending each cache sweep (microseconds)", + 45'000); + XRPL_METRIC_COUNTER_ADD( + app, + telemetry::metric::sweepMallocTrimMinorFaultsTotal, + "Minor page faults taken inside the sweep's malloc_trim call", + 900); + XRPL_METRIC_COUNTER_ADD( + app, + telemetry::metric::sweepMallocTrimReclaimedKbTotal, + "Resident kilobytes returned to the OS by the sweep's malloc_trim", + 262'144); + XRPL_METRIC_COUNTER_INC( + app, + telemetry::metric::rotationCopyNodeRestoreTotal, + "Nodes re-stored during rotation because they were missing from both backends"); + + auto const data = provider.collect(); + + // State: no series under any of the four names, and nothing else leaked in. + EXPECT_EQ(data.count("sweep_malloc_trim_us"), 0u); + EXPECT_EQ(data.count("sweep_malloc_trim_minor_faults_total"), 0u); + EXPECT_EQ(data.count("sweep_malloc_trim_reclaimed_kb_total"), 0u); + EXPECT_EQ(data.count("rotation_copy_node_restore_total"), 0u); + EXPECT_EQ(data.size(), 0u); + + // Cause, not just state: the isEnabled() gate short-circuited before any + // macro asked for a meter, so no instrument was ever created. + EXPECT_EQ(app.registry().meterCalls(), 0); +} + #endif // XRPL_ENABLE_TELEMETRY diff --git a/src/tests/libxrpl/telemetry/MetricsRegistry.cpp b/src/tests/libxrpl/telemetry/MetricsRegistry.cpp index f21ddfe8319..1e26926452e 100644 --- a/src/tests/libxrpl/telemetry/MetricsRegistry.cpp +++ b/src/tests/libxrpl/telemetry/MetricsRegistry.cpp @@ -19,6 +19,41 @@ * when XRPL_ENABLE_TELEMETRY is defined MetricsRegistry.cpp is not * compiled into this binary (see src/tests/libxrpl/CMakeLists.txt) and * its out-of-line symbols are unresolvable here. + * + * Tests cover: + * - Construction with telemetry disabled (no-op behavior). + * - start()/stop() lifecycle when disabled. + * - Synchronous instrument recording methods do not crash when disabled. + * - Double stop() is safe. + * - Destructor handles cleanup without crash. + * - Compile-time-disabled proof for the sync-diagnostics gauges: the whole + * async-gauge registration surface is compiled away, and a full disabled + * lifecycle never touches any ServiceRegistry service -- including the + * Overlay and AmendmentTable the WP-A7 gauges would read. + * + * NOTE: These tests only exercise the no-op path (telemetry disabled). + * When XRPL_ENABLE_TELEMETRY is defined, MetricsRegistry.cpp pulls in + * xrpld symbols that cannot be linked into this standalone test binary, + * so the tests are compiled out. + * + * CONSEQUENCE for the sync-diagnostics gauges (`unl_quorum`, + * `clock_close_offset_seconds`, `sync_state`, + * `server_stall_events_total`, `sync_acquire`, `shamap_cache_hit_rate`, + * `jobq_saturation`, `peer_ledger_supply`, + * `peerfinder_slot_census`, `amendment_block`, `nodestore_state`): + * this file CANNOT assert an observed gauge + * value, because on this build the gauges do not exist -- their registration + * methods and the OTel instrument members are inside + * `#ifdef XRPL_ENABLE_TELEMETRY`, and there is no MeterProvider at all. What + * is provable here, and what the tests below assert, is the complementary + * half: that nothing is registered and no service is consulted. The exact + * observed values (trusted_keys=5, quorum=4, offset=-3, the sync_state / + * stall-episode values, the acquire-progress / cache-hit-rate values, the + * per-type backlog / pool-saturation values, the peer-supply / + * slot-census / amendment-countdown values, and the nodestore + * read/write mean-latency values) are + * asserted in MetricMacros.cpp, which is the file compiled when telemetry IS + * enabled. */ #include @@ -430,6 +465,8 @@ TEST(MetricsRegistryScaledMean, default_scale_is_one) #include #include #include +#include +#include using namespace xrpl; @@ -761,4 +798,173 @@ TEST_F(MetricsRegistryTest, destructor_calls_stop) // If we get here without crash, the destructor handled stop. } +// ----------------------------------------------------------------- +// Sync-diagnostics gauges: compile-time-disabled proof. +// +// `unl_quorum` reads ValidatorList::trustedKeyCount() and quorum(); +// `clock_close_offset_seconds` reads TimeKeeper::closeOffset(); `sync_state` and +// `server_stall_events_total` read NetworkOPs and LoadManager; `sync_acquire` +// reads InboundLedgers::acquireProgress() and `shamap_cache_hit_rate` reads the +// node Family's tree-node cache; `jobq_saturation` reads +// JobQueue::getWorkerSaturation(); `peer_ledger_supply` and +// `peerfinder_slot_census` read Overlay::getPeerLedgerSupply() / +// getSlotCensus() and `amendment_block` reads +// AmendmentTable::firstUnsupportedExpected(). All are +// reached through the ServiceRegistry, and MockServiceRegistry::getValidators() +// / getTimeKeeper() / getOPs() / getLoadManager() / getInboundLedgers() / +// getNodeFamily() / getJobQueue() / getOverlay() / getAmendmentTable() THROW +// std::logic_error. So "no +// gauge callback ran" is directly observable here: had registerAsyncGauges() run +// and had a callback fired, one of those accessors would have thrown. +// +// Honest scope note: these tests do NOT assert an observed gauge value. On this +// build the gauges are not compiled at all (see the file header), so there is no +// value to read -- inventing one would be fiction. The value assertions live in +// MetricMacros.cpp. What is asserted here is the other half of the contract: +// registration is absent and no service is consulted. +// ----------------------------------------------------------------- + +// The observable-gauge registration surface is compiled OUT when telemetry is +// disabled: `meter()` -- the only accessor the gauges and the XRPL_METRIC_* +// macros use to reach the OTel SDK -- does not exist as a member at all. This is +// a compile-time assertion, so it fails the build (not the run) if the accessor +// ever escapes its #ifdef and drags the SDK into a telemetry-off build. +TEST_F(MetricsRegistryTest, disabled_build_exposes_no_meter_accessor) +{ + // Detects `registry.meter()` being callable. Under #ifndef + // XRPL_ENABLE_TELEMETRY it must not be, so the trait is false. + auto hasMeter = [](T* r) { return requires { r->meter(); }; }; + EXPECT_FALSE(hasMeter(static_cast(nullptr))); + + // The enable flag is still queryable and reports exactly false -- the class + // is a no-op, not an absent type. + telemetry::MetricsRegistry const registry(false, mockApp_, j_); + EXPECT_FALSE(registry.isEnabled()); +} + +// A full disabled lifecycle registers no gauge and therefore consults NO +// ServiceRegistry service. Asserting the cause, not just the absence of a crash: +// every MockServiceRegistry accessor a sync-diagnostics gauge would need throws, +// so reaching the end without an exception proves no callback ran. +TEST_F(MetricsRegistryTest, disabled_lifecycle_never_consults_gauge_services) +{ + telemetry::MetricsRegistry registry(false, mockApp_, j_); + + // start() is where registerAsyncGauges() -- and with it + // registerUnlQuorumGauge() / registerClockSkewGauge() / + // registerSyncStateGauge() / registerStallEventsCounter() / + // registerSyncAcquireGauge() / registerCacheHitRateDetailGauge() / + // registerJobQueueBacklogGauge() / registerJobQueueSaturationGauge() / + // registerPeerLedgerSupplyGauge() / registerSlotCensusGauge() / + // registerAmendmentBlockGauge() / registerNodeStoreGauge() -- + // would run. + EXPECT_NO_THROW(registry.start("http://localhost:4318/v1/metrics")); + + // detachCallbacks() is the shutdown hook the real gauges honour. It must be + // safe and idempotent even though there is nothing to detach. + EXPECT_NO_THROW(registry.detachCallbacks()); + EXPECT_NO_THROW(registry.detachCallbacks()); + + // Still disabled after a full start(): start() must not flip the flag. + EXPECT_FALSE(registry.isEnabled()); + + EXPECT_NO_THROW(registry.stop()); + + // Positive control: the mock DOES throw when a gauge-backing service is + // actually requested. Without this, "nothing threw" would be vacuous -- it + // could mean the mock is permissive rather than that no callback ran. + EXPECT_THROW(mockApp_.getValidators(), std::logic_error); + EXPECT_THROW(mockApp_.getTimeKeeper(), std::logic_error); + // The two services the WP-A2 sync-state signals read. sync_state needs both + // (NetworkOPs for the gate/duration/ledgers-behind, LoadManager for stall + // seconds) and server_stall_events_total needs the second, so either one + // firing would have thrown above. + EXPECT_THROW(mockApp_.getOPs(), std::logic_error); + EXPECT_THROW(mockApp_.getLoadManager(), std::logic_error); + // The two services the WP-A3 acquire signals read: sync_acquire polls the + // in-flight acquire collection, shamap_cache_hit_rate polls the node + // Family's tree-node cache. Neither was consulted above. + EXPECT_THROW(mockApp_.getInboundLedgers(), std::logic_error); + EXPECT_THROW(mockApp_.getNodeFamily(), std::logic_error); + // The service the WP-A4 job-queue gauge reads: jobq_saturation polls + // getWorkerSaturation() on the JobQueue. Not consulted above, so the + // gauge never took the JobQueue mutex on a telemetry-off build. + EXPECT_THROW(mockApp_.getJobQueue(), std::logic_error); + // The service both WP-A7 peer gauges read: peer_ledger_supply polls + // getPeerLedgerSupply(), which walks the active-peer list, and + // peerfinder_slot_census polls getSlotCensus(), which takes the PeerFinder + // lock. Both go through the Overlay, so a single throw here proves neither + // gauge walked the peer list nor took the PeerFinder lock on a + // telemetry-off build. + EXPECT_THROW(mockApp_.getOverlay(), std::logic_error); + // The service the WP-A7 amendment countdown reads: amendment_block polls + // firstUnsupportedExpected() on the AmendmentTable, which takes that + // table's mutex. Not consulted above, so the countdown never ran. (Its + // `warned` half reads NetworkOPs, already covered by the getOPs() check.) + EXPECT_THROW(mockApp_.getAmendmentTable(), std::logic_error); + // The service the nodestore gauge reads: nodestore_state polls + // getStoreDurationUs()/getStoreCount() and + // getFetchDurationUs()/getFetchTotalCount() on the node-store Database, + // alongside its I/O totals and write-queue detail. Not consulted above, + // so the gauge never read those atomics on a telemetry-off build. + EXPECT_THROW(mockApp_.getNodeStore(), std::logic_error); +} + +// Even asking for enabled=true registers no sync-diagnostics gauge on a +// telemetry-off build. isEnabled() faithfully echoes the constructor argument +// (the flag lives outside the #ifdef), so the flag alone does NOT prove the +// gauges are inert -- the mock does: a full start()/stop() cycle with +// enabled=true still consults no service, so no callback was ever registered. +// Asserts the exact flag value on BOTH construction paths. +TEST_F(MetricsRegistryTest, enabled_flag_alone_registers_no_gauges_when_compiled_out) +{ + telemetry::MetricsRegistry enabledRequest(true, mockApp_, j_); + + // The flag is echoed back exactly, true not false: it is a plain member, + // not gated on XRPL_ENABLE_TELEMETRY. + EXPECT_TRUE(enabledRequest.isEnabled()); + + // Yet the whole lifecycle stays inert. If registerAsyncGauges() had run and + // registered registerUnlQuorumGauge()/registerClockSkewGauge()/ + // registerSyncStateGauge()/registerStallEventsCounter()/ + // registerSyncAcquireGauge()/registerCacheHitRateDetailGauge()/ + // registerJobQueueBacklogGauge()/registerJobQueueSaturationGauge()/ + // registerPeerLedgerSupplyGauge()/registerSlotCensusGauge()/ + // registerAmendmentBlockGauge()/registerNodeStoreGauge(), a + // callback would reach getValidators()/getTimeKeeper()/getOPs()/ + // getLoadManager()/getInboundLedgers()/getNodeFamily()/getJobQueue()/ + // getOverlay()/getAmendmentTable()/getNodeStore() and + // throw std::logic_error. + EXPECT_NO_THROW(enabledRequest.start("http://localhost:4318/v1/metrics")); + EXPECT_NO_THROW(enabledRequest.detachCallbacks()); + EXPECT_NO_THROW(enabledRequest.stop()); + + // Contrast: enabled=false reports exactly false. + telemetry::MetricsRegistry const disabledRequest(false, mockApp_, j_); + EXPECT_FALSE(disabledRequest.isEnabled()); +} + +// The `state_changes_total` counter no longer has a registry-owned wrapper +// method: WP-A2 moved it to a labelled call-site macro in +// NetworkOPsImp::setMode so it can carry {from,to}. This compile-time +// assertion is the regression guard -- if someone reintroduces +// incrementStateChanges(), the unlabelled instrument would coexist with the +// labelled one and Prometheus would carry two conflicting versions of the same +// metric name. +TEST_F(MetricsRegistryTest, state_changes_counter_has_no_registry_wrapper) +{ + auto hasIncrementStateChanges = [](T* r) { + return requires { r->incrementStateChanges(); }; + }; + EXPECT_FALSE(hasIncrementStateChanges(static_cast(nullptr))); + + // Positive control: a sibling parity counter that WAS deliberately kept as + // a registry wrapper is still detectable, so the trait above is really + // probing for the method and not vacuously false. + auto hasIncrementLedgersClosed = [](T* r) { + return requires { r->incrementLedgersClosed(); }; + }; + EXPECT_TRUE(hasIncrementLedgersClosed(static_cast(nullptr))); +} + #endif // !XRPL_ENABLE_TELEMETRY diff --git a/src/tests/libxrpl/telemetry/SpanGuardScope.cpp b/src/tests/libxrpl/telemetry/SpanGuardScope.cpp index 8795b4f6147..95a0b0d9088 100644 --- a/src/tests/libxrpl/telemetry/SpanGuardScope.cpp +++ b/src/tests/libxrpl/telemetry/SpanGuardScope.cpp @@ -733,6 +733,232 @@ TEST_F(SpanGuardScopeTest, scopedGuard_cross_thread_death_asserts_at_wrong_store #endif } +// =========================================================================== +// Per-ledger trace join (WP-B3) +// =========================================================================== +// +// One ledger's spans are produced on threads that share no context: +// ledger.acquire on a JtLedgerData worker, ledger.validate from whichever thread +// enters LedgerMaster::checkAccept (a peer thread via handleNewValidation, the +// acquire-completion job, or the consensus thread via switchLCL), +// consensus.validation.accept from a validation worker, ledger.store from a +// fourth. No ambient context reaches across those boundaries, so each used to be +// its own single-span trace and a slow ledger could not be read as one unit. +// +// The join derives the trace id from the ledger hash, which every one of those +// sites already holds. The whole contract is therefore: spans built from the +// SAME hash share a trace id, spans built from DIFFERENT hashes do not, and each +// is a true root rather than hanging off an unrelated ambient parent. That is a +// property of hashSpan plus the key, with no LedgerMaster state involved, so it +// is asserted here against the same in-memory exporter the tests above use. +// +// The production call site (LedgerMaster::makeLedgerTraceSpan) cannot be called +// from this binary, which links libxrpl and not xrpld. What it adds over the raw +// factory is two attributes and the arguments it forwards; the causality +// property the emitters depend on, and that no compiler enforces, is the one +// asserted here. + +/** + * Build a distinct 32-byte stand-in for a ledger hash. + * + * Full width on purpose: the emitters pass the whole 32-byte hash and hashSpan + * consumes the first 16, so a 32-byte input is what the real call looks like. + * The seed varies the FIRST byte, inside the 16 that become the trace id, so two + * seeds really do produce two trace ids -- a seed placed in the tail would be + * truncated away and the difference assertion would pass for the wrong reason. + * + * @param seed Distinguishes one fake ledger from another. + * @return 32 bytes whose leading 16 differ whenever the seed differs. + */ +std::array +makeLedgerHashBytes(std::uint8_t seed) +{ + std::array h{}; + for (std::size_t i = 0; i < h.size(); ++i) + h[i] = static_cast(seed + i); + return h; +} + +// THE CONTRACT, positive half: three stages of one ledger, created +// independently, all land in ONE trace whose id IS the ledger hash. This is what +// makes a slow ledger readable as one connected trace instead of three orphans. +TEST_F(SpanGuardScopeTest, ledgerJoin_same_hash_puts_every_stage_in_one_trace) +{ + auto const h = makeLedgerHashBytes(0x11); + { + // Deliberately not nested and not in pipeline order: each stage is built + // on its own, exactly as the three emitters do on three threads. + auto acquire = + SpanGuard::hashSpan(TraceCategory::Ledger, "ledger.acquire", h.data(), h.size()); + auto validate = + SpanGuard::hashSpan(TraceCategory::Ledger, "ledger.validate", h.data(), h.size()); + auto store = SpanGuard::hashSpan(TraceCategory::Ledger, "ledger.store", h.data(), h.size()); + ASSERT_TRUE(static_cast(acquire)); + ASSERT_TRUE(static_cast(validate)); + ASSERT_TRUE(static_cast(store)); + } + + auto spans = spanData()->GetSpans(); + ASSERT_EQ(spans.size(), 3u); + + auto* acquire = findSpan(spans, "ledger.acquire"); + auto* validate = findSpan(spans, "ledger.validate"); + auto* store = findSpan(spans, "ledger.store"); + ASSERT_NE(acquire, nullptr); + ASSERT_NE(validate, nullptr); + ASSERT_NE(store, nullptr); + + // ONE trace: every stage shares the trace id. This is the whole point. + EXPECT_EQ(validate->GetTraceId(), acquire->GetTraceId()); + EXPECT_EQ(store->GetTraceId(), acquire->GetTraceId()); + + // And the trace id IS the key's first 16 bytes -- not merely equal to each + // other by chance. That is what lets an operator go from a ledger hash to + // its trace and back. + EXPECT_EQ(std::memcmp(acquire->GetTraceId().Id().data(), h.data(), 16), 0); + + // Still three DISTINCT spans, not one span reused. + EXPECT_NE(acquire->GetSpanId(), validate->GetSpanId()); + EXPECT_NE(validate->GetSpanId(), store->GetSpanId()); +} + +// THE CONTRACT, negative half: two different ledgers must never share a trace. +// Without this the join would be useless -- a single trace would accumulate +// every ledger the node ever touched. +TEST_F(SpanGuardScopeTest, ledgerJoin_different_hashes_never_share_a_trace) +{ + auto const first = makeLedgerHashBytes(0x20); + auto const second = makeLedgerHashBytes(0x60); + { + auto a = SpanGuard::hashSpan( + TraceCategory::Ledger, "ledger.validate", first.data(), first.size()); + auto b = SpanGuard::hashSpan( + TraceCategory::Ledger, "ledger.store", second.data(), second.size()); + ASSERT_TRUE(static_cast(a)); + ASSERT_TRUE(static_cast(b)); + } + + auto spans = spanData()->GetSpans(); + ASSERT_EQ(spans.size(), 2u); + auto* a = findSpan(spans, "ledger.validate"); + auto* b = findSpan(spans, "ledger.store"); + ASSERT_NE(a, nullptr); + ASSERT_NE(b, nullptr); + + EXPECT_NE(a->GetTraceId(), b->GetTraceId()); + // Each still matches its OWN key, so they are separate because the keys + // differ, not because one of them failed to adopt its hash at all. + EXPECT_EQ(std::memcmp(a->GetTraceId().Id().data(), first.data(), 16), 0); + EXPECT_EQ(std::memcmp(b->GetTraceId().Id().data(), second.data(), 16), 0); +} + +// A joined span is a TRUE ROOT, never a child of whatever was active on the +// emitting thread. checkAccept is entered from a peer thread, from the +// acquire-completion job and from the consensus thread; if the span inherited an +// ambient parent it would be swallowed into an unrelated trace on some of those +// paths and its trace id would no longer be the ledger hash. +TEST_F(SpanGuardScopeTest, ledgerJoin_ignores_an_ambient_parent) +{ + auto const h = makeLedgerHashBytes(0x33); + { + // An unrelated span active on this thread: the situation on a reused + // job-queue worker. + ScopedSpanGuard const ambient(TraceCategory::Rpc, "rpc", "command"); + ASSERT_TRUE(static_cast(ambient)); + + auto joined = + SpanGuard::hashSpan(TraceCategory::Ledger, "ledger.validate", h.data(), h.size()); + ASSERT_TRUE(static_cast(joined)); + } + + auto spans = spanData()->GetSpans(); + auto* ambientSpan = findSpan(spans, "rpc.command"); + auto* joined = findSpan(spans, "ledger.validate"); + ASSERT_NE(ambientSpan, nullptr); + ASSERT_NE(joined, nullptr); + + // No parent, and NOT in the ambient span's trace. + EXPECT_FALSE(joined->GetParentSpanId().IsValid()); + EXPECT_NE(joined->GetTraceId(), ambientSpan->GetTraceId()); + // The deterministic id survived: the ambient span did not displace it. + EXPECT_EQ(std::memcmp(joined->GetTraceId().Id().data(), h.data(), 16), 0); +} + +// The validation-accept span joins the trace of the ledger it validates, so +// "which validation drove this acceptance, and how long did the acceptance take" +// is one trace. It is keyed on the VALIDATED ledger hash -- the key +// ledger.validate uses -- and deliberately NOT on the previous-ledger hash that +// seeds the consensus round trace, which stays a separate trace. +TEST_F(SpanGuardScopeTest, ledgerJoin_validationAccept_joins_the_validated_ledger) +{ + auto const validated = makeLedgerHashBytes(0x41); + auto const previous = makeLedgerHashBytes(0x81); + { + auto validation = SpanGuard::hashSpan( + TraceCategory::Ledger, + "consensus.validation.accept", + validated.data(), + validated.size()); + auto accept = SpanGuard::hashSpan( + TraceCategory::Ledger, "ledger.validate", validated.data(), validated.size()); + // The round trace, seeded on the PREVIOUS ledger: a different trace. + auto round = SpanGuard::hashSpan( + TraceCategory::Consensus, "consensus.round", previous.data(), previous.size()); + ASSERT_TRUE(static_cast(validation)); + ASSERT_TRUE(static_cast(accept)); + ASSERT_TRUE(static_cast(round)); + } + + auto spans = spanData()->GetSpans(); + auto* validation = findSpan(spans, "consensus.validation.accept"); + auto* accept = findSpan(spans, "ledger.validate"); + auto* round = findSpan(spans, "consensus.round"); + ASSERT_NE(validation, nullptr); + ASSERT_NE(accept, nullptr); + ASSERT_NE(round, nullptr); + + // Joined to the acceptance of the ledger it validates... + EXPECT_EQ(validation->GetTraceId(), accept->GetTraceId()); + // ...and separate from the round trace, which is keyed on another ledger. + EXPECT_NE(validation->GetTraceId(), round->GetTraceId()); +} + +// A key SHORTER than 16 bytes cannot seed a trace id, so hashSpan yields a null +// guard rather than a span in a garbage trace. The emitters always pass a full +// 32-byte hash, so this is the guard rail: a truncated key degrades to "no span" +// instead of to a wrong join. +TEST_F(SpanGuardScopeTest, ledgerJoin_too_short_a_key_yields_no_span) +{ + std::array const tooShort{1, 2, 3, 4, 5, 6, 7, 8}; + auto span = SpanGuard::hashSpan( + TraceCategory::Ledger, "ledger.validate", tooShort.data(), tooShort.size()); + EXPECT_FALSE(static_cast(span)); + + // Nothing exported at all, rather than a span carrying an invalid trace id. + EXPECT_EQ(spanData()->GetSpans().size(), 0u); +} + +// DISABLED PATH: with no Telemetry instance installed the join is a no-op that +// still compiles and returns a usable null guard, which is why +// LedgerMaster::makeLedgerTraceSpan needs no #ifdef or enabled-check of its own. +// Not a fixture test: the point is that NO instance is installed. +TEST(SpanGuardLedgerJoinDisabled, join_is_a_no_op_when_telemetry_is_absent) +{ + Telemetry::setInstance(nullptr); + + auto const h = makeLedgerHashBytes(0x55); + auto span = SpanGuard::hashSpan(TraceCategory::Ledger, "ledger.validate", h.data(), h.size()); + + // Null guard, and its whole surface is still safe to drive. + EXPECT_FALSE(static_cast(span)); + span.setAttribute("ledger_hash", "deadbeef"); + span.setAttribute("ledger_seq", static_cast(42)); + span.setOk(); + // It yields no context either, so no caller can build a child in a bogus + // trace off the back of it. + EXPECT_FALSE(span.spanContext().isValid()); +} + } // namespace } // namespace xrpl::telemetry diff --git a/src/tests/libxrpl/telemetry/SyncStateSignals.cpp b/src/tests/libxrpl/telemetry/SyncStateSignals.cpp new file mode 100644 index 00000000000..b0c2f3abd29 --- /dev/null +++ b/src/tests/libxrpl/telemetry/SyncStateSignals.cpp @@ -0,0 +1,140 @@ +/** + * @file SyncStateSignals.cpp + * Unit tests for the decision rule behind the sync-state stall signals. + * + * `sync_state{metric="server_stall_seconds"}` and + * `server_stall_events_total` are both derived from one rule: + * LoadManager::evaluateStall(). The registry callbacks that export them only + * read atomics, so the rule is the only place a defect can hide -- an + * off-by-one on the threshold, or an episode counted once per tick instead of + * once per stall, would silently misreport every stall. + * + * The rule is a pure `static constexpr` member, so these tests assert it + * directly: no LoadManager instance, no monitor thread, no sleeping, and no + * test-only mutator added to production code to make it reachable. + * + * Compiled only when XRPL_ENABLE_TELEMETRY is defined, because that is the + * configuration in which the test target has `src/` on its include path and can + * therefore reach . The rule itself is not + * telemetry-conditional; only this file's ability to include the header is. + */ + +#ifdef XRPL_ENABLE_TELEMETRY + +#include + +#include + +#include + +using namespace xrpl; +using namespace std::chrono_literals; + +namespace { + +/** + * The threshold production uses (LoadManager::run's kReportingIntervalSeconds). + */ +constexpr auto kThreshold = 10s; + +} // namespace + +// Below the threshold nothing is reported: a brief scheduling hiccup is not a +// stall, so both outputs stay at their healthy values. Asserts the boundary +// exactly at threshold-1, not merely "some small value". +TEST(SyncStateSignals, sub_threshold_stall_reports_healthy) +{ + auto const zero = LoadManager::evaluateStall(0, 0s, kThreshold); + EXPECT_EQ(zero.seconds, 0U); + EXPECT_FALSE(zero.newEpisode); + + // One second below the threshold is still healthy. + auto const justUnder = LoadManager::evaluateStall(0, 9s, kThreshold); + EXPECT_EQ(justUnder.seconds, 0U); + EXPECT_FALSE(justUnder.newEpisode); +} + +// The threshold is inclusive: exactly 10 s is reportable and starts an episode. +// This is the boundary the log line uses, so gauge and log must agree here. +TEST(SyncStateSignals, threshold_is_inclusive_and_starts_an_episode) +{ + auto const atThreshold = LoadManager::evaluateStall(0, kThreshold, kThreshold); + EXPECT_EQ(atThreshold.seconds, 10U); + EXPECT_TRUE(atThreshold.newEpisode); +} + +// One continuous stall spanning many ticks is ONE episode. The seconds value +// tracks the growing duration while newEpisode stays false after the first +// tick -- this is what makes "one long stall" distinguishable from "repeated +// short stalls" on the dashboard. +TEST(SyncStateSignals, continuous_stall_counts_exactly_one_episode) +{ + // Tick 1: crosses the threshold. + auto const first = LoadManager::evaluateStall(0, 10s, kThreshold); + EXPECT_EQ(first.seconds, 10U); + EXPECT_TRUE(first.newEpisode); + + // Ticks 2..4: still stalled, longer each time, but the SAME episode. + auto const second = LoadManager::evaluateStall(first.seconds, 11s, kThreshold); + EXPECT_EQ(second.seconds, 11U); + EXPECT_FALSE(second.newEpisode); + + auto const third = LoadManager::evaluateStall(second.seconds, 90s, kThreshold); + EXPECT_EQ(third.seconds, 90U); + EXPECT_FALSE(third.newEpisode); + + auto const fourth = LoadManager::evaluateStall(third.seconds, 600s, kThreshold); + EXPECT_EQ(fourth.seconds, 600U); + EXPECT_FALSE(fourth.newEpisode); +} + +// Recovery clears the seconds, and a LATER stall is a NEW episode. Without the +// healthy tick in between this would be indistinguishable from the continuous +// case above, so this is the test that pins the transition semantics. +TEST(SyncStateSignals, stall_after_recovery_is_a_new_episode) +{ + // Stalled, then recovered: seconds drop back to 0 and no episode starts on + // the recovery tick itself. + auto const stalled = LoadManager::evaluateStall(0, 30s, kThreshold); + EXPECT_EQ(stalled.seconds, 30U); + EXPECT_TRUE(stalled.newEpisode); + + auto const recovered = LoadManager::evaluateStall(stalled.seconds, 0s, kThreshold); + EXPECT_EQ(recovered.seconds, 0U); + EXPECT_FALSE(recovered.newEpisode); + + // Stalling again after recovery: a second, distinct episode. + auto const again = LoadManager::evaluateStall(recovered.seconds, 15s, kThreshold); + EXPECT_EQ(again.seconds, 15U); + EXPECT_TRUE(again.newEpisode); +} + +// A stall that decays to a sub-threshold value counts as recovered, so the next +// crossing is a new episode. Edge case: the previous value was non-zero but +// below the threshold, which must be treated as healthy, not as "still stalled". +TEST(SyncStateSignals, decay_below_threshold_is_treated_as_recovered) +{ + // Contrived previous value: below threshold, so it reads as healthy even + // though it is non-zero. (Production never publishes such a value -- it + // stores 0 when healthy -- so this pins the rule, not just the caller.) + auto const crossing = LoadManager::evaluateStall(9, kThreshold, kThreshold); + EXPECT_EQ(crossing.seconds, 10U); + EXPECT_TRUE(crossing.newEpisode); +} + +// Compile-time proof that the rule is genuinely constexpr and side-effect free: +// if it ever grows state or a runtime-only dependency, these fail the build +// rather than the run. +TEST(SyncStateSignals, rule_is_evaluated_at_compile_time) +{ + static_assert(LoadManager::evaluateStall(0, 10s, 10s).newEpisode); + static_assert(LoadManager::evaluateStall(0, 10s, 10s).seconds == 10U); + static_assert(!LoadManager::evaluateStall(0, 9s, 10s).newEpisode); + static_assert(LoadManager::evaluateStall(0, 9s, 10s).seconds == 0U); + static_assert(!LoadManager::evaluateStall(10, 20s, 10s).newEpisode); + + // Keeps the test body non-empty for readers scanning for an assertion. + EXPECT_TRUE(LoadManager::evaluateStall(0, 10s, 10s).newEpisode); +} + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/xrpld/app/consensus/RCLConsensus.cpp b/src/xrpld/app/consensus/RCLConsensus.cpp index b326cd7f43a..a9625350c20 100644 --- a/src/xrpld/app/consensus/RCLConsensus.cpp +++ b/src/xrpld/app/consensus/RCLConsensus.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -513,6 +514,25 @@ RCLConsensus::Adaptor::makeAcceptSpan(Result const& result) span->setAttribute(cs::attr::proposers, static_cast(result.proposers)); span->setAttribute( cs::attr::roundTimeMs, static_cast(result.roundTime.read().count())); + + // Round duration as a native histogram, alongside the span attribute above. + // The two answer different questions and neither replaces the other: the + // attribute says how long THIS round took, readable inside the trace next + // to the proposers and disputes that explain it; the histogram gives the + // distribution over time, which is what an alert or an SLO panel needs and + // what a trace query cannot cheaply produce fleet-wide. The histogram is + // also unsampled, so it stays complete when tracing is head-sampled down. + // + // This is the one site: makeAcceptSpan() is the single function both the + // synchronous (onForceAccept) and asynchronous (onAccept) accept paths call + // once per round, so recording here cannot double-count and cannot be + // skipped. It runs once per round, never per peer, per proposal or per + // transaction. + XRPL_METRIC_HISTOGRAM_RECORD( + app_, + telemetry::metric::consensusRoundDurationMs, + "Wall-clock duration of a completed consensus round in milliseconds", + result.roundTime.read().count()); span->setAttribute(cs::attr::quorum, static_cast(app_.getValidators().quorum())); span->setAttribute(cs::attr::disputesCount, static_cast(result.disputes.size())); char const* stateStr = [&] { diff --git a/src/xrpld/app/consensus/RCLValidations.cpp b/src/xrpld/app/consensus/RCLValidations.cpp index c587a04cf01..3208b3d12e4 100644 --- a/src/xrpld/app/consensus/RCLValidations.cpp +++ b/src/xrpld/app/consensus/RCLValidations.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include #include #include @@ -175,10 +177,37 @@ handleNewValidation( // masterKey is seated only if validator is trusted or listed auto const outcome = validations.add(calcNodeID(masterKey.value_or(signingKey)), val); + // Span this validation into the trace of the ledger it validates. The + // acceptance it may trigger emits its own span from LedgerMaster, possibly + // on another thread; both derive the trace id from the same ledger hash, so + // the two land in one trace. + // + // Started before the outcome is checked, so a rejected validation is + // recorded with its real status. Otherwise "validations are counting" and + // "validations are all rejected" both look like silence on a stuck node. + // + // Trusted only: untrusted validations cannot move acceptance, so this stays + // bounded by the UNL size per ledger. + namespace cs = telemetry::consensus::span; + std::optional span; + if (val->isTrusted()) + { + span.emplace(LedgerMaster::makeLedgerTraceSpan(cs::validationAccept, hash, seq)); + span->setAttribute( + cs::attr::validationStatus, cs::validationStatusValue(static_cast(outcome))); + span->setAttribute(cs::attr::fullValidation, val->isFull()); + span->setAttribute(cs::attr::acceptGated, bypassAccept == BypassAccept::Yes); + } + if (outcome == ValStatus::Current) { if (val->isTrusted()) { + // The span stays alive across checkAccept below, so its duration is + // the time this validation spent driving the acceptance decision -- + // the number that grows when a node is slow to validate. A rejected + // validation never reaches checkAccept, so its span is just the + // record that it arrived and was refused. if (bypassAccept == BypassAccept::Yes) { XRPL_ASSERT(j, "xrpl::handleNewValidation : journal is available"); diff --git a/src/xrpld/app/ledger/InboundLedger.h b/src/xrpld/app/ledger/InboundLedger.h index 0cac2719a69..9636b5571b9 100644 --- a/src/xrpld/app/ledger/InboundLedger.h +++ b/src/xrpld/app/ledger/InboundLedger.h @@ -14,11 +14,13 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -27,6 +29,7 @@ #include #include #include +#include #include #include @@ -110,18 +113,76 @@ class InboundLedger final : public TimeoutCounter, void runData(); + /** + * Mark this acquire as still alive, so the sweeper does not reclaim it. + * + * InboundLedgers::sweep() destroys any acquire whose last action is more + * than a minute old, and that destruction is what the telemetry reports as + * `outcome=abandoned`. Anything that represents real progress must therefore + * call this, or a fetch that is working normally can be deleted for looking + * idle. + * + * @note Thread-safe and lock-free: a relaxed store of the clock's tick + * count. Called from peer threads on the receive path, from the + * acquiring thread, and read by the sweeper on the timer thread, so it + * cannot be a plain member. + */ void touch() { - lastAction_ = clock_.now(); + lastAction_.store(clock_.now().time_since_epoch().count(), std::memory_order_relaxed); } + /** + * When this acquire last made progress, for the sweeper's age check. + * + * @note Thread-safe and lock-free: a relaxed load. A value one tick stale is + * acceptable against a 60-second sweep interval. + */ clock_type::time_point getLastAction() const { - return lastAction_; + return clock_type::time_point{ + clock_type::duration{lastAction_.load(std::memory_order_relaxed)}}; } + /** + * Outstanding missing SHAMap nodes in one of this acquire's two trees. + * + * Refreshed by trigger() after each getMissingNodes() sweep, which already + * computes the count as a byproduct of its walk — this accessor adds no + * traversal of its own and does not take the acquire lock. + * + * Read by the telemetry observable-gauge callback (~10 s cadence). A count + * that stays flat and non-zero across ticks means the acquire will never + * finish; a shrinking count means it is slow but alive. + * + * @param type Which tree to report: SHAMapType::TRANSACTION selects the + * transaction tree, every other value selects the account-state tree. + * @return Node count from the most recent sweep of that tree; 0 before the + * first sweep and after the tree completes. + * + * @note Thread-safe and lock-free: a relaxed atomic load. The reader + * tolerates a value one sweep out of date. + */ + [[nodiscard]] int + getMissingNodeCount(SHAMapType type) const noexcept; + + /** + * Number of peer packets stashed in receivedData_ awaiting processing. + * + * A deep stash means node data is arriving faster than runData() can apply + * it, which is a processing bottleneck rather than a peer-supply one. + * + * @return Current stash depth; 0 when nothing is pending. + * + * @note Thread-safe and lock-free: a relaxed atomic load of a counter + * mirrored on every push/drain, so it never blocks the receive path + * nor waits on receivedDataLock_. + */ + [[nodiscard]] std::size_t + getReceivedDataDepth() const noexcept; + private: enum class TriggerReason { Added, Reply, Timeout }; @@ -188,8 +249,158 @@ class InboundLedger final : public TimeoutCounter, std::vector neededStateHashes(int max, SHAMapSyncFilter const* filter) const; + /** + * Re-publish the missing-node counts from the completion flags. + * + * Called under mtx_ wherever a tree flips to complete. A tree that needs no + * more nodes must publish 0: otherwise the last sweep's count lingers and a + * finished acquire keeps reporting a flat non-zero, which is exactly the + * "permanently stuck" reading the gauge exists to detect. Idempotent, so it + * is safe to call from every flip site. + */ + void + refreshMissingNodeCounts() noexcept; + + /** + * Zero the missing-node counts because this acquire has finished. + * + * Unconditional, so it also clears after a timeout or failure, where the + * have-tree flags are never set and the flag-guarded refresh would leave a + * stale non-zero count visible to the gauge. + */ + void + clearMissingNodeCounts() noexcept; + + /** + * Fold one processed batch into the acquire totals and emit its telemetry. + * + * Both processData() branches (header batch and node batch) finished with + * the same three steps, so they share this one helper: mark progress, add to + * stats_, and emit the per-outcome add-node counters. + * + * @param san Outcome tally for the batch just processed. + * @return Number of good nodes in the batch, which is processData()'s + * "useful data from this peer" return value. + * + * @note Called once per received packet, after the per-node loop inside + * receiveNode() has completed. The tallies are already aggregated, so + * the counters are emitted once per batch and never per node. + * @note Call with mtx_ held, as both call sites already do. + */ + int + recordBatchOutcome(SHAMapAddNode const& san); + + /** + * End the acquire span exactly once, stamping the outcome it reached. + * + * An acquire can leave through four exits: done(), the local-store + * shortcut in init(), the "can never be acquired" exit in init(), and the + * destructor when the sweeper drops a fetch that never finished. Every one + * of them calls this, so the span always carries an `outcome` and its + * duration always ends at the real exit instead of stretching to whenever + * the object happened to be destroyed. Without that, the one case worth + * detecting -- an acquire that never completes -- was the one case with no + * span data. + * + * The outcome is derived from the acquire's own flags rather than passed + * in, so no call site can label an exit wrongly: + * failed_ -> "failed", else complete_ -> "complete", else "abandoned". + * "abandoned" therefore means exactly "destroyed with no result". + * + * Idempotent: the span handle is cleared here, so the second and later + * calls do nothing. A no-op when telemetry is disabled (the guard is + * inactive) or when the span was never created. + * + * @param peerCount Peers still reachable for this fetch, or nullopt when + * the caller cannot safely look them up. The destructor passes + * nullopt because it can run while InboundLedgers holds its + * collection lock, and the lookup would take the Overlay lock + * underneath it. + * + * @note noexcept, and every fallible step is wrapped, because the + * destructor calls this: an escaping exception there would + * terminate the process during unwinding. + * @note Called once per acquire exit, never on a per-SHAMap-node path. + */ + void + finalizeAcquireSpan(std::optional peerCount) noexcept; + + /** + * Open the span for the fetch phase now in progress and close any phase + * whose data has arrived. + * + * One acquire is really three sequential fetches -- the header, then the + * account-state tree, then the transaction tree -- and on a fresh sync the + * state tree dominates. The parent span is flat, so it cannot say which of + * the three a stuck acquire is stuck in. These child spans can. + * + * Written as an idempotent state sync over the `have*_` flags rather than + * as open/close calls scattered through the fetch code: the flags are the + * real phase boundary, so deriving the span state from them cannot drift + * out of step with the fetch, and the function is safe to call from any + * progress point however often. + * + * The tree phases never open before the header arrives, because the header + * is what names their root hashes -- until then there is nothing to fetch. + * + * @note noexcept and allocation-free on the disabled path: with telemetry + * off the parent span is inactive, so no child is ever created. + * @note Call with mtx_ held, as every call site does. Called at phase + * boundaries and per received packet, never per SHAMap node. + */ + void + syncPhaseSpans() noexcept; + + /** + * Start one phase child span, parented to the acquire span. + * + * Parented through the acquire span's captured context rather than the + * thread's ambient context, so a phase opened on a JtLedgerData worker + * still lands under the right acquire. + * + * @param span The phase span handle to fill; untouched if already open. + * @param name Full span name from LedgerSpanNames.h. + * + * @note A no-op when the acquire span is absent or inactive, which is what + * makes the whole phase-span feature cost nothing when telemetry is + * disabled. + */ + void + beginPhaseSpan(std::optional& span, std::string_view name) noexcept; + + /** + * End one phase child span exactly once, stamping its outcome. + * + * Idempotent by the same rule as finalizeAcquireSpan(): the handle is + * cleared, so a later call finds nothing and cannot overwrite the outcome + * the real phase end recorded. + * + * @param span The phase span handle to end. + * @param complete Whether this phase's data was fully assembled. + * @param missingNodes Nodes still outstanding in this phase's tree, or + * nullopt for the header phase, which has no tree. + * + * @note noexcept, and the attribute writes are wrapped, because this is + * reached from ~InboundLedger via finalizeAcquireSpan(). + */ + void + endPhaseSpan( + std::optional& span, + bool complete, + std::optional missingNodes) noexcept; + clock_type& clock_; - clock_type::time_point lastAction_; + + /** + * Tick count of the last action, as the clock's duration rep. + * + * Stored as the raw rep rather than a `time_point` so it can be atomic: the + * receive path writes it from peer threads while InboundLedgers::sweep() + * reads it from the timer thread. Relaxed on both sides -- the sweeper + * compares against a 60-second threshold, so a value one tick out of date + * cannot change its decision. + */ + std::atomic lastAction_; std::shared_ptr ledger_; bool haveHeader_{false}; @@ -222,15 +433,66 @@ class InboundLedger final : public TimeoutCounter, std::unique_ptr peerSet_; /** - * Spans the acquire lifecycle: started in init(), finalized in done() - * with the outcome (complete/failed), timeout count, and peer count. - * Gives operators visibility into back-fill / fork-recovery cost, which - * previously emitted no span or metric. + * Outstanding missing nodes in the account-state tree, as counted by the + * last getMissingNodes() sweep in trigger(). Relaxed atomic: written by the + * acquiring thread, read by the telemetry gauge callback, and a value one + * sweep stale is acceptable for a ~10 s gauge. + */ + std::atomic missingStateNodes_{0}; + + /** + * Outstanding missing nodes in the transaction tree. Same ownership and + * staleness contract as missingStateNodes_. + */ + std::atomic missingTxNodes_{0}; + + /** + * Mirror of receivedData_.size(), maintained under receivedDataLock_ on + * every push and drain. Exists so the telemetry gauge callback can read the + * depth without contending for that lock on the node-receive path. + */ + std::atomic receivedDataDepth_{0}; + + /** + * Spans the acquire lifecycle: started in init(), ended by + * finalizeAcquireSpan() with the outcome (complete/failed/abandoned), + * timeout count, and peer count. Gives operators visibility into + * back-fill / fork-recovery cost, which previously emitted no span or + * metric. + * Held for the object's whole lifetime so the destructor can still stamp + * an outcome on a fetch that never reached done(). * Thread-free: emplaced by the acquiring thread, reset on a JtLedgerData - * worker. A SpanGuard owns no thread-local Scope, so it can be destroyed - * on the worker without corrupting the origin thread's context stack. + * worker or in the destructor. A SpanGuard owns no thread-local Scope, so + * it can be destroyed on any thread without corrupting the origin + * thread's context stack. */ std::optional acquireSpan_; + + /** + * Child spans for the three fetch phases of this acquire, each a child of + * acquireSpan_ and each open only while its phase is in progress. + * + * Present so a stuck acquire names the phase it is stuck in: on a fresh + * sync the account-state tree is nearly all of the work, and the flat + * parent span cannot separate it from the small transaction tree or from + * the header wait that gates both. + * + * Same ownership contract as acquireSpan_: written under mtx_ or from the + * destructor, and thread-free, so a phase may be ended on whichever worker + * receives its last node. + */ + std::optional headerSpan_; + std::optional asTreeSpan_; + std::optional txTreeSpan_; + + /** + * True once the acquire has exhausted its timeout budget, so each phase + * still open at that point reports `timeout` rather than `abandoned`. + * Distinct from `failed_`, which the same path also sets: `failed_` is how + * the timer loop stops, while this is what says the cause was peers not + * supplying data rather than data that would not apply. + */ + bool timedOut_{false}; }; } // namespace xrpl diff --git a/src/xrpld/app/ledger/InboundLedgers.h b/src/xrpld/app/ledger/InboundLedgers.h index e288201c660..6a59bc2c215 100644 --- a/src/xrpld/app/ledger/InboundLedgers.h +++ b/src/xrpld/app/ledger/InboundLedgers.h @@ -91,6 +91,56 @@ class InboundLedgers virtual std::size_t cacheSize() = 0; + + /** + * Aggregate acquire-progress snapshot across every in-flight acquire. + * + * Bounded, pre-aggregated telemetry: one value per field regardless of how + * many acquires are in flight, so the derived metric cannot grow a series + * per ledger. Per-ledger detail stays available on the `ledger.acquire` + * span, which is where unbounded identity belongs. + */ + struct AcquireProgress + { + /** + * Largest outstanding account-state node count of any in-flight + * acquire. The max, not the sum, so one stuck acquire stays visible + * instead of being averaged away by healthy ones. + */ + int maxMissingStateNodes{0}; + + /** + * Largest outstanding transaction-tree node count of any in-flight + * acquire. + */ + int maxMissingTxNodes{0}; + + /** + * Total unprocessed peer packets stashed across all in-flight acquires. + * Summed because it measures one shared processing backlog. + */ + std::size_t receivedDataDepth{0}; + + /** + * Number of acquires currently in flight, so the three values above can + * be read in context (all zero with zero acquires is idle, not healthy). + */ + std::size_t inFlight{0}; + }; + + /** + * Collect the aggregate acquire-progress snapshot. + * + * Intended for the telemetry observable gauge, polled about every 10 s. + * Takes the collection lock only long enough to copy the handles, then reads + * each acquire's relaxed atomics without holding it, so it never blocks the + * node-receive path. + * + * @return Aggregated progress; an all-zero value with `inFlight == 0` when + * nothing is being acquired. + */ + [[nodiscard]] virtual AcquireProgress + acquireProgress() = 0; }; std::unique_ptr diff --git a/src/xrpld/app/ledger/LedgerMaster.h b/src/xrpld/app/ledger/LedgerMaster.h index 32163fd57b9..a0a325a140f 100644 --- a/src/xrpld/app/ledger/LedgerMaster.h +++ b/src/xrpld/app/ledger/LedgerMaster.h @@ -24,6 +24,7 @@ #include #include #include +#include #include @@ -35,6 +36,7 @@ #include #include #include +#include #include #include @@ -279,6 +281,162 @@ class LedgerMaster : public AbstractFetchPackContainer std::optional txnIdFromIndex(uint32_t ledgerSeq, uint32_t txnIndex); + /** + * Ledgers fully validated but not yet published to clients. + * + * The publish pipeline (doAdvance) lags validation by design, but the + * gap must drain. A gap that stays positive and grows means validation + * is healthy while publishing is not, which is a different fault from + * anything the acquire or quorum signals can show. + * + * @return Non-negative publish lag in ledgers; 0 when caught up, and 0 + * before the first ledger is validated. + * + * @note Safe to call from any thread, including a telemetry + * observable-gauge callback: two relaxed atomic loads, no lock. The two + * sequences are read independently, so a reading taken while + * setValidLedger() and setPubLedger() are both running may be off by + * one ledger for one poll. That is immaterial for a lag trend and is + * the price of not taking the LedgerMaster mutex on the poll thread. + */ + [[nodiscard]] std::int64_t + getPublishLag() const noexcept + { + auto const valid = + static_cast(validLedgerSeq_.load(std::memory_order_relaxed)); + auto const published = + static_cast(pubLedgerSeq_.load(std::memory_order_relaxed)); + auto const lag = valid - published; + return lag > 0 ? lag : 0; + } + + /** + * Trusted validations counted at the most recent pre-accept gate. + * + * checkAccept() refuses to declare a ledger validated until this tally + * reaches getQuorumTarget(). Exposing the tally is what separates + * "validations are accumulating, just slowly" from "validations arrive + * but never reach quorum". + * + * @return Trusted validation count at the last gate evaluation; 0 before + * the first one. + * + * @note Safe to call from any thread: one relaxed atomic load, no lock. + */ + [[nodiscard]] std::int64_t + getTrustedValidationTally() const noexcept + { + return lastTrustedTally_.load(std::memory_order_relaxed); + } + + /** + * Validations required at the most recent pre-accept gate. + * + * @return Quorum threshold at the last gate evaluation; 0 before the + * first one. Reports std::numeric_limits::max() when the + * validator list has switched quorum off entirely (see + * getNeededValidations()), so the value never wraps negative and a + * tally-versus-target panel cannot invert on a node that can never + * validate. + * + * @note Safe to call from any thread: one relaxed atomic load, no lock. + */ + [[nodiscard]] std::int64_t + getQuorumTarget() const noexcept + { + return lastQuorumTarget_.load(std::memory_order_relaxed); + } + + /** + * Time from construction until the first ledger passed the pre-accept + * gate, in microseconds. + * + * A one-shot measurement, like the time-to-first-FULL signal: it is + * written once and never changes, so it has no trend. Exactly two + * readings are meaningful — a duration, meaning the node reached its + * first fully-validated ledger and this is how long that took, or 0, + * meaning it never has. + * + * @return Microseconds to the first fully-validated ledger; 0 until then. + * + * @note Safe to call from any thread: one relaxed atomic load, no lock. + */ + [[nodiscard]] std::int64_t + getTimeToFirstValidatedUs() const noexcept + { + return timeToFirstValidatedUs_.load(std::memory_order_relaxed); + } + + /** + * Start a span that joins the per-ledger trace keyed on the ledger hash. + * + * One ledger's spans are produced on threads that know nothing about each + * other: `ledger.acquire` on a JtLedgerData worker, `ledger.validate` from + * whichever thread calls checkAccept (a peer thread via + * handleNewValidation, the JtLedgerData "AcqDone" job, or the consensus + * thread via switchLCL), and `ledger.store` from a fourth. No ambient + * context reaches across those boundaries, so before this each ledger's + * spans were separate one-span traces and a slow ledger could not be read + * as one unit. + * + * They are joined without propagating anything: `SpanGuard::hashSpan()` + * derives the trace id from the first 16 bytes of a hash, so every span + * that passes the SAME ledger hash lands in the SAME trace, and spans for + * different ledgers stay in different traces. Every one of these sites + * already holds the ledger hash, so nothing new has to be plumbed through. + * This is the pattern the apply pipeline already uses to join preflight, + * preclaim and the transactor on the transaction id + * (`libxrpl/tx/applySteps.cpp`). + * + * The span is a true trace root (deterministic trace id, no parent), so + * the spans of one ledger are siblings under one trace rather than a + * parent/child chain — which is the honest shape, since none of them + * causes another directly and their order varies with the sync path taken. + * + * Which stage a span is stays encoded in the span NAME rather than in an + * extra attribute: the collector already exposes the span name as the + * `span_name` label on the derived metrics, so a separate phase attribute + * would be a second copy of the same fact that could disagree with it. + * + * @param name Full span name (e.g. "ledger.validate"). hashSpan + * takes one complete name, not a prefix/suffix pair, so + * pass the joined constant. + * @param ledgerHash The ledger's own 32-byte hash: the join key. Its + * first 16 bytes become the trace id, and the whole + * hash is recorded as the `ledger_hash` attribute. + * @param seq Ledger sequence, recorded as `ledger_seq` so a trace + * can be found by ledger number. + * @return An active guard, or a null (no-op) guard when telemetry is off, + * built without telemetry, or ledger tracing is not enabled. + * + * Example — join the acceptance decision to the rest of the ledger's trace: + * @code + * auto span = makeLedgerTraceSpan( + * ledger_span::validate, ledger->header().hash, ledger->header().seq); + * span.setAttribute(ledger_span::attr::validations, tvc); + * @endcode + * + * Example — edge case: with tracing disabled the guard is null and every + * method on it is a no-op, so no call site needs its own check: + * @code + * auto span = makeLedgerTraceSpan(ledger_span::store, hash, seq); + * // span is false; setAttribute() below does nothing. + * @endcode + * + * @note Called once per ledger per stage, never per SHAMap node or per + * transaction. Costs one span plus two attributes when tracing is on, and + * one predicted branch when it is off. + * @note Thread-safe: builds a fresh guard from its arguments and the global + * Telemetry instance, touching no LedgerMaster state, so it may be called + * from any thread and while mutex_ is held. + * @note Limitation: the join holds only for spans keyed on the SAME ledger + * hash. A stage that has only a sequence number (a by-seq lookup before the + * hash is known) cannot join the trace, and a genuinely different ledger + * always gets a different trace. + */ + [[nodiscard]] static telemetry::SpanGuard + makeLedgerTraceSpan(std::string_view name, uint256 const& ledgerHash, std::uint32_t seq); + private: void setValidLedger(std::shared_ptr const& l); @@ -391,6 +549,54 @@ class LedgerMaster : public AbstractFetchPackContainer // Time that the previous upgrade warning was issued. TimeKeeper::time_point upgradeWarningPrevTime_; + // --- Sync diagnostics: the pre-accept quorum gate ----------------------- + // + // checkAccept() computes a trusted-validation tally and the quorum it must + // reach, then returns early when the tally is short. Both values used to + // exist only for the duration of that call and a trace log line, so a node + // with peers and validators that still refuses to validate looked + // identical to an idle one. These snapshots keep the last gate evaluation + // readable by the metrics poll thread. + // + // checkAccept (per validated ledger, holds mutex_) + // | computes tvc, minVal + // +--> lastTrustedTally_ / lastQuorumTarget_ (relaxed stores) + // | + // +--> gate passes --> timeToFirstValidatedUs_ (once per process) + // + // OTel reader thread (~10 s tick) + // +--> getTrustedValidationTally() / getQuorumTarget() + // getTimeToFirstValidatedUs() / getPublishLag() (relaxed loads) + // + // Deliberately atomics rather than values guarded by mutex_: the emit path + // holds mutex_ while the metric macro takes an OTel-internal lock, so a + // reader that took mutex_ from inside an OTel callback would invert that + // order. Lock-free reads make the inversion impossible. + + /** + * Trusted validations counted at the last checkAccept gate; 0 before the + * first gate evaluation. + */ + std::atomic lastTrustedTally_{0}; + + /** + * Validations required at the last checkAccept gate; 0 before the first + * gate evaluation, and int64 max when quorum is switched off entirely. + */ + std::atomic lastQuorumTarget_{0}; + + /** + * Microseconds from construction to the first ledger that passed the + * pre-accept gate. Written exactly once; 0 means it has not happened. + */ + std::atomic timeToFirstValidatedUs_{0}; + + /** + * Construction time, the epoch the first-validated milestone measures + * from. Steady, so it is unaffected by wall-clock or NTP adjustments. + */ + std::chrono::steady_clock::time_point const startTime_{std::chrono::steady_clock::now()}; + private: struct Stats { diff --git a/src/xrpld/app/ledger/LedgerReplayTask.h b/src/xrpld/app/ledger/LedgerReplayTask.h index d908a36fc07..96e9dea0f6d 100644 --- a/src/xrpld/app/ledger/LedgerReplayTask.h +++ b/src/xrpld/app/ledger/LedgerReplayTask.h @@ -152,6 +152,26 @@ class LedgerReplayTask final : public TimeoutCounter, void tryAdvance(ScopedLockType& sl); + /** + * Record this task reaching a terminal state, for telemetry. + * + * Replay degrading to plain ledger acquisition is otherwise silent: every + * terminal path here only sets `complete_`/`failed_` and writes a log + * line, so a replay that never succeeds looks identical to one that was + * never attempted. Each terminal site calls this exactly once. + * + * @param outcome Terminal state, used as the metric's `outcome` label: + * `success`, `timeout`, `build_failed` or + * `parameter_failed`. A fixed set of four literals, so the + * label cardinality is bounded. + * + * @note No-op when telemetry is compiled out or disabled at runtime -- the + * counter macro carries its own guard, so callers need no `#ifdef`. + * @note Called from terminal paths only, never per delta or per ledger. + */ + void + recordOutcome(char const* outcome) const; + InboundLedgers& inboundLedgers_; LedgerReplayer& replayer_; TaskParameter parameter_; diff --git a/src/xrpld/app/ledger/detail/InboundLedger.cpp b/src/xrpld/app/ledger/detail/InboundLedger.cpp index 7444d00b160..1819bf89baf 100644 --- a/src/xrpld/app/ledger/detail/InboundLedger.cpp +++ b/src/xrpld/app/ledger/detail/InboundLedger.cpp @@ -11,6 +11,8 @@ #include #include #include +#include +#include #include #include @@ -30,22 +32,24 @@ #include // IWYU pragma: keep #include #include +#include #include #include #include -#include #include #include #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -103,16 +107,28 @@ InboundLedger::init(ScopedLockType& collectionLock) collectionLock.unlock(); // Span the acquire lifecycle so back-fill / fork-recovery cost is - // observable. Finalized in done() with the outcome and timeout count. + // observable. Finalized by finalizeAcquireSpan() on whichever exit this + // acquire takes, including the destructor. { using namespace telemetry; // acquireSpan_ is emplaced here but reset() on a JtLedgerData worker // thread. A SpanGuard is thread-free (owns no thread-local Scope), so it // can be created here and destroyed on the worker with no scope to strip. + // hashSpan, not span: the trace id is derived from hash_[0:16], so this + // acquire lands in the same trace as the ledger.validate, + // ledger.store and consensus.validation.accept spans for the same + // ledger, which run on other threads. One trace then shows whether a + // slow ledger was slow to fetch, to be accepted, or to be stored. The + // phase children below inherit this trace id automatically. acquireSpan_.emplace( - SpanGuard::span(TraceCategory::Ledger, seg::ledger, ledger_span::op::acquire)); + SpanGuard::hashSpan( + TraceCategory::Ledger, ledger_span::acquireFull, hash_.data(), hash_.kBytes)); if (*acquireSpan_) { + // The hash is the one identity known at every acquire's start (a + // by-hash fetch begins with seq_ == 0), so it is what makes a + // still-running or abandoned fetch identifiable in a trace search. + acquireSpan_->setAttribute(ledger_span::attr::ledgerHash, to_string(hash_).c_str()); acquireSpan_->setAttribute(ledger_span::attr::ledgerSeq, static_cast(seq_)); // Map the acquire reason to its span-attribute value. A switch // keeps the mapping flat and exhaustive over the Reason enum. @@ -134,10 +150,33 @@ InboundLedger::init(ScopedLockType& collectionLock) tryDB(app_.getNodeFamily().db()); if (failed_) + { + // tryDB proved the ledger can never be acquired. This exit never + // reaches done(), so finalize here or the span would carry no outcome. + finalizeAcquireSpan(getPeerCount()); return; + } + + // Whether the local node store already held the whole ledger. Emitted once + // per new acquire (init() runs exactly once), never per node, so the cost is + // a single labelled counter Add. This is what separates disk-bound sync + // ("everything was local, we are just slow to read it") from peer-bound sync + // ("nothing was local, every node must come over the wire"). + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + telemetry::metric::syncAcquireSourceTotal, + "Ledger acquires by where the data came from", + {{telemetry::label::source, + std::string( + complete_ ? telemetry::lval::acquire_source::local + : telemetry::lval::acquire_source::network)}}); if (!complete_) { + // Open the span for whichever phase the local lookup left outstanding, + // so the phase timeline starts at the same instant the network fetch + // does rather than at the first reply. + syncPhaseSpans(); addPeers(); queueJob(sl); return; @@ -145,6 +184,12 @@ InboundLedger::init(ScopedLockType& collectionLock) JLOG(journal_.debug()) << "Acquiring ledger we already have in " << " local store. " << hash_; + + // The local store already had everything, so the fetch is over here and + // never goes through done(). Finalize now: the span's duration then covers + // only the store read, not the storeLedger/checkAccept work below. + finalizeAcquireSpan(getPeerCount()); + XRPL_ASSERT( ledger_->header().seq < kXrpLedgerEarliestFees || ledger_->read(keylet::feeSettings()), "xrpl::InboundLedger::init : valid ledger fees"); @@ -168,6 +213,42 @@ InboundLedger::init(ScopedLockType& collectionLock) app_.getLedgerMaster().checkAccept(ledger_); } +int +InboundLedger::getMissingNodeCount(SHAMapType type) const noexcept +{ + return (type == SHAMapType::TRANSACTION ? missingTxNodes_ : missingStateNodes_) + .load(std::memory_order_relaxed); +} + +std::size_t +InboundLedger::getReceivedDataDepth() const noexcept +{ + return receivedDataDepth_.load(std::memory_order_relaxed); +} + +void +InboundLedger::refreshMissingNodeCounts() noexcept +{ + if (haveState_) + missingStateNodes_.store(0, std::memory_order_relaxed); + if (haveTransactions_) + missingTxNodes_.store(0, std::memory_order_relaxed); +} + +void +InboundLedger::clearMissingNodeCounts() noexcept +{ + // Unconditional, unlike refreshMissingNodeCounts(): this runs when the + // acquire is over, however it ended. A timed-out or failed acquire never + // sets the have-tree flags, so the flag-guarded refresh above would leave + // its last sweep count latched. The gauge maxes over every acquire still in + // the collection, and eviction waits on a one-minute grace plus the sweep + // interval, so a latched count would report a finished node as stuck for + // minutes -- inverting the one signal that separates stuck from slow. + missingStateNodes_.store(0, std::memory_order_relaxed); + missingTxNodes_.store(0, std::memory_order_relaxed); +} + std::size_t InboundLedger::getPeerCount() const { @@ -227,12 +308,27 @@ InboundLedger::~InboundLedger() // the whole acquisition has to start over. That is the expensive case, // so it is counted apart from a cheap abort that had nothing yet. app_.getAcquireStats().recordAbort(haveHeader_ || haveState_ || haveTransactions_); + + // Activate the still-open span so this abort line carries its trace_id, + // which is what links the abandoned span to the reason it was dropped. + // Non-owning and popped at the end of this block, before the span ends. + auto acquireActivation = telemetry::activateIfLive(acquireSpan_); + JLOG(journal_.debug()) << "Acquire " << hash_ << " abort " << ((timeouts_ == 0) ? std::string() : (std::string("timeouts:") + std::to_string(timeouts_) + " ")) << stats_.get(); } + + // Last exit. A fetch dropped here (swept for making no progress, or torn + // down at shutdown) reached no result, so this is what stamps + // outcome=abandoned instead of exporting a span with no outcome at all. + // Already-finalized acquires are untouched -- the helper is idempotent. + // The peer count is not read here: this destructor can run under the + // InboundLedgers collection lock, and getPeerCount() would take the Overlay + // lock underneath it. + finalizeAcquireSpan(std::nullopt); } static std::vector @@ -374,6 +470,10 @@ InboundLedger::tryDB(node_store::Database& srcDB) } } + // A tree satisfied from the local store never ran a sweep, so publish its + // zero here rather than leaving the gauge on a stale count. + refreshMissingNodeCounts(); + if (haveTransactions_ && haveState_) { JLOG(journal_.debug()) << "Had everything locally"; @@ -410,6 +510,10 @@ InboundLedger::onTimer(bool wasProgress, ScopedLockType&) { JLOG(journal_.warn()) << timeouts_ << " timeouts for ledger " << hash_; } + // Record WHY before done() finalizes the spans. failed_ alone reads as + // "the data was bad"; this path is "no peer supplied it in time", and + // any phase still open is stamped `timeout` because of this flag. + timedOut_ = true; failed_ = true; done(); return; @@ -424,6 +528,16 @@ InboundLedger::onTimer(bool wasProgress, ScopedLockType&) std::size_t const pc = getPeerCount(); JLOG(journal_.debug()) << "No progress(" << pc << ") for ledger " << hash_; + // A timeout with no node received since the previous one means this + // acquire is stalled. Fires on the acquire timer (once every 3 s at + // most), not on any per-node path, so one counter Add here is free. + // A climbing rate here alongside a flat missing-node count is the + // signature of a sync that will never complete. + XRPL_METRIC_COUNTER_INC( + app_, + telemetry::metric::syncAcquireNoProgressTotal, + "Ledger-acquire timeouts where no new node arrived"); + // addPeers triggers if the reason is not HISTORY // So if the reason IS HISTORY, need to trigger after we add // otherwise, we need to trigger before we add @@ -459,6 +573,196 @@ InboundLedger::pmDowncast() return shared_from_this(); } +void +InboundLedger::beginPhaseSpan( + std::optional& span, + std::string_view name) noexcept +{ + // Already open, or there is no parent to hang it on. The second case is + // also the disabled path: with telemetry off acquireSpan_ is inactive, so + // no phase span is ever created and this whole feature costs one branch. + if (span || !acquireSpan_ || !*acquireSpan_) + return; + + // Parented through the acquire span's OWN captured context, not the + // thread's ambient context: a phase can open on a JtLedgerData worker where + // the ambient context is unrelated, and childSpan(name) would then attach + // it to whatever happened to be active there. + auto child = telemetry::SpanGuard::childSpan(name, acquireSpan_->spanContext()); + if (!child) + return; + // The identity every phase shares with its parent, so a phase span found + // on its own in a search still says which ledger it belongs to. + child.setAttribute(telemetry::ledger_span::attr::ledgerHash, to_string(hash_).c_str()); + if (seq_ != 0) + { + child.setAttribute( + telemetry::ledger_span::attr::ledgerSeq, static_cast(seq_)); + } + span.emplace(std::move(child)); +} + +void +InboundLedger::endPhaseSpan( + std::optional& span, + bool complete, + std::optional missingNodes) noexcept +{ + // Idempotent: the handle is cleared below, so a second call finds nothing + // and cannot overwrite the outcome the real phase end recorded. + if (!span) + return; + + // Wrapped because ~InboundLedger reaches this through + // finalizeAcquireSpan(): an exception escaping a destructor during + // unwinding would terminate the process. + try + { + if (*span) + { + using namespace telemetry; + // Same shared rule for every phase, so no phase can mislabel its + // own end and a phase still open when the acquire dies reports + // `timeout` (budget gone) or `abandoned` (swept) rather than + // nothing at all. + span->setAttribute( + ledger_span::attr::outcome, + ledger_span::phaseOutcome(failed_, complete, timedOut_)); + span->setAttribute(ledger_span::attr::timedOut, timedOut_); + // The count the phase's last getMissingNodes() sweep already + // produced -- read, never recomputed, so no second tree walk. A + // non-zero value on a timed-out phase is the "peers are not serving + // this tree" signature. + if (missingNodes) + { + span->setAttribute( + ledger_span::attr::missingNodes, static_cast(*missingNodes)); + } + } + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Telemetry must never break an acquire. A span missing one attribute + // is still worth exporting, so fall through and end it below. + } + + // End the span outside the try so it happens on every path, and + // unconditionally so it never leaks even when it was inactive. + span.reset(); +} + +void +InboundLedger::syncPhaseSpans() noexcept +{ + // Nothing to parent to: telemetry off, ledger category disabled, or the + // acquire already finalized. One branch on the disabled path. + if (!acquireSpan_ || !*acquireSpan_) + return; + + using namespace telemetry; + + // The header gates both trees, so it is the only phase that can be open + // before there is anything else to fetch. + if (!haveHeader_) + { + beginPhaseSpan(headerSpan_, ledger_span::acquireHeader); + return; + } + // The header arrived. Close its span with no missing-node count -- a header + // is a single object, not a tree. + endPhaseSpan(headerSpan_, /*complete=*/true, /*missingNodes=*/std::nullopt); + + // Both trees are fetched concurrently once their root hashes are known, so + // both spans can be open at once. Each closes when its own tree completes, + // which is what lets a trace show the state tree still running long after + // the transaction tree finished -- the normal shape of a fresh sync. + if (haveState_) + { + endPhaseSpan(asTreeSpan_, /*complete=*/true, getMissingNodeCount(SHAMapType::STATE)); + } + else + { + beginPhaseSpan(asTreeSpan_, ledger_span::acquireAsTree); + } + + if (haveTransactions_) + { + endPhaseSpan(txTreeSpan_, /*complete=*/true, getMissingNodeCount(SHAMapType::TRANSACTION)); + } + else + { + beginPhaseSpan(txTreeSpan_, ledger_span::acquireTxTree); + } +} + +void +InboundLedger::finalizeAcquireSpan(std::optional peerCount) noexcept +{ + // Close any phase still open BEFORE the parent ends, so no child span + // outlives its parent. A phase open at this point is one that never + // finished: it takes `timeout` when the retry budget ran out and + // `abandoned` when the acquire was swept, which is exactly the case these + // spans exist to show. Each carries the last missing-node count its own + // sweep produced, so a stuck phase reports how much it was still waiting + // for. + // Ahead of the acquire-span check below because each is independently + // idempotent: on a second call they are already empty, and when telemetry + // is off they were never created. + endPhaseSpan(headerSpan_, haveHeader_, std::nullopt); + endPhaseSpan(asTreeSpan_, haveState_, getMissingNodeCount(SHAMapType::STATE)); + endPhaseSpan(txTreeSpan_, haveTransactions_, getMissingNodeCount(SHAMapType::TRANSACTION)); + + // Idempotent: the handle is cleared below, so a later exit finds nothing to + // finalize and cannot overwrite the outcome the real exit recorded. + if (!acquireSpan_) + return; + + // The attribute writes are wrapped because the destructor is one of the + // callers: an exception escaping there during unwinding would terminate the + // process. Each setAttribute is itself noexcept today; the try is the + // structural guarantee that stays correct if that ever changes. + try + { + if (*acquireSpan_) + { + using namespace telemetry; + // Derived from this acquire's own flags by the shared rule, so no + // call site can mislabel an exit and every exit gets an outcome. + // Neither flag set means the fetch was dropped while in flight. + acquireSpan_->setAttribute( + ledger_span::attr::outcome, ledger_span::acquireOutcome(failed_, complete_)); + acquireSpan_->setAttribute( + ledger_span::attr::timeouts, static_cast(timeouts_)); + if (peerCount) + { + acquireSpan_->setAttribute( + ledger_span::attr::peerCount, static_cast(*peerCount)); + } + // A by-hash acquire starts with seq_ == 0 and learns the sequence + // only when the header arrives, so re-stamp it here. OTel + // attributes are last-write-wins, so this replaces the placeholder + // 0 set at init() and lets the trace be found by ledger number. + if (seq_ != 0) + { + acquireSpan_->setAttribute( + ledger_span::attr::ledgerSeq, static_cast(seq_)); + } + } + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Telemetry must never break an acquire, and this also runs from the + // destructor. A span missing one attribute is still worth exporting, so + // fall through and end it below. + } + + // End the span, outside the try so it happens on every path. Unconditional + // so the span never leaks even when it was inactive, and so this helper is + // exactly-once: a later exit sees an empty handle and returns above. + // ~SpanGuard is implicitly noexcept, so this cannot throw out of here. + acquireSpan_.reset(); +} + void InboundLedger::recordCompletionOnce() { @@ -487,6 +791,10 @@ InboundLedger::done() signaled_ = true; touch(); + // The acquire is over on every path through here, so the missing-node + // counts must stop being reported. See clearMissingNodeCounts(). + clearMissingNodeCounts(); + // done() is the funnel every peer-driven outcome passes through, but not // every outcome: init() can satisfy an acquisition from the local store // and return without reaching here. Both call the same idempotent helper @@ -494,26 +802,12 @@ InboundLedger::done() // excluded; the give-up path counts those itself. recordCompletionOnce(); - // Finalize the acquire span with the outcome, timeout count, and peer - // count. Keep it active as the ambient context across the finalize and - // the outcome log below so that log line carries the span's trace_id. - // The activation is non-owning; acquireSpan_ still owns the span. The - // activation pops at the end of this block (restoring the prior context) - // while acquireSpan_ is still alive, then reset() ends the span. + // Keep the span active as the ambient context across the outcome log so + // that line carries the span's trace_id. The activation is non-owning; + // acquireSpan_ still owns the span. It pops at the end of this block, while + // the span is still alive, and only then is the span finalized and ended. { auto acquireActivation = telemetry::activateIfLive(acquireSpan_); - if (acquireSpan_ && *acquireSpan_) - { - using namespace telemetry; - acquireSpan_->setAttribute( - ledger_span::attr::outcome, - failed_ ? std::string_view(ledger_span::val::failed) - : std::string_view(ledger_span::val::complete)); - acquireSpan_->setAttribute( - ledger_span::attr::timeouts, static_cast(timeouts_)); - acquireSpan_->setAttribute( - ledger_span::attr::peerCount, static_cast(getPeerCount())); - } JLOG(journal_.debug()) << "Acquire " << hash_ << (failed_ ? " fail " : " ") << ((timeouts_ == 0) ? std::string() @@ -522,9 +816,7 @@ InboundLedger::done() << stats_.get(); // acquireActivation pops here, before the span is ended below. } - // End the acquire span after its outcome log. Unconditional so the span - // never leaks even when it was inactive. - acquireSpan_.reset(); + finalizeAcquireSpan(getPeerCount()); XRPL_ASSERT(complete_ || failed_, "xrpl::InboundLedger::done : complete or failed"); @@ -592,6 +884,13 @@ InboundLedger::trigger(std::shared_ptr const& peer, TriggerReason reason) stream << ss.str(); } + // Open the span for whatever phase is now outstanding. Placed here, at the + // top of the one function every progress point funnels through (a reply, a + // timeout, a newly added peer), so a phase span exists for the whole time + // that phase is being requested. Idempotent, so repeated triggers within + // one phase do nothing. + syncPhaseSpans(); + if (!haveHeader_) { tryDB(app_.getNodeFamily().db()); @@ -723,6 +1022,12 @@ InboundLedger::trigger(std::shared_ptr const& peer, TriggerReason reason) auto nodes = ledger_->stateMap().getMissingNodes(kMissingNodesFind, &filter); sl.lock(); + // Publish the outstanding count for the telemetry gauge. The sweep + // above already produced it, so this is one relaxed atomic store per + // sweep and never per tree node -- getMissingNodes() walks thousands + // of nodes internally and must stay free of metric work. + missingStateNodes_.store(static_cast(nodes.size()), std::memory_order_relaxed); + // Make sure nothing happened while we released the lock if (!failed_ && !complete_ && !haveState_) { @@ -791,6 +1096,10 @@ InboundLedger::trigger(std::shared_ptr const& peer, TriggerReason reason) auto nodes = ledger_->txMap().getMissingNodes(kMissingNodesFind, &filter); + // Same contract as the state-tree store above: one atomic store per + // sweep, outside the per-node walk. + missingTxNodes_.store(static_cast(nodes.size()), std::memory_order_relaxed); + if (nodes.empty()) { if (!ledger_->txMap().isValid()) @@ -827,6 +1136,12 @@ InboundLedger::trigger(std::shared_ptr const& peer, TriggerReason reason) } } + // A tree may have completed in the blocks above without the whole acquire + // completing (the usual shape: the small transaction tree finishes long + // before the state tree). Close that phase now so its span duration is the + // real fetch time rather than stretching to the next trigger. + syncPhaseSpans(); + if (complete_ || failed_) { JLOG(journal_.debug()) << "Done:" << (complete_ ? " complete" : "") @@ -916,6 +1231,17 @@ InboundLedger::takeHeader(std::string const& data) if (ledger_->header().accountHash.isZero()) haveState_ = true; + // An empty tree is complete on arrival of the header, with no sweep to + // publish its count. + refreshMissingNodeCounts(); + + // The header phase ends exactly here, and the tree phases become openable + // for the first time -- until now their root hashes were unknown. Doing it + // here rather than at the next trigger() is what keeps the header span's + // duration equal to the real header wait, which on a fresh node is the + // first thing that can stall. + syncPhaseSpans(); + ledger_->txMap().setSynching(); ledger_->stateMap().setSynching(); @@ -1011,6 +1337,17 @@ InboundLedger::receiveNode(protocol::TMLedgerData const& packet, SHAMapAddNode& haveState_ = true; } + // The tree finished on this batch, so the last sweep's count is now + // stale. Publishing 0 here is what stops a completed acquire from + // reading as a permanently stuck one. Outside the per-node loop above. + refreshMissingNodeCounts(); + + // This is the batch that completed a tree, so close that phase's span + // here. Without it the phase would stay open until the next trigger() + // and its duration would absorb the wait for the other tree. Outside + // the per-node loop above, so it runs once per completing batch. + syncPhaseSpans(); + if (haveTransactions_ && haveState_) { complete_ = true; @@ -1119,6 +1456,17 @@ InboundLedger::gotData( return false; receivedData_.emplace_back(peer, data); + // Mirror the depth for the telemetry gauge, which must not take this lock. + receivedDataDepth_.store(receivedData_.size(), std::memory_order_relaxed); + + // A peer just answered, so this acquire is making progress even if its turn + // to apply the data has not come up yet. Without this the sweeper's one + // minute idle test measures the wait for a JtLedgerData slot rather than + // real inactivity, and deletes fetches that are still being served: on a + // fresh mainnet sync that produced 490 abandoned acquires against zero + // expired retry budgets, because only the constructor, update() and done() + // ever refreshed the timestamp. + touch(); if (receiveDispatched_) return false; @@ -1186,11 +1534,7 @@ InboundLedger::processData(std::shared_ptr peer, protocol::TMLedgerData co return -1; } - if (san.isUseful()) - progress_ = true; - - stats_ += san; - return san.getGood(); + return recordBatchOutcome(san); } if ((packet.type() == protocol::liTX_NODE) || (packet.type() == protocol::liAS_NODE)) @@ -1222,16 +1566,43 @@ InboundLedger::processData(std::shared_ptr peer, protocol::TMLedgerData co << ((packet.type() == protocol::liTX_NODE) ? "TX" : "AS") << " node stats: " << san.get(); - if (san.isUseful()) - progress_ = true; - - stats_ += san; - return san.getGood(); + return recordBatchOutcome(san); } return -1; } +int +InboundLedger::recordBatchOutcome(SHAMapAddNode const& san) +{ + if (san.isUseful()) + progress_ = true; + + stats_ += san; + + // Emit the tallies the trace log above already printed. receiveNode() walks + // every node in the packet, so these MUST stay out here: the loop has + // finished and the tallies are aggregated, giving at most three counter Adds + // per received packet rather than per node. The split is what separates real + // progress (good) from wasted bandwidth (duplicate) and a misbehaving peer + // (invalid) -- traffic-level metrics show all three as healthy throughput. + auto const emit = [this](char const* outcome, int count) { + if (count <= 0) + return; + XRPL_METRIC_COUNTER_ADD_LABELED( + app_, + telemetry::metric::syncAddnodeTotal, + "SHAMap nodes received during ledger acquire, by outcome", + static_cast(count), + {{telemetry::label::outcome, std::string(outcome)}}); + }; + emit(telemetry::lval::addnode::good, san.getGood()); + emit(telemetry::lval::addnode::duplicate, san.getDuplicate()); + emit(telemetry::lval::addnode::invalid, san.getBad()); + + return san.getGood(); +} + namespace detail { // Track the amount of useful data that each peer returns struct PeerDataCounts @@ -1332,10 +1703,13 @@ InboundLedger::runData() if (receivedData_.empty()) { receiveDispatched_ = false; + receivedDataDepth_.store(0, std::memory_order_relaxed); break; } data.swap(receivedData_); + // The stash was just drained into `data`; keep the mirror in step. + receivedDataDepth_.store(receivedData_.size(), std::memory_order_relaxed); } for (auto& entry : data) diff --git a/src/xrpld/app/ledger/detail/InboundLedgers.cpp b/src/xrpld/app/ledger/detail/InboundLedgers.cpp index 03d4e3554d5..d3ef81cb18a 100644 --- a/src/xrpld/app/ledger/detail/InboundLedgers.cpp +++ b/src/xrpld/app/ledger/detail/InboundLedgers.cpp @@ -25,10 +25,12 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -440,6 +442,37 @@ class InboundLedgersImp : public InboundLedgers return ledgers_.size(); } + AcquireProgress + acquireProgress() override + { + // Copy the handles under the lock, then read each acquire's atomics + // without it -- same pattern gotFetchPack() uses, so the ~10 s telemetry + // poll never contends with the node-receive path. + std::vector> acquires; + { + ScopedLockType const sl(lock_); + acquires.reserve(ledgers_.size()); + for (auto const& it : ledgers_) + { + XRPL_ASSERT( + it.second, "xrpl::InboundLedgersImp::acquireProgress : non-null ledger"); + acquires.push_back(it.second); + } + } + + AcquireProgress out; + out.inFlight = acquires.size(); + for (auto const& acquire : acquires) + { + out.maxMissingStateNodes = + std::max(out.maxMissingStateNodes, acquire->getMissingNodeCount(SHAMapType::STATE)); + out.maxMissingTxNodes = std::max( + out.maxMissingTxNodes, acquire->getMissingNodeCount(SHAMapType::TRANSACTION)); + out.receivedDataDepth += acquire->getReceivedDataDepth(); + } + return out; + } + private: clock_type& clock_; diff --git a/src/xrpld/app/ledger/detail/LedgerDeltaAcquire.cpp b/src/xrpld/app/ledger/detail/LedgerDeltaAcquire.cpp index 7ac85b892e3..4cb449a08fe 100644 --- a/src/xrpld/app/ledger/detail/LedgerDeltaAcquire.cpp +++ b/src/xrpld/app/ledger/detail/LedgerDeltaAcquire.cpp @@ -10,6 +10,8 @@ #include #include #include +#include +#include #include #include @@ -105,6 +107,20 @@ LedgerDeltaAcquire::trigger(std::size_t limit, ScopedLockType& sl) { JLOG(journal_.debug()) << "Fall back for " << hash_; timerInterval_ = LedgerReplayParameters::kSubTaskFallbackTimeout; + + // Same fallback as the skip-list stage, for the delta + // stage: too few replay-capable peers, so the whole + // ledger is acquired instead of just its delta. The + // `stage` label separates the two, because they fail + // for different reasons and are fixed differently. + // Emitted once, on the transition into fallback. + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + telemetry::metric::ledgerReplayFallbackTotal, + "Replay sub-acquires that fell back to a full ledger acquire", + {{telemetry::label::stage, + std::string(telemetry::lval::replay_fallback::delta)}}); + fallBack_ = true; } } diff --git a/src/xrpld/app/ledger/detail/LedgerMaster.cpp b/src/xrpld/app/ledger/detail/LedgerMaster.cpp index 452d4d96982..e4478344593 100644 --- a/src/xrpld/app/ledger/detail/LedgerMaster.cpp +++ b/src/xrpld/app/ledger/detail/LedgerMaster.cpp @@ -18,6 +18,8 @@ #include #include #include +#include +#include #include #include @@ -68,23 +70,38 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include #include #include #include #include +#include #include #include namespace xrpl { +// Full span names for the per-ledger trace join (see +// LedgerMaster::makeLedgerTraceSpan). SpanGuard::hashSpan() takes ONE complete +// span name, unlike SpanGuard::span(), which takes a prefix and a suffix — so +// the joined form is needed. Composed here from the authoritative `op::` +// suffixes in LedgerSpanNames.h rather than written out, so the wire span names +// stay "ledger.validate" / "ledger.store" by construction and no new string +// literal exists anywhere. +static constexpr auto kLedgerValidateSpan = + telemetry::join(telemetry::seg::ledger, telemetry::ledger_span::op::validate); +static constexpr auto kLedgerStoreSpan = + telemetry::join(telemetry::seg::ledger, telemetry::ledger_span::op::store); + // Don't catch up more than 100 ledgers (cannot exceed 256) static constexpr int kMaxLedgerGap{100}; @@ -145,6 +162,34 @@ LedgerMaster::LedgerMaster( { } +telemetry::SpanGuard +LedgerMaster::makeLedgerTraceSpan( + std::string_view const name, + uint256 const& ledgerHash, + std::uint32_t const seq) +{ + using namespace telemetry; + + // The join: hashSpan derives the trace id from ledgerHash[0:16], so every + // stage that passes the same ledger hash lands in the same trace with no + // context propagation between the threads. See the header for why. + auto span = + SpanGuard::hashSpan(TraceCategory::Ledger, name, ledgerHash.data(), ledgerHash.kBytes); + + // Guarded so the hash-to-string conversion is skipped entirely when the + // span is null (telemetry off, or ledger tracing disabled). + if (span) + { + // ledger_hash is the join key, so it is recorded on every stage: it is + // what an operator searches by to pull up the whole trace for one + // ledger, and it is how a reader confirms two spans really are the same + // ledger rather than a trace-id coincidence. + span.setAttribute(ledger_span::attr::ledgerHash, to_string(ledgerHash).c_str()); + span.setAttribute(ledger_span::attr::ledgerSeq, static_cast(seq)); + } + return span; +} + LedgerIndex LedgerMaster::getCurrentLedgerIndex() { @@ -460,8 +505,10 @@ bool LedgerMaster::storeLedger(std::shared_ptr ledger) { using namespace telemetry; - auto span = SpanGuard::span(TraceCategory::Ledger, seg::ledger, ledger_span::op::store); - span.setAttribute(ledger_span::attr::ledgerSeq, static_cast(ledger->header().seq)); + // Same ledger-hash join as ledger.validate: persisting a ledger is part of + // that ledger's story, so a slow store shows up in the same trace as the + // acquire that fetched it rather than as an unrelated one-span trace. + auto span = makeLedgerTraceSpan(kLedgerStoreSpan, ledger->header().hash, ledger->header().seq); bool const validated = ledger->header().validated; // Returns true if we already had the ledger @@ -977,15 +1024,69 @@ LedgerMaster::checkAccept(std::shared_ptr const& ledger) auto validations = app_.getValidators().negativeUNLFilter( app_.getValidations().getTrustedForLedger(ledger->header().hash, ledger->header().seq)); auto const tvc = validations.size(); + + // --- Sync diagnostics: snapshot the pre-accept quorum gate -------------- + // Stored before the shortfall check, so a node that keeps failing the gate + // still reports both numbers. That is the case these signals exist for: + // the tally alone cannot say whether validations are accumulating toward + // quorum (slow) or plateaued below it (stuck) without the target beside it. + // Two relaxed stores, once per validated ledger, not in a loop. + lastTrustedTally_.store(static_cast(tvc), std::memory_order_relaxed); + + // ValidatorList disables quorum by returning SIZE_MAX, which getNeededValidations + // passes straight through. Casting that to int64_t would wrap to -1 and make the + // tally look like it exceeds the target on a node that can never validate, so + // report the disabled state as int64 max instead: the target then reads far above + // any tally, which is the truthful signal. Mirrors the same fix in the unl_quorum + // gauge (MetricsRegistry::registerUnlQuorumGauge). + lastQuorumTarget_.store( + minVal == std::numeric_limits::max() ? std::numeric_limits::max() + : static_cast(minVal), + std::memory_order_relaxed); + if (tvc < minVal) // nothing we can do { JLOG(journal_.trace()) << "Only " << tvc << " validations for " << ledger->header().hash; + + // Trusted validations did not reach quorum, so this ledger will not be + // declared validated. Previously trace-only, which made a node that + // peers and receives validations yet never validates indistinguishable + // from an idle one. One macro call at the gate, never in a loop. + // + // Emitted while mutex_ is held. That is unavoidable here (the gate and + // its early return are inside the locked section) and consistent with + // the telemetry this function already emits under the same lock -- the + // ledger.validate span below, and the validation tracker reached + // through setValidLedger. It cannot deadlock against the metrics poll: + // every accessor the sync gauges read is a lock-free atomic load, so no + // OTel callback ever acquires mutex_. + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + telemetry::metric::ledgerQuorumShortfallTotal, + "Pre-accept gate rejections because trusted validations were below quorum", + {{telemetry::label::stage, std::string(telemetry::lval::quorum_shortfall::preAccept)}}); return; } + // The gate passed, so this node now has a fully-validated ledger. Record how + // long that took, exactly once per process: a fresh node that never gets + // here keeps reporting 0, which is the diagnostic reading. Writing under + // mutex_ makes the once-only check exclusive without a compare-exchange. + if (timeToFirstValidatedUs_.load(std::memory_order_relaxed) == 0) + { + auto const elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - startTime_); + // Clamp to 1: a 0 would be indistinguishable from "never reached". + timeToFirstValidatedUs_.store( + std::max(1, elapsed.count()), std::memory_order_relaxed); + } + using namespace telemetry; - auto valSpan = SpanGuard::span(TraceCategory::Ledger, seg::ledger, ledger_span::op::validate); - valSpan.setAttribute(ledger_span::attr::ledgerSeq, static_cast(ledger->header().seq)); + // Keyed on the ledger hash so the acceptance decision joins the same trace + // as that ledger's acquire and store spans, even though all three run on + // different threads. ledger_seq and ledger_hash come from the helper. + auto valSpan = + makeLedgerTraceSpan(kLedgerValidateSpan, ledger->header().hash, ledger->header().seq); valSpan.setAttribute(ledger_span::attr::validations, static_cast(tvc)); JLOG(journal_.info()) << "Advancing accepted ledger to " << ledger->header().seq diff --git a/src/xrpld/app/ledger/detail/LedgerReplayTask.cpp b/src/xrpld/app/ledger/detail/LedgerReplayTask.cpp index e7cd0312472..98f08ca1d5b 100644 --- a/src/xrpld/app/ledger/detail/LedgerReplayTask.cpp +++ b/src/xrpld/app/ledger/detail/LedgerReplayTask.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include @@ -208,13 +210,37 @@ LedgerReplayTask::tryAdvance(ScopedLockType& sl) complete_ = true; JLOG(journal_.info()) << "Completed " << hash_; + + // Terminal success. Emitted once per task: the loop above is guarded by + // isDone() at every entry point, so a completed task cannot re-enter + // and double-count. + recordOutcome(telemetry::lval::replay_outcome::success); } catch (std::runtime_error const&) { failed_ = true; + + // A delta failed to build on top of its parent, so the replayed range + // cannot be reconstructed. Previously not logged at all here, only + // reflected in failed_. + recordOutcome(telemetry::lval::replay_outcome::buildFailed); } } +void +LedgerReplayTask::recordOutcome(char const* outcome) const +{ + // One labelled counter Add per terminal event, never per delta or per + // ledger. Replay quietly falling back to plain acquisition is the failure + // this makes visible: without it, a replay that never succeeds is + // indistinguishable from one that was never attempted. + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + telemetry::metric::ledgerReplayOutcomeTotal, + "Ledger replay tasks by terminal outcome", + {{telemetry::label::outcome, std::string(outcome)}}); +} + void LedgerReplayTask::updateSkipList( uint256 const& hash, @@ -229,6 +255,12 @@ LedgerReplayTask::updateSkipList( { JLOG(journal_.error()) << "Parameter update failed " << hash_; failed_ = true; + + // The acquired skip list did not match what this task asked for, + // so the task is abandoned before any delta is fetched. A distinct + // outcome from a timeout: this one indicates a peer served an + // inconsistent skip list, not a slow or absent peer. + recordOutcome(telemetry::lval::replay_outcome::parameterFailed); return; } } @@ -247,6 +279,11 @@ LedgerReplayTask::onTimer(bool progress, ScopedLockType& sl) { failed_ = true; JLOG(journal_.debug()) << "LedgerReplayTask Failed, too many timeouts " << hash_; + + // The task ran out of retries waiting for its deltas. This is the + // outcome that pairs with the fallback counters: the sub-acquires gave + // up, and so did the task above them. + recordOutcome(telemetry::lval::timeout); } else { diff --git a/src/xrpld/app/ledger/detail/LedgerSpanNames.h b/src/xrpld/app/ledger/detail/LedgerSpanNames.h index 4de4d248245..9449593e44e 100644 --- a/src/xrpld/app/ledger/detail/LedgerSpanNames.h +++ b/src/xrpld/app/ledger/detail/LedgerSpanNames.h @@ -12,11 +12,24 @@ * ledger.store (LedgerMaster — ledger storage) * ledger.validate (LedgerMaster — ledger validation acceptance) * ledger.acquire (InboundLedger — fetch a missing ledger from peers) + * +-- ledger.acquire.header (the ledger header / liBASE phase) + * +-- ledger.acquire.astree (the account-state SHAMap phase) + * +-- ledger.acquire.txtree (the transaction SHAMap phase) + * ledger.serve (PeerImp — serve a peer's ledger-data request) * tx.apply (BuildLedger — transaction application) + * txset.acquire (TransactionAcquire — fetch a proposed tx set) + * + * Why the acquire phases are separate child spans: a fresh sync is + * dominated by the account-state tree, but the flat parent span cannot + * separate it from the (usually tiny) transaction tree or from the header + * wait that gates both. Each phase carries its own missing-node count and + * timeout flag, so the phase that is stuck names itself. */ #include +#include + namespace xrpl::telemetry::ledger_span { // ===== Span operation suffixes =============================================== @@ -27,8 +40,42 @@ inline constexpr auto store = makeStr("store"); inline constexpr auto validate = makeStr("validate"); inline constexpr auto apply = makeStr("apply"); inline constexpr auto acquire = makeStr("acquire"); +inline constexpr auto serve = makeStr("serve"); } // namespace op +// ===== Span prefixes ========================================================= + +namespace prefix { +/** + * "txset" — root prefix for transaction-set acquisition spans. A tx set is + * not a ledger, so it gets its own root rather than hiding under `ledger.`. + */ +inline constexpr auto txset = makeStr("txset"); +} // namespace prefix + +// ===== Full span names ======================================================= +// +// Joined names for the factories that take one complete span name rather than +// a prefix/suffix pair (childSpan(name) / childSpan(name, ctx)). + +/** + * The three phases of one ledger acquisition, each a child of ledger.acquire. + * + * `astree` and `txtree` are single words on purpose: they are the wire span + * names a dashboard and TraceQL query match on, so they must stay stable and + * unambiguous rather than reading as a further-nested `as.tree`. + */ +/** + * The parent acquire span's full name. Named here rather than composed at the + * call site because it is passed to hashSpan(), which takes one complete name, + * and because the phase names below are built from the same two segments. + */ +inline constexpr auto acquireFull = join(seg::ledger, op::acquire); + +inline constexpr auto acquireHeader = join(join(seg::ledger, op::acquire), makeStr("header")); +inline constexpr auto acquireAsTree = join(join(seg::ledger, op::acquire), makeStr("astree")); +inline constexpr auto acquireTxTree = join(join(seg::ledger, op::acquire), makeStr("txtree")); + // ===== Attribute keys ======================================================== namespace attr { @@ -50,11 +97,51 @@ inline constexpr auto validations = makeStr("validations"); /** * ledger.acquire attrs (InboundLedger fetch lifecycle). + * + * The target ledger's identity uses the shared `ledgerHash` / `ledgerSeq` + * keys aliased above, so a stuck acquire is findable in TraceQL by the exact + * ledger it was fetching. Both are per-ledger values and therefore stay + * trace-only: they are Tempo-searchable but are never promoted to spanmetrics + * dimensions, which would mint one metric series per ledger. */ inline constexpr auto acquireReason = makeStr("acquire_reason"); inline constexpr auto timeouts = makeStr("timeouts"); inline constexpr auto peerCount = makeStr("peer_count"); inline constexpr auto outcome = makeStr("outcome"); + +/** + * Per-phase ledger.acquire attrs (header / AS-tree / TX-tree child spans). + * + * `missingNodes` is the count the phase's last getMissingNodes() sweep already + * produced, so recording it costs nothing extra; `timedOut` says whether the + * phase ended on the retry budget rather than on completion. Both are bounded + * only by the tree size, so `missing_nodes` stays span-only (Tempo-searchable) + * while `timed_out` — two values — is safe as a spanmetrics dimension. + */ +inline constexpr auto missingNodes = makeStr("missing_nodes"); +inline constexpr auto timedOut = makeStr("timed_out"); + +/** + * txset.acquire attrs (TransactionAcquire fetch lifecycle). + * + * The set's own root hash identifies which proposed set stalled, so like + * `ledgerHash` it is per-object and stays span-only. `durationMs` is recorded + * explicitly rather than left to the span's own duration because the span is + * ended from the acquiring thread and its wall time is the number an operator + * reads directly off a trace. + */ +inline constexpr auto txSetHash = makeStr("txset_hash"); +inline constexpr auto durationMs = makeStr("duration_ms"); + +/** + * ledger.serve attrs (the JtLedgerReq worker answering a peer). + * + * `objectType` names which of the four request kinds was served, and + * `servedNodes` how many SHAMap nodes the reply carried — accumulated by the + * reply-assembly loop and written once after it, never per node. + */ +inline constexpr auto objectType = makeStr("object_type"); +inline constexpr auto servedNodes = makeStr("served_nodes"); } // namespace attr // ===== Attribute values ====================================================== @@ -62,15 +149,263 @@ inline constexpr auto outcome = makeStr("outcome"); namespace val { /** * ledger.acquire outcome values. + * + * Every acquire records exactly one of these before its span closes, so a + * span-derived outcome rate never silently drops an acquire that was still + * in flight. + * + * - complete: all header, transaction and state data was assembled. + * - failed: the fetch reached a terminal error (bad data, or the retry + * budget ran out). + * - abandoned: the InboundLedger was destroyed while still fetching, which + * happens when the sweeper drops an acquire that made no + * progress, or on shutdown. This is the outcome for a fetch + * that never reached a result at all -- the case that used to + * leave the span with no outcome and a duration stretched to + * the sweep interval instead of the real fetch time. */ inline constexpr auto complete = makeStr("complete"); inline constexpr auto failed = makeStr("failed"); +inline constexpr auto abandoned = makeStr("abandoned"); +/** + * Extra terminal value for the per-phase acquire and txset.acquire spans. + * + * `ledger.acquire` itself never uses this: an acquire that exhausts its retry + * budget sets failed_, so its outcome is `failed`. A phase and a tx set can + * instead end while the parent fetch is still alive, and `timeout` is what + * names that -- the phase ran out of time, but nothing failed permanently. + */ +inline constexpr auto timeout = makeStr("timeout"); + /** * ledger.acquire reason values (mirror InboundLedger::Reason). */ inline constexpr auto history = makeStr("history"); inline constexpr auto consensus = makeStr("consensus"); inline constexpr auto generic = makeStr("generic"); + +/** + * ledger.serve object_type values (mirror the protobuf `itype` request kinds). + * + * Four bounded values, one per TMGetLedger info type: the ledger header + * (`liBASE`), the transaction tree (`liTX_NODE`), the account-state tree + * (`liAS_NODE`), and a proposed transaction set (`liTS_CANDIDATE`). Serving + * the state tree is what a syncing peer needs most, so telling it apart from + * the cheap header replies is the point of the split. + */ +inline constexpr auto header = makeStr("header"); +inline constexpr auto txTree = makeStr("tx"); +inline constexpr auto asTree = makeStr("as"); +inline constexpr auto txSet = makeStr("txset"); + +/** + * ledger.serve outcome values. + * + * - complete: a reply with at least one node was sent. + * - partial: nodes were sent but the reply hit a size cap, so the requester + * must ask again for the rest. + * - refused: nothing was sent. The paired `serve_refused_total` counter + * carries the specific cause; the span records only that this + * request went unanswered, so a trace shows the gap. + */ +inline constexpr auto partial = makeStr("partial"); +inline constexpr auto refused = makeStr("refused"); } // namespace val +// ===== Outcome rule ========================================================== + +/** + * Pick the terminal `outcome` value for an acquire from its own state flags. + * + * This is the whole rule behind "every ledger.acquire span carries an + * outcome". InboundLedger has four exit paths (done(), the local-store + * shortcut, the "can never be acquired" exit, and the destructor when a fetch + * is swept), and all of them derive the value here instead of naming one, so + * no exit can label itself wrongly and no exit can be added without getting an + * outcome. + * + * The "neither flag" case is the one that matters: an acquire destroyed while + * still fetching reached no result, and reporting it as `abandoned` is what + * keeps a stuck-then-swept fetch in the outcome rate instead of vanishing + * from it. + * + * Kept here, next to the values it returns, as a pure constexpr function: it + * has no dependency on InboundLedger and can therefore be asserted directly + * from the lib-only test binary, which cannot link xrpld. + * + * @param failed The acquire's `failed_` flag (terminal error). + * @param complete The acquire's `complete_` flag (all data assembled). + * @return `failed` when failed is set, `complete` when only complete is set, + * otherwise `abandoned`. + * + * Example -- the three live cases: + * @code + * acquireOutcome(false, true); // "complete" -- normal success + * acquireOutcome(true, false); // "failed" -- terminal error + * acquireOutcome(false, false); // "abandoned" -- swept mid-fetch + * @endcode + * + * Example -- edge case: a failure recorded on an otherwise complete acquire + * still reports `failed`, because a fetch that hit a terminal error is not a + * success no matter what else was assembled: + * @code + * acquireOutcome(true, true); // "failed" + * @endcode + * + * @note Pure and side-effect free; safe to call from any thread, including a + * destructor (it allocates nothing and cannot throw). + */ +[[nodiscard]] constexpr std::string_view +acquireOutcome(bool failed, bool complete) noexcept +{ + if (failed) + return val::failed; + if (complete) + return val::complete; + return val::abandoned; +} + +/** + * Pick the terminal `outcome` for ONE acquire phase (header / AS-tree / + * TX-tree) or for a tx-set fetch. + * + * The sibling of acquireOutcome() for the units whose lifetime is shorter than + * the whole fetch. It takes one more input, `timedOut`, because these units + * have a fourth end state the parent does not: a unit can stop because the + * retry budget expired. Reporting that as `failed` would say "bad data" and as + * `abandoned` would say "we stopped caring", so `timeout` names it directly -- + * and it is the value a stuck fresh sync shows, which is why these spans + * exist. + * + * Precedence is `timeout` > `failed` > `complete` > `abandoned`, and the + * timeout-first order is the load-bearing part. The exhausted-budget path in + * both emitters sets the terminal `failed_` flag as well, because that flag is + * how the TimeoutCounter base stops its timer loop -- so `timedOut` implies + * `failed`, and checking `failed` first would relabel every timeout as a data + * fault and erase the distinction. A genuine data fault never sets `timedOut`, + * so nothing is lost the other way round. + * + * A pure constexpr function with no dependency on InboundLedger or + * TransactionAcquire, so the whole rule is assertable from the lib-only test + * binary, which cannot link xrpld. + * + * @param failed The unit's `failed_` flag (terminal error). + * @param complete The unit's `complete_` flag (all data assembled). + * @param timedOut Whether the unit's retry budget expired before it ended. + * @return `timeout`, `failed`, `complete` or `abandoned`, in that precedence. + * + * Example -- the healthy and the stuck case: + * @code + * phaseOutcome(false, true, false); // "complete" -- assembled in time + * phaseOutcome(true, false, true); // "timeout" -- budget gone; the failed_ + * // flag the same path sets is subsumed + * @endcode + * + * Example -- edge case: a unit torn down mid-fetch with no flag set at all + * still reports a value, so it stays in the outcome rate: + * @code + * phaseOutcome(false, false, false); // "abandoned" + * @endcode + * + * @note Pure and side-effect free; safe to call from any thread, including a + * destructor (it allocates nothing and cannot throw). + */ +[[nodiscard]] constexpr std::string_view +phaseOutcome(bool failed, bool complete, bool timedOut) noexcept +{ + if (timedOut) + return val::timeout; + if (failed) + return val::failed; + if (complete) + return val::complete; + return val::abandoned; +} + +/** + * Pick the `object_type` value for a served ledger-data request from the + * protobuf `itype` the peer asked with. + * + * Kept as a rule rather than a per-branch literal because + * `processLedgerRequest` reaches its exits from four different places, and a + * per-branch value would let two of them disagree about the same request. It + * takes a plain int so this header stays free of the protobuf headers and the + * rule remains assertable from the lib-only test binary; the caller passes + * `m->itype()`, whose enum values are fixed by the wire protocol. + * + * @param itype The protobuf TMLedgerInfoType: 0 `liBASE`, 1 `liTX_NODE`, + * 2 `liAS_NODE`, 3 `liTS_CANDIDATE`. + * @return `header`, `tx`, `as` or `txset`; `header` for an unrecognised value, + * which cannot occur because onMessage() rejects the request first. + * + * Example -- the two request kinds a syncing peer sends most: + * @code + * serveObjectType(2); // "as" -- account-state tree, the bulk of a sync + * serveObjectType(0); // "header" -- the cheap liBASE reply + * @endcode + * + * @note Pure and side-effect free. + */ +[[nodiscard]] constexpr std::string_view +serveObjectType(int itype) noexcept +{ + switch (itype) + { + case 1: + return val::txTree; + case 2: + return val::asTree; + case 3: + return val::txSet; + default: + return val::header; + } +} + +/** + * Pick the `outcome` value for a served ledger-data request from the size of + * the reply it produced. + * + * `processLedgerRequest` has eight exits and only one of them sends anything, + * so a per-branch value would be seven chances to mislabel. Deriving it from + * the reply itself removes that: the reply's node count is the one piece of + * state that already exists at every exit, and is zero on all seven refusals. + * + * A reply that reached the soft cap is `partial`, not `complete`: the assembly + * loop stopped early and the requester must ask again for the rest, so counting + * it as a success would hide the round trips a large tree really costs. + * + * @param servedNodes Nodes in the reply, i.e. `ledgerData.nodes_size()`. + * @param softCap The reply-size cap the assembly loop stops at + * (`Tuning::kSoftMaxReplyNodes`). Passed in so this header needs no + * overlay dependency and the rule stays assertable from the lib-only + * test binary. + * @return `refused` for an empty reply, `partial` at or above the cap, + * otherwise `complete`. + * + * Example -- the served and the refused case: + * @code + * serveOutcome(12, 128); // "complete" -- whole request answered + * serveOutcome(0, 128); // "refused" -- nothing sent; the paired + * // serve_refused_total counter says why + * @endcode + * + * Example -- edge case: a reply that filled the cap is not a success, because + * the peer must come back for the remainder: + * @code + * serveOutcome(128, 128); // "partial" + * @endcode + * + * @note Pure and side-effect free; safe to call while unwinding. + */ +[[nodiscard]] constexpr std::string_view +serveOutcome(int servedNodes, int softCap) noexcept +{ + if (servedNodes <= 0) + return val::refused; + if (servedNodes >= softCap) + return val::partial; + return val::complete; +} + } // namespace xrpl::telemetry::ledger_span diff --git a/src/xrpld/app/ledger/detail/SkipListAcquire.cpp b/src/xrpld/app/ledger/detail/SkipListAcquire.cpp index 8ebd14083ad..c69ae82d76f 100644 --- a/src/xrpld/app/ledger/detail/SkipListAcquire.cpp +++ b/src/xrpld/app/ledger/detail/SkipListAcquire.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include @@ -100,6 +102,21 @@ SkipListAcquire::trigger(std::size_t limit, ScopedLockType& sl) { JLOG(journal_.debug()) << "Fall back for " << hash_; timerInterval_ = LedgerReplayParameters::kSubTaskFallbackTimeout; + + // Too few peers support ledger replay, so this + // sub-task gives up on the skip-list shortcut and + // acquires the whole ledger instead. That silently + // defeats the replay optimisation -- debug-log-only + // until now. Emitted on the transition into fallback + // (fallBack_ is still false here), not at the acquire + // call below, which re-runs on every later trigger. + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + telemetry::metric::ledgerReplayFallbackTotal, + "Replay sub-acquires that fell back to a full ledger acquire", + {{telemetry::label::stage, + std::string(telemetry::lval::replay_fallback::skiplist)}}); + fallBack_ = true; } } diff --git a/src/xrpld/app/ledger/detail/TransactionAcquire.cpp b/src/xrpld/app/ledger/detail/TransactionAcquire.cpp index 62312b04d26..8c9e260ae50 100644 --- a/src/xrpld/app/ledger/detail/TransactionAcquire.cpp +++ b/src/xrpld/app/ledger/detail/TransactionAcquire.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -14,11 +15,14 @@ #include #include #include +#include #include #include +#include #include +#include #include #include #include @@ -50,32 +54,114 @@ TransactionAcquire::TransactionAcquire( map_->setUnbacked(); } +TransactionAcquire::~TransactionAcquire() +{ + // Last exit. A set dropped here (swept by newRound because it never + // arrived, or torn down at shutdown) reached no result, so this is what + // stamps outcome=abandoned instead of exporting a span with no outcome at + // all. Already-finalized acquisitions are untouched: the helper is + // idempotent. + finalizeAcquireSpan(); +} + void -TransactionAcquire::done() +TransactionAcquire::finalizeAcquireSpan() noexcept { - // We hold a PeerSet lock and so cannot do real work here + // Idempotent: the handle is cleared below, so a later exit finds nothing to + // finalize and cannot overwrite the outcome the real exit recorded. + if (!acquireSpan_) + return; - if (failed_) + // The attribute writes are wrapped because the destructor is one of the + // callers: an exception escaping there during unwinding would terminate the + // process. Each setAttribute is itself noexcept today; the try is the + // structural guarantee that stays correct if that ever changes. + try { - JLOG(journal_.debug()) << "Failed to acquire TX set " << hash_; + if (*acquireSpan_) + { + using namespace telemetry; + // Derived from this object's own flags by the shared rule, so no + // call site can mislabel an exit and every exit gets an outcome. + // No flag set means the set was dropped while still in flight. + acquireSpan_->setAttribute( + ledger_span::attr::outcome, + ledger_span::phaseOutcome(failed_, complete_, timedOut_)); + acquireSpan_->setAttribute( + ledger_span::attr::timeouts, static_cast(timeouts_)); + // Recorded explicitly as well as implied by the span's own + // duration: this is the number an operator reads straight off a + // trace when asking how long a proposed set took to arrive. + acquireSpan_->setAttribute( + ledger_span::attr::durationMs, + static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now() - acquireStart_) + .count())); + // Peers still tracked for this fetch. Safe here, unlike the ledger + // equivalent: PeerSet::getPeerIds() returns its own member set and + // takes no Overlay lock. + acquireSpan_->setAttribute( + ledger_span::attr::peerCount, + static_cast(peerSet_->getPeerIds().size())); + } } - else + catch (...) // NOLINT(bugprone-empty-catch) + { + // Telemetry must never break an acquisition, and this also runs from + // the destructor. A span missing one attribute is still worth + // exporting, so fall through and end it below. + } + + // End the span, outside the try so it happens on every path. Unconditional + // so the span never leaks even when it was inactive, and so this helper is + // exactly-once: a later exit sees an empty handle and returns above. + // ~SpanGuard is implicitly noexcept, so this cannot throw out of here. + acquireSpan_.reset(); +} + +void +TransactionAcquire::done() +{ + // We hold a PeerSet lock and so cannot do real work here + + // Keep the span active as the ambient context across the outcome log below + // so that line carries the span's trace_id. The activation is non-owning; + // acquireSpan_ still owns the span. It pops at the end of this block, while + // the span is still alive, and only then is the span finalized and ended. { - JLOG(journal_.debug()) << "Acquired TX set " << hash_; - map_->setImmutable(); - - uint256 const& hash(hash_); - std::shared_ptr const& map(map_); - auto const pap = &app_; - // Note that, when we're in the process of shutting down, addJob() - // may reject the request. If that happens then giveSet() will - // not be called. That's fine. According to David the giveSet() call - // just updates the consensus and related structures when we acquire - // a transaction set. No need to update them if we're shutting down. - app_.getJobQueue().addJob(JtTxnData, "ComplAcquire", [pap, hash, map]() { - pap->getInboundTransactions().giveSet(hash, map, true); - }); + auto acquireActivation = telemetry::activateIfLive(acquireSpan_); + + if (failed_) + { + JLOG(journal_.debug()) << "Failed to acquire TX set " << hash_; + } + else + { + JLOG(journal_.debug()) << "Acquired TX set " << hash_; + map_->setImmutable(); + + uint256 const& hash(hash_); + std::shared_ptr const& map(map_); + auto const pap = &app_; + // Note that, when we're in the process of shutting down, addJob() + // may reject the request. If that happens then giveSet() will + // not be called. That's fine. According to David the giveSet() + // call just updates the consensus and related structures when we + // acquire a transaction set. No need to update them if we're + // shutting down. + app_.getJobQueue().addJob(JtTxnData, "ComplAcquire", [pap, hash, map]() { + pap->getInboundTransactions().giveSet(hash, map, true); + }); + } + // acquireActivation pops here, before the span is ended below. } + + // The normal exit. done() is reached from trigger() (complete or invalid + // data) and from onTimer() (timeout budget exhausted), and both are + // terminal, so the span ends here rather than waiting for the object to be + // released. TimeoutCounter::isDone() is already true by now, so the timer + // loop will not call back in. + finalizeAcquireSpan(); } void @@ -83,6 +169,11 @@ TransactionAcquire::onTimer(bool progress, ScopedLockType& psl) { if (timeouts_ > kMaxTimeouts) { + // Record WHY before done() finalizes the span. failed_ alone would + // report `failed`, which reads as bad data; this path is instead "no + // peer ever supplied the set", and that is the distinction a stuck + // fresh sync turns on. + timedOut_ = true; failed_ = true; done(); return; @@ -245,6 +336,37 @@ TransactionAcquire::init(int numPeers) { ScopedLockType sl(mtx_); + // Start the clock and the span before the first peer is asked. addPeers() + // below calls trigger() per peer, and trigger() can reach done() in the + // same call stack, so anything set up after it could be missed entirely by + // a set that resolves immediately. + acquireStart_ = std::chrono::steady_clock::now(); + + // Span the acquisition so a proposed tx set that never arrives is + // traceable. TransactionAcquire had no telemetry at all before this, so a + // consensus round stalled waiting on a set looked identical to an idle one. + // Finalized by finalizeAcquireSpan() on whichever exit this object takes, + // including the destructor. + { + using namespace telemetry; + // acquireSpan_ is emplaced here but may be reset on a JtTxnData worker + // thread. A SpanGuard is thread-free (owns no thread-local Scope), so it + // can be created here and destroyed on the worker with no scope to + // strip. Category Ledger: a tx set is ledger-fetch traffic, and this + // shares the `trace_ledger` flag with ledger.acquire so the two halves + // of a stuck sync cannot be enabled apart. + acquireSpan_.emplace( + SpanGuard::span( + TraceCategory::Ledger, ledger_span::prefix::txset, ledger_span::op::acquire)); + if (*acquireSpan_) + { + // The set's root hash is the only identity it has -- there is no + // sequence number -- so it is what makes a stalled fetch findable + // in a trace search. + acquireSpan_->setAttribute(ledger_span::attr::txSetHash, to_string(hash_).c_str()); + } + } + addPeers(numPeers); setTimer(sl); diff --git a/src/xrpld/app/ledger/detail/TransactionAcquire.h b/src/xrpld/app/ledger/detail/TransactionAcquire.h index 5b33066390b..4b88f526ad3 100644 --- a/src/xrpld/app/ledger/detail/TransactionAcquire.h +++ b/src/xrpld/app/ledger/detail/TransactionAcquire.h @@ -10,16 +10,59 @@ #include #include #include +#include +#include #include #include +#include #include #include namespace xrpl { // VFALCO TODO rename to PeerTxRequest -// A transaction set we are trying to acquire +/** + * A transaction set we are trying to acquire. + * + * The tx-set sibling of InboundLedger: same TimeoutCounter base, same + * trigger / onTimer / takeNodes shape, fetching the transaction SHAMap a + * consensus proposal referenced rather than a whole ledger. + * + * +-----------------+ + * | TimeoutCounter | timer loop, timeouts_/complete_/failed_ flags + * +--------+--------+ + * | + * +--------v-----------------------------+ + * | TransactionAcquire | + * | map_ : the tx SHAMap being | + * | assembled | + * | peerSet_ : peers asked for nodes | + * | acquireSpan_ : "txset.acquire" span | + * +--------------------------------------+ + * + * Acquisition lifecycle and where the span ends: + * + * init() -- span starts, timer set + * | + * +--> trigger() ---- root or missing nodes requested from a peer + * | ^ | + * | | v + * +--> takeNodes() -- nodes applied, trigger() again + * | + * +--> onTimer() ---- retry, or give up past the timeout budget + * | + * v + * done() -- span finalized (complete | failed | timeout) + * or + * ~TransactionAcquire() -- span finalized (abandoned) when the set is + * dropped by the round sweep or at shutdown + * + * @note Thread safety: unchanged by the span. `acquireSpan_` is written only + * under `mtx_` or from the destructor, both of which exclude every other + * writer. A SpanGuard owns no thread-local scope, so it can be ended on + * whichever JtTxnData worker reaches the terminal path. + */ class TransactionAcquire final : public TimeoutCounter, public std::enable_shared_from_this, public CountedObject @@ -28,7 +71,7 @@ class TransactionAcquire final : public TimeoutCounter, using pointer = std::shared_ptr; TransactionAcquire(Application& app, uint256 const& hash, std::unique_ptr peerSet); - ~TransactionAcquire() override = default; + ~TransactionAcquire() override; SHAMapAddNode takeNodes( @@ -46,6 +89,33 @@ class TransactionAcquire final : public TimeoutCounter, bool haveRoot_{false}; std::unique_ptr peerSet_; + /** + * When this acquisition started, set in init() before the first peer is + * asked. Base for the span's `duration_ms` attribute. + */ + std::chrono::steady_clock::time_point acquireStart_; + + /** + * True once the timeout budget has been exhausted, so the span's outcome + * reads `timeout` rather than the bare `failed` the flag alone would give. + * Distinct from `failed_`, which onTimer() also sets on that path: the two + * together are what separate "peers never supplied the set" from "a peer + * supplied an invalid one". + */ + bool timedOut_{false}; + + /** + * Spans the whole acquisition: started in init(), ended by + * finalizeAcquireSpan() on whichever exit this object takes, including the + * destructor. Held for the object's lifetime so a set that is dropped + * mid-fetch still reports an outcome instead of exporting a span with + * none. + * Thread-free: a SpanGuard holds no thread-local scope, so it may be + * created on the acquiring thread and ended on a JtTxnData worker or in + * the destructor. + */ + std::optional acquireSpan_; + void onTimer(bool progress, ScopedLockType& peerSetLock) override; @@ -59,6 +129,36 @@ class TransactionAcquire final : public TimeoutCounter, trigger(std::shared_ptr const&); std::weak_ptr pmDowncast() override; + + /** + * End the acquire span exactly once, stamping the outcome it reached. + * + * A tx-set acquisition can leave through two exits: done(), reached from + * trigger() on success or invalid data and from onTimer() when the timeout + * budget runs out, and the destructor, when the round sweep in + * InboundTransactions::newRound drops a set that never arrived. Both call + * this, so the span always carries an `outcome` and its duration always + * ends at the real exit rather than stretching to whenever the object + * happened to be released. + * + * The outcome is derived from this object's own flags by the shared + * phaseOutcome() rule rather than passed in, so no call site can mislabel + * an exit and no exit added later can forget one: + * failed_ -> "failed", timedOut_ -> "timeout", complete_ -> "complete", + * otherwise "abandoned" (dropped with no result). + * + * Idempotent: the span handle is cleared here, so the second and later + * calls do nothing and cannot overwrite the outcome the real exit + * recorded. + * + * @note noexcept, and the attribute writes are wrapped, because the + * destructor calls this: an exception escaping a destructor during + * unwinding would terminate the process. + * @note Called once per acquisition, never on the per-node path in + * takeNodes(). + */ + void + finalizeAcquireSpan() noexcept; }; } // namespace xrpl diff --git a/src/xrpld/app/main/Application.cpp b/src/xrpld/app/main/Application.cpp index 0580672cdb8..d037e325d78 100644 --- a/src/xrpld/app/main/Application.cpp +++ b/src/xrpld/app/main/Application.cpp @@ -39,6 +39,8 @@ #include #include #include +#include +#include #include #include @@ -1143,12 +1145,82 @@ class ApplicationImp : public Application, public BasicApp << "; size after: " << cachedSLEs_.size(); } - mallocTrim("doSweep", journal_); + trimHeapAndRecord(); // Set timer to do another sweep later. setSweepTimer(); } + /** + * Return free heap pages to the OS at the end of a sweep, and record what + * that cost. + * + * Split out of doSweep() so the metric emit site is one small unit rather + * than a further three statements on an already-long function. + * + * Why this is instrumented: `malloc_trim` runs after EVERY cache sweep, and + * its cost scales with the resident heap, so a node with a large existing + * database pays a per-sweep penalty a fresh one does not -- the leading + * explanation for "an existing database syncs slower than an empty one" on + * glibc. The report used to be discarded here, so none of it was visible. + * + * doSweep() -> trimHeapAndRecord() -> mallocTrim() + * | | + * | MallocTrimReport (duration, + * | minor faults, RSS before/after) + * v + * 3 OTel instruments + * + * Cost: one histogram Record and two counter Adds per sweep, at a cadence + * of SizedItem::SweepInterval (10-120 s), so this is free. + * + * @note The minor-fault count covers the trim call only. It cannot show the + * faults taken later, as the caches refill and touch the pages the + * trim handed back -- see the runbook branch for how to read it. + */ + void + trimHeapAndRecord() + { + MallocTrimReport const report = mallocTrim("doSweep", journal_); + + // Nothing was measured: not Linux/glibc, so there is no trim to report + // and a zero would falsely claim a free one. + if (!report.supported) + return; + + if (report.durationUs.count() >= 0) + { + XRPL_METRIC_HISTOGRAM_RECORD( + *this, + telemetry::metric::sweepMallocTrimUs, + "Duration of the malloc_trim call ending each cache sweep (microseconds)", + report.durationUs.count()); + } + + if (report.minfltDelta > 0) + { + XRPL_METRIC_COUNTER_ADD( + *this, + telemetry::metric::sweepMallocTrimMinorFaultsTotal, + "Minor page faults taken inside the sweep's malloc_trim call", + static_cast(report.minfltDelta)); + } + + // deltaKB() is after-minus-before, so a successful trim is NEGATIVE. + // Publish the reclaimed amount as a positive cumulative total and drop + // the case where RSS grew across the call (another thread allocating + // faster than the trim released): a counter cannot go down, and "grew" + // is not a reclaim of a negative size. + if (auto const deltaKB = report.deltaKB(); deltaKB < 0) + { + XRPL_METRIC_COUNTER_ADD( + *this, + telemetry::metric::sweepMallocTrimReclaimedKbTotal, + "Resident kilobytes returned to the OS by the sweep's malloc_trim", + static_cast(-deltaKB)); + } + } + LedgerIndex getMaxDisallowedLedger() override { diff --git a/src/xrpld/app/main/LoadManager.cpp b/src/xrpld/app/main/LoadManager.cpp index 36f82e392b5..5abc54e0324 100644 --- a/src/xrpld/app/main/LoadManager.cpp +++ b/src/xrpld/app/main/LoadManager.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -115,6 +116,13 @@ LoadManager::run() static constexpr auto kStallFatalLogMessageTimeLimit = 90s; static constexpr auto kStallLogicErrorTimeLimit = 600s; + // Publish the stall state for telemetry before acting on it, so the + // gauge still sees the final duration on the tick that logicErrors. + // An unarmed detector reports healthy: its heartbeat is not yet + // meaningful, so a large elapsed time there is not a stall. + updateStallState( + armed ? timeSpentStalled : 0s, std::chrono::seconds(kReportingIntervalSeconds)); + if (armed && (timeSpentStalled >= kReportingIntervalSeconds)) { // Report the stalled condition every reportingIntervalSeconds @@ -171,6 +179,23 @@ LoadManager::run() } } +void +LoadManager::updateStallState( + std::chrono::seconds const stalled, + std::chrono::seconds const reportThreshold) +{ + // Read the previous tick's value before overwriting it: the healthy -> + // reportable transition is what defines a new episode. Only the monitor + // thread writes these, so the read-then-write needs no atomicity as a pair. + auto const state = evaluateStall( + currentStallSeconds_.load(std::memory_order_relaxed), stalled, reportThreshold); + + currentStallSeconds_.store(state.seconds, std::memory_order_relaxed); + + if (state.newEpisode) + stallEventCount_.fetch_add(1, std::memory_order_relaxed); +} + //------------------------------------------------------------------------------ std::unique_ptr diff --git a/src/xrpld/app/main/LoadManager.h b/src/xrpld/app/main/LoadManager.h index 5d3f07e9965..db34b40309d 100644 --- a/src/xrpld/app/main/LoadManager.h +++ b/src/xrpld/app/main/LoadManager.h @@ -2,8 +2,10 @@ #include +#include #include #include +#include #include #include #include @@ -23,6 +25,25 @@ class Application; * * The warning system is used instead of merely dropping, because hostile * peers can just reconnect anyway. + * + * Besides warning peers, the monitor thread is the only place that knows how + * long the server's main loop has been unresponsive. That duration used to + * exist only inside a log line; it is now also published through the two + * stall accessors below so telemetry can chart it. + * + * +-------------+ heartbeat() +--------------------+ + * | main loop |--------------->| LoadManager | + * +-------------+ | (monitor thread) | + * +---------+----------+ + * stall state | | fee changes + * v v + * currentStallSeconds_ LoadFeeTrack + * stallEventCount_ + * ^ + * | getCurrentStallSeconds() + * | getStallEventCount() + * telemetry::MetricsRegistry + * (sync_state gauge callback) */ class LoadManager { @@ -66,6 +87,43 @@ class LoadManager void heartbeat(); + /** + * Seconds the server's main loop has currently been unresponsive. + * + * Zero means healthy: either the heartbeat is current or the stall + * detector is not armed yet. A non-zero value is the same duration the + * monitor thread logs as "Server stalled for N seconds", refreshed on + * its one-second tick. + * + * @return Current stall duration in seconds, 0 when not stalled. + * + * @note Safe to call from any thread, including a telemetry + * observable-gauge callback: one relaxed atomic load, no lock. + */ + [[nodiscard]] std::uint32_t + getCurrentStallSeconds() const + { + return currentStallSeconds_.load(std::memory_order_relaxed); + } + + /** + * Number of distinct stall episodes seen since process start. + * + * Counts once per episode, on the tick the stall is first reported, not + * once per second of stalling. So a rising count means new stalls keep + * happening, which is a different fault from one long stall (that shows + * up as a large getCurrentStallSeconds() with a flat count). + * + * @return Monotonic stall-episode count. + * + * @note Safe to call from any thread; one relaxed atomic load. + */ + [[nodiscard]] std::uint64_t + getStallEventCount() const + { + return stallEventCount_.load(std::memory_order_relaxed); + } + //-------------------------------------------------------------------------- void @@ -74,10 +132,87 @@ class LoadManager void stop(); + /** + * What one monitor tick concludes about the stall state. + * + * @see evaluateStall + */ + struct StallState + { + /** + * Stall seconds to publish for this tick; 0 when healthy. + */ + std::uint32_t seconds; + /** + * True only on the tick that begins a new stall episode. + */ + bool newEpisode; + }; + + /** + * Decide the stall state for one monitor tick. + * + * A stall counts as reportable at the same threshold the monitor's log line + * uses, so the gauge and the log never disagree about whether the server is + * stalled. Below the threshold the tick reports healthy (0 seconds), which + * is why a brief scheduling hiccup does not register as a stall. + * + * An episode begins on the healthy -> reportable transition only, so one + * continuous stall increments the episode count exactly once no matter how + * many ticks it spans. That is what lets an operator tell one long stall + * (large `seconds`, flat count) from repeated short ones (rising count). + * + * Pure and side-effect free: it is the whole decision rule for the stall + * signals, kept separate from the atomics it feeds so the rule can be + * asserted directly without a test-only mutator on LoadManager. + * + * @param previousSeconds Value published by the previous tick. + * @param stalled Stall duration measured on this tick. + * @param reportThreshold Duration at which a stall becomes reportable. + * @return The seconds to publish and whether a new episode started. + * + * Example -- a stall crossing the threshold, then persisting: + * @code + * // First tick over the 10 s threshold: publishes 10, starts an episode. + * auto first = LoadManager::evaluateStall(0, 10s, 10s); // {10, true} + * // Still stalled 20 s later: publishes 20, but the SAME episode. + * auto next = LoadManager::evaluateStall(10, 30s, 10s); // {30, false} + * @endcode + * + * Example -- edge case: a sub-threshold blip never becomes an episode. + * @code + * auto blip = LoadManager::evaluateStall(0, 9s, 10s); // {0, false} + * @endcode + */ + [[nodiscard]] static constexpr StallState + evaluateStall( + std::uint32_t const previousSeconds, + std::chrono::seconds const stalled, + std::chrono::seconds const reportThreshold) noexcept + { + bool const reportable = stalled >= reportThreshold; + bool const wasReportable = previousSeconds >= reportThreshold.count(); + return StallState{ + .seconds = reportable ? static_cast(stalled.count()) : 0U, + .newEpisode = reportable && !wasReportable}; + } + private: void run(); + /** + * Publish the stall state read by the telemetry gauge. + * + * Applies evaluateStall() to this tick and stores the result. Called once + * per monitor tick, only from the monitor thread. + * + * @param stalled Stall duration measured on this tick. + * @param reportThreshold Duration at which a stall becomes reportable. + */ + void + updateStallState(std::chrono::seconds stalled, std::chrono::seconds reportThreshold); + private: Application& app_; beast::Journal const journal_; @@ -91,6 +226,19 @@ class LoadManager std::chrono::steady_clock::time_point lastHeartbeat_; bool armed_; + /** + * Seconds the main loop has been unresponsive as of the last monitor + * tick; 0 when healthy. Written only by the monitor thread, read by + * telemetry, hence atomic rather than mutex-guarded. + */ + std::atomic currentStallSeconds_{0}; + + /** + * Monotonic count of stall episodes since process start. Written only by + * the monitor thread; never reset. + */ + std::atomic stallEventCount_{0}; + friend std::unique_ptr makeLoadManager(Application& app, beast::Journal journal); }; diff --git a/src/xrpld/app/misc/NetworkOPs.cpp b/src/xrpld/app/misc/NetworkOPs.cpp index ca7e14990e3..0b233aee7b5 100644 --- a/src/xrpld/app/misc/NetworkOPs.cpp +++ b/src/xrpld/app/misc/NetworkOPs.cpp @@ -31,6 +31,8 @@ #include #include #include +#include +#include #include #include #include @@ -292,6 +294,21 @@ class NetworkOPsImp final : public NetworkOPs return std::chrono::duration_cast( std::chrono::steady_clock::now() - start_); } + + /** + * Microseconds from process start to the first FULL transition. This + * is the same quantity reported as `initial_sync_duration_us` in + * json(); reading it alone avoids copying the whole counter array. + * Thread-safe. + * + * @return Time to first FULL, or 0 if FULL was never reached. + */ + std::uint64_t + initialSyncDurationUs() const + { + std::scoped_lock const lock(mutex_); + return initialSyncUs_; + } }; /** @@ -379,12 +396,18 @@ class NetworkOPsImp final : public NetworkOPs std::chrono::microseconds getServerStateDurationUs() const override; + std::uint64_t + getInitialSyncDurationUs() const override; + std::string strOperatingMode(OperatingMode const mode, bool const admin) const override; std::string strOperatingMode(bool const admin = false) const override; + std::uint32_t + getLedgersBehindNetwork() const override; + // // Transaction operations. // @@ -1010,6 +1033,43 @@ NetworkOPsImp::getServerStateDurationUs() const return accounting_.currentStateDurationUs(); } +std::uint64_t +NetworkOPsImp::getInitialSyncDurationUs() const +{ + return accounting_.initialSyncDurationUs(); +} + +std::uint32_t +NetworkOPsImp::getLedgersBehindNetwork() const +{ + // The network tip is the highest ledger sequence any connected peer says it + // holds. Peers report their range in mtSTATUS_CHANGE, which PeerImp caches, + // so this is a read of already-received data: no new network round trip. + std::uint32_t networkTarget = 0; + registry_.get().getOverlay().foreach([&networkTarget](std::shared_ptr const& peer) { + std::uint32_t minSeq = 0; + std::uint32_t maxSeq = 0; + peer->ledgerRange(minSeq, maxSeq); + networkTarget = std::max(networkTarget, maxSeq); + }); + + auto const validated = registry_.get().getLedgerMaster().getValidLedgerIndex(); + + // A node that has validated nothing is not "behind" by the whole sequence + // space; the distance is undefined until there is a validated ledger to + // measure from. Returning the raw difference here reported ~105.9 million on + // a fresh mainnet start, which is a real reading of a meaningless quantity: + // it auto-scaled every consumer's axis and would trip any threshold. Report + // zero until the first ledger is validated, and let the sync-state signals + // say how far along the initial acquire is. + if (validated == 0) + return 0; + + // Floor at zero: we can legitimately be ahead of every peer's reported + // range, and a peer that has reported nothing yet leaves the target at 0. + return networkTarget > validated ? networkTarget - validated : 0; +} + inline std::string NetworkOPsImp::strOperatingMode(bool const admin /* = false */) const { @@ -2115,6 +2175,17 @@ NetworkOPsImp::switchLastClosedLedger(std::shared_ptr const& newLC // set the newLCL as our last closed ledger -- this is abnormal code JLOG(journal_.error()) << "JUMP last closed ledger to " << newLCL->header().hash; + // This node was told the network's last closed ledger is not the one it + // built on, and is discarding its own chain tip to follow. Log-only until + // now, so a node repeatedly thrashing between chains left no time series + // to correlate against the rest of the sync pipeline. Rare event, one + // counter Add, no labels: the ledger hash and sequence would both be + // unbounded as label values, and the log line above already carries them. + XRPL_METRIC_COUNTER_INC( + registry_.get(), + telemetry::metric::ledgerJumpTotal, + "Forced jumps of the last closed ledger to a divergent chain"); + clearNeedNetworkLedger(); // Update fee computations. @@ -2661,13 +2732,28 @@ NetworkOPsImp::setMode(OperatingMode om) if (mode_ == om) return; + // Capture the mode we are leaving before overwriting it: the transition + // edge, not just the destination, is what tells flapping apart from a + // clean climb to FULL. + auto const prevMode = mode_.load(); + mode_ = om; accounting_.mode(om); - // Record state change for OTel dashboard parity counter. - if (auto* mr = registry_.get().getMetricsRegistry()) - mr->incrementStateChanges(); + // Record the mode transition labelled with source and destination, so the + // dashboard can chart which edges of the sync state machine are traversed + // (e.g. repeated full->connected flapping vs. a one-way + // tracking->connected->full climb). Only reached on a real mode change, + // never in a hot loop, and there are only five modes so the label + // cardinality is bounded. strOperatingMode(mode, admin=false) supplies the + // names, which keeps them identical to the ones server_info reports. + XRPL_METRIC_COUNTER_INC_LABELED( + registry_.get(), + telemetry::metric::stateChangesTotal, + "Total operating mode changes", + {{telemetry::label::from, strOperatingMode(prevMode, false)}, + {telemetry::label::to, strOperatingMode(om, false)}}); JLOG(journal_.info()) << "STATE->" << strOperatingMode(); pubServer(); diff --git a/src/xrpld/app/misc/SHAMapStoreImp.cpp b/src/xrpld/app/misc/SHAMapStoreImp.cpp index e41837d2065..9107140592f 100644 --- a/src/xrpld/app/misc/SHAMapStoreImp.cpp +++ b/src/xrpld/app/misc/SHAMapStoreImp.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include @@ -273,6 +275,16 @@ SHAMapStoreImp::copyNode(std::uint64_t& nodeCount, SHAMapTreeNode const& node) dbRotating_->store(NodeObjectType::AccountNode, std::move(s.modData()), hash, 0); JLOG(journal_.warn()) << "copyNode: re-stored node missing from both backends, hash=" << hash << " type=" << static_cast(node.getType()); + // One extra write per rescued node, on top of the whole-state-map walk + // the rotation already performs. Rotation-time writes compete with sync + // I/O, and this branch was warn-log-only, so the volume was invisible + // unless an operator was reading logs. The node hash is deliberately NOT + // a label: it is unbounded runtime data and would mint one series per + // node. Correlate a spike against the log line by node and time. + XRPL_METRIC_COUNTER_INC( + app_, + telemetry::metric::rotationCopyNodeRestoreTotal, + "Nodes re-stored during rotation because they were missing from both backends"); } if ((++nodeCount % checkHealthInterval_) == 0u) { diff --git a/src/xrpld/app/misc/ValidatorList.h b/src/xrpld/app/misc/ValidatorList.h index 3f9039eab86..849ee36c6db 100644 --- a/src/xrpld/app/misc/ValidatorList.h +++ b/src/xrpld/app/misc/ValidatorList.h @@ -713,6 +713,27 @@ class ValidatorList hash_set getTrustedMasterKeys() const; + /** + * get the number of trusted master public keys + * + * Takes the shared (read) lock and returns the size of the trusted + * master key set. O(1): unlike getTrustedMasterKeys() it copies + * nothing, so it is cheap enough to poll periodically. + * + * A count below quorum() means the node can never fully validate a + * ledger, which is the difference between "still syncing" and + * "will never sync". + * + * @par Thread Safety + * + * May be called concurrently, including from a metrics callback + * thread; it only takes the same shared lock as the other getters. + * + * @return the number of trusted master public keys + */ + std::size_t + trustedKeyCount() const; + /** * get the validator list threshold * @return the threshold diff --git a/src/xrpld/app/misc/ValidatorSite.h b/src/xrpld/app/misc/ValidatorSite.h index 7e7ad098ebb..ea3bdd4433d 100644 --- a/src/xrpld/app/misc/ValidatorSite.h +++ b/src/xrpld/app/misc/ValidatorSite.h @@ -18,6 +18,7 @@ #include #include #include +#include #include namespace xrpl { @@ -270,6 +271,36 @@ class ValidatorSite */ bool missingSite(std::scoped_lock const&); + + /** + * Record the terminal outcome of one validator list fetch attempt. + * + * Increments the `unl_fetch_total` counter with a `site` label (the + * URI as configured, so the series is stable across redirects) and an + * `outcome` label. Called exactly once per completed fetch attempt. + * Redirects are not terminal, so no outcome is recorded for them. + * + * A node with no reachable list site never gains trusted keys and can + * never reach quorum, so this counter is the signal that separates + * "misconfigured or unreachable list site" from other sync stalls. + * + * The single call to the metric macro lives here rather than at each + * outcome site because the macro caches its instrument in a + * function-local static, one per call site. + * + * @param siteIdx Index into sites_ of the site that was fetched. + * @param outcome Outcome label, e.g. "accepted", "fetch_error". + * @param sitesLock Proof that sitesMutex_ is held; sites_ is read here. + * + * @note Fetches are periodic (minutes apart), not a hot path, so the + * cost of building the label set per call is irrelevant. No-op when + * telemetry is compiled out or disabled. + */ + void + reportFetchOutcome( + std::size_t siteIdx, + std::string_view outcome, + std::scoped_lock const& sitesLock); }; } // namespace xrpl diff --git a/src/xrpld/app/misc/detail/ValidatorList.cpp b/src/xrpld/app/misc/detail/ValidatorList.cpp index a9e71561581..0d0b9f52413 100644 --- a/src/xrpld/app/misc/detail/ValidatorList.cpp +++ b/src/xrpld/app/misc/detail/ValidatorList.cpp @@ -2078,6 +2078,13 @@ ValidatorList::getTrustedMasterKeys() const return trustedMasterKeys_; } +std::size_t +ValidatorList::trustedKeyCount() const +{ + std::shared_lock const readLock{mutex_}; + return trustedMasterKeys_.size(); +} + std::size_t ValidatorList::getListThreshold() const { diff --git a/src/xrpld/app/misc/detail/ValidatorSite.cpp b/src/xrpld/app/misc/detail/ValidatorSite.cpp index 73ec3e0d6fd..cfd10cbdd0b 100644 --- a/src/xrpld/app/misc/detail/ValidatorSite.cpp +++ b/src/xrpld/app/misc/detail/ValidatorSite.cpp @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include @@ -37,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -380,6 +383,62 @@ ValidatorSite::onTimer(std::size_t siteIdx, error_code const& ec) } } +void +ValidatorSite::reportFetchOutcome( + std::size_t siteIdx, + std::string_view outcome, + std::scoped_lock const& sitesLock) +{ + // loadedResource is set in Site's constructor and never reassigned, so + // it is always non-null. Unlike activeResource (reset once a fetch + // completes) it also keeps the URI exactly as configured, which keeps + // the time series stable when a site redirects. + // + // The label is rebuilt from the parsed parts rather than using the raw + // configured URI. [validator_list_sites] accepts userinfo in the URI and + // ParsedUrl retains it in username/password even though the fetch itself + // never sends it, so the label was the one place a configured + // `https://user:pass@host` could surface -- and from there it would reach + // the collector, Prometheus and every dashboard. + // + // Scheme, host, port and path, with the path truncated at the first '?' or + // '#'. Three details: + // - The port appears only when it differs from the scheme's default. The + // Resource constructor fills in 443/https and 80/http when the config + // omits one, so `pUrl.port` alone cannot say whether an operator asked + // for a port; comparing against the default can. Always printing it + // would rewrite the existing `https://vl.ripple.com` series as + // `https://vl.ripple.com:443/` and break continuity, while always + // dropping it would merge two genuinely different local sites that + // differ only by port. + // - parseUrl's path group is `(/.*)?`, which is greedy to end of string, so + // a query or fragment lands inside `path`. A list URL authenticated by + // `?token=...` would otherwise leak through the label the same way + // userinfo would. + // - parseUrl's host group permits '@', so a malformed URI with two of them + // leaves the tail of the userinfo in `domain`. Cutting at the last '@' + // keeps that out of the label too. + auto const& url = sites_[siteIdx].loadedResource->pUrl; + + std::string host = url.domain; + if (auto const at = host.rfind('@'); at != std::string::npos) + host.erase(0, at + 1); + + std::string siteLabel = url.scheme + "://" + host; + + auto const defaultPort = url.scheme == "https" ? 443 : 80; + if (url.port && *url.port != defaultPort) + siteLabel += ":" + std::to_string(*url.port); + + siteLabel += url.path.substr(0, url.path.find_first_of("?#")); + + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + telemetry::metric::unlFetchTotal, + "Validator list fetch attempts, by site and outcome", + {{telemetry::label::site, siteLabel}, {telemetry::label::outcome, std::string(outcome)}}); +} + void ValidatorSite::parseJsonResponse( std::string const& res, @@ -496,6 +555,9 @@ ValidatorSite::parseJsonResponse( sites_[siteIdx].refreshInterval = refresh; sites_[siteIdx].nextRefresh = clock_type::now() + sites_[siteIdx].refreshInterval; } + + // Last on purpose: if anything above throws, the caller's catch counts it. + reportFetchOutcome(siteIdx, to_string(applyResult.bestDisposition()), sitesLock); } std::shared_ptr @@ -552,7 +614,7 @@ ValidatorSite::onSiteFetch( sites_[siteIdx].lastRequestEndpoint = endpoint; JLOG(j_.debug()) << "Got completion for " << sites_[siteIdx].activeResource->uri << " " << endpoint; - auto onError = [&](std::string const& errMsg, bool retry) { + auto onError = [&](std::string const& errMsg, bool retry, std::string_view outcome) { sites_[siteIdx].lastRefreshStatus.emplace( Site::Status{ .refreshed = clock_type::now(), @@ -560,6 +622,7 @@ ValidatorSite::onSiteFetch( .message = errMsg}); if (retry) sites_[siteIdx].nextRefresh = clock_type::now() + kErrorRetryInterval; + reportFetchOutcome(siteIdx, outcome, lockSites); // See if there's a copy saved locally from last time we // saw the list. @@ -569,7 +632,7 @@ ValidatorSite::onSiteFetch( { JLOG(j_.warn()) << "Problem retrieving from " << sites_[siteIdx].activeResource->uri << " " << endpoint << " " << ec.value() << ":" << ec.message(); - onError("fetch error", true); + onError("fetch error", true, telemetry::lval::unl_fetch::fetchError); } else { @@ -605,14 +668,14 @@ ValidatorSite::onSiteFetch( JLOG(j_.warn()) << "Request for validator list at " << sites_[siteIdx].activeResource->uri << " " << endpoint << " returned bad status: " << res.result_int(); - onError("bad result code", true); + onError("bad result code", true, telemetry::lval::unl_fetch::badStatus); } } } catch (std::exception const& ex) { JLOG(j_.error()) << "Exception in " << __func__ << ": " << ex.what(); - onError(ex.what(), false); + onError(ex.what(), false, telemetry::lval::unl_fetch::parseError); } } sites_[siteIdx].activeResource.reset(); @@ -633,12 +696,15 @@ ValidatorSite::onTextFetch( { std::scoped_lock const lockSites{sitesMutex_}; { + // Both failures share one catch, so the label is set where detected. + std::string_view outcome = telemetry::lval::unl_fetch::parseError; try { if (ec) { JLOG(j_.warn()) << "Problem retrieving from " << sites_[siteIdx].activeResource->uri << " " << ec.value() << ": " << ec.message(); + outcome = telemetry::lval::unl_fetch::fetchError; throw std::runtime_error{"fetch error"}; } @@ -654,6 +720,7 @@ ValidatorSite::onTextFetch( .refreshed = clock_type::now(), .disposition = ListDisposition::Invalid, .message = ex.what()}); + reportFetchOutcome(siteIdx, outcome, lockSites); } sites_[siteIdx].activeResource.reset(); } diff --git a/src/xrpld/overlay/Overlay.h b/src/xrpld/overlay/Overlay.h index 6cc229f5a0c..4b3c442113c 100644 --- a/src/xrpld/overlay/Overlay.h +++ b/src/xrpld/overlay/Overlay.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -31,6 +32,62 @@ class context; namespace xrpl { +/** + * How much of the ledger range this node needs its peer set can serve. + * + * Aggregated from the per-peer ranges advertised in mtSTATUS_CHANGE. The + * distinction this exists to draw: a node that is behind because no connected + * peer holds the sequence it wants is a supply problem to be fixed by changing + * the peer set, while a node whose peers all hold that sequence is a + * throughput problem. Without the aggregate the two look identical. + * + * Example usage -- the supply verdict from a telemetry gauge callback: + * @code + * auto const supply = overlay.getPeerLedgerSupply(validatedSeq); + * if (supply.peersReporting > 0 && supply.peersServingNext == 0) + * // peers are connected, but none holds the next ledger needed + * @endcode + * + * Example usage -- edge case: nothing has advertised a range yet: + * @code + * auto const supply = overlay.getPeerLedgerSupply(validatedSeq); + * if (supply.peersReporting == 0) + * // supplyMinSeq / supplyMaxSeq are 0 and mean "unknown", not "empty" + * @endcode + * + * @note A peer that has not yet sent a status change advertises [0, 0]. Such + * peers are excluded from every field, so `peersReporting` is the + * denominator that makes the other counts readable: zero serving out of + * zero reporting is silence, zero out of many is a genuine gap. + */ +struct PeerLedgerSupply +{ + /** + * Connected peers that have advertised a non-empty ledger range. + */ + std::int64_t peersReporting{0}; + + /** + * Reporting peers whose range covers this node's validated sequence. + */ + std::int64_t peersServingValidated{0}; + + /** + * Reporting peers whose range covers validated + 1, the next one needed. + */ + std::int64_t peersServingNext{0}; + + /** + * Lowest sequence any reporting peer offers; 0 when none report. + */ + std::int64_t supplyMinSeq{0}; + + /** + * Highest sequence any reporting peer offers; 0 when none report. + */ + std::int64_t supplyMaxSeq{0}; +}; + /** * Manages the set of connected peers. */ @@ -242,6 +299,41 @@ class Overlay : public beast::PropertyStream::Source */ [[nodiscard]] virtual json::Value txMetrics() const = 0; + + /** + * Returns how much of the sequence range this node needs its peers can + * actually serve. + * + * Each peer advertises the smallest and largest ledger it holds in + * mtSTATUS_CHANGE, which the connection caches. Those ranges are never + * compared against each other, so "no peer on the network holds the + * ledger I need" is today indistinguishable from "my peers are slow". + * This aggregates them into that answer. + * + * @param validatedSeq This node's validated sequence; the ledger it is + * currently able to serve from. + * @return The supply counts and the sequence window the peer set covers. + * + * @note O(peers), taking the peer-list lock once. Intended for a ~10 s + * telemetry poll, never a per-message path. + */ + [[nodiscard]] virtual PeerLedgerSupply + getPeerLedgerSupply(std::uint32_t validatedSeq) const = 0; + + /** + * Returns PeerFinder slot occupancy and address-cache depth. + * + * Forwarded from PeerFinder, which owns the counts. Exposed on Overlay + * because that is the only handle the rest of the server holds; the + * PeerFinder itself is private to the overlay implementation. + * + * Not `const`: the PeerFinder lock is a plain member, so no method on + * that path can be const. + * + * @return One consistent snapshot of all nine fields. + */ + [[nodiscard]] virtual PeerFinder::SlotCensus + getSlotCensus() = 0; }; } // namespace xrpl diff --git a/src/xrpld/overlay/detail/ConnectAttempt.cpp b/src/xrpld/overlay/detail/ConnectAttempt.cpp index 0f0b3242ded..5dbe05cbbb1 100644 --- a/src/xrpld/overlay/detail/ConnectAttempt.cpp +++ b/src/xrpld/overlay/detail/ConnectAttempt.cpp @@ -6,10 +6,14 @@ #include #include #include +#include #include +#include +#include #include #include +#include #include #include #include @@ -19,6 +23,8 @@ #include #include #include +#include +#include #include #include @@ -36,10 +42,12 @@ #include #include +#include #include #include #include #include +#include #include #include @@ -79,6 +87,15 @@ ConnectAttempt::~ConnectAttempt() if (slot_ != nullptr) overlay_.peerFinder().onClosed(slot_); JLOG(journal_.trace()) << "~ConnectAttempt"; + + // Last resort for an attempt torn down without reaching a terminal path + // (overlay shutdown, or an operation_aborted early return). The span ends + // here with no `outcome`, which is the honest record: the dial really did + // take this long and really did not conclude. Attempts that did conclude + // already ended their span in reportOutcome(), so this is a no-op for them + // -- ~optional on an empty handle. Nothing here can throw: ~SpanGuard is + // noexcept and no attribute is written. + dialSpan_.reset(); } void @@ -99,6 +116,32 @@ ConnectAttempt::stop() void ConnectAttempt::run() { + // Start the dial clock before any async operation is initiated. The + // constructor is too early (it only builds the object; the caller decides + // when to dial), and after setTimer() would be too late: setTimer() already + // queues a strand-bound wait that can read dialStart_ via reportOutcome(). + dialStart_ = std::chrono::steady_clock::now(); + + // Span the dial beside the clock it shares, and for the same reason: the + // constructor is too early and setTimer() already queues a wait that can + // reach reportOutcome(). A fresh trace root -- a dial is the first thing a + // starting node does, so there is nothing to parent it to, and freshRoot() + // stops it inheriting whatever unrelated span happens to be active on the + // thread that decided to dial. + { + using namespace telemetry; + dialSpan_.emplace( + SpanGuard::freshRoot(TraceCategory::Peer, seg::peer, peer_span::op::dial)); + if (*dialSpan_) + { + // Which peer. Deliberately span-only, never a metric label: one + // series per peer address would be unbounded cardinality. + dialSpan_->setAttribute( + peer_span::attr::remoteEndpoint, + to_string(beast::IPAddressConversion::fromAsio(remoteEndpoint_)).c_str()); + } + } + setTimer(); stream_.next_layer().async_connect( @@ -107,6 +150,56 @@ ConnectAttempt::run() strand_, [self = shared_from_this()](error_code const& ec) { self->onConnect(ec); })); } +void +ConnectAttempt::reportOutcome(std::string_view outcome) +{ + if (outcomeReported_) + return; + outcomeReported_ = true; + + // Elapsed time is computed inline, not in a named local, so nothing is + // left unused when the macros compile away. Microseconds are converted to + // fractional milliseconds: `duration` would put a + // comma in a non-variadic macro argument, which the preprocessor splits. + XRPL_METRIC_HISTOGRAM_RECORD( + app_, + telemetry::metric::overlayDialLatencyMs, + "Time from starting an outbound peer dial to its terminal outcome, in milliseconds", + std::chrono::duration_cast( + std::chrono::steady_clock::now() - dialStart_) + .count() / + 1000.0); + + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + telemetry::metric::overlayConnectTotal, + "Outbound peer connection attempts, by terminal outcome", + {{telemetry::label::outcome, std::string(outcome)}}); + + // End the span with the SAME outcome value the counter just recorded, from + // the same funnel, so the two can never disagree. The first-call-wins guard + // above makes this exactly-once at no extra cost. + if (dialSpan_) + { + if (*dialSpan_) + { + using namespace telemetry; + dialSpan_->setAttribute(peer_span::attr::outcome, outcome); + // Same elapsed time the histogram above records, stamped on the + // individual attempt so one slow dial is findable in a trace + // instead of only visible in an aggregate p95. + dialSpan_->setAttribute( + peer_span::attr::durationMs, + static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now() - dialStart_) + .count())); + } + // Unconditional, so the span never leaks even when it was inactive, and + // so the destructor's reset() finds an empty handle. + dialSpan_.reset(); + } +} + //------------------------------------------------------------------------------ void @@ -192,6 +285,7 @@ ConnectAttempt::onTimer(error_code ec) close(); return; } + reportOutcome(telemetry::peer_span::val::timeout); fail("Timeout"); } @@ -205,6 +299,7 @@ ConnectAttempt::onConnect(error_code ec) if (ec == boost::asio::error::operation_aborted) return; + reportOutcome(telemetry::peer_span::val::tcpFail); fail("onConnect", ec); return; } @@ -216,6 +311,7 @@ ConnectAttempt::onConnect(error_code ec) socket_.local_endpoint(ec); if (ec) { + reportOutcome(telemetry::peer_span::val::tcpFail); fail("onConnect", ec); return; } @@ -241,6 +337,7 @@ ConnectAttempt::onHandshake(error_code ec) if (ec == boost::asio::error::operation_aborted) return; + reportOutcome(telemetry::peer_span::val::tlsFail); fail("onHandshake", ec); return; } @@ -248,6 +345,7 @@ ConnectAttempt::onHandshake(error_code ec) auto const localEndpoint = socket_.local_endpoint(ec); if (ec) { + reportOutcome(telemetry::peer_span::val::tlsFail); fail("onHandshake", ec); return; } @@ -255,13 +353,20 @@ ConnectAttempt::onHandshake(error_code ec) if (!overlay_.peerFinder().onConnected( slot_, beast::IPAddressConversion::fromAsio(localEndpoint))) { - fail("Duplicate connection"); + // Not a TLS failure: the handshake succeeded and PeerFinder then + // recognised the remote address as our own. Logic::onConnected has + // exactly one false-returning path and it is the self-connect check + // ("Logic dropping as self connect"), so this branch means we dialled + // ourselves -- a local misconfiguration, not an unreachable peer. + reportOutcome(telemetry::peer_span::val::selfConnection); + fail("Self connection"); return; } auto const sharedValue = makeSharedValue(*streamPtr_, journal_); if (!sharedValue) { + reportOutcome(telemetry::peer_span::val::tlsFail); close(); // makeSharedValue logs return; } @@ -303,6 +408,7 @@ ConnectAttempt::onWrite(error_code ec) if (ec == boost::asio::error::operation_aborted) return; + reportOutcome(telemetry::peer_span::val::upgradeFail); fail("onWrite", ec); return; } @@ -340,6 +446,7 @@ ConnectAttempt::onRead(error_code ec) return; } + reportOutcome(telemetry::peer_span::val::upgradeFail); fail("onRead", ec); return; } @@ -357,8 +464,19 @@ ConnectAttempt::onShutdown(error_code ec) return; } + // A cancelled shutdown is us tearing the attempt down, not the peer failing + // it. Every other handler here guards this; without the same guard an + // ordinary overlay stop was counted as upgrade_fail, inflating that outcome + // on any node that shuts down while dials are in flight. + if (ec == boost::asio::error::operation_aborted) + { + close(); + return; + } + if (ec != boost::asio::error::eof) { + reportOutcome(telemetry::peer_span::val::upgradeFail); fail("onShutdown", ec); return; } @@ -406,10 +524,13 @@ ConnectAttempt::processResponse() } } + // The HTTP-503 block above only harvests redirect hints and falls through, + // so this is the first terminal point for a response we cannot upgrade. if (!OverlayImpl::isPeerUpgrade(response_)) { JLOG(journal_.info()) << "Unable to upgrade to peer protocol: " << response_.result() << " (" << response_.reason() << ")"; + reportOutcome(telemetry::peer_span::val::upgradeFail); close(); return; } @@ -426,6 +547,7 @@ ConnectAttempt::processResponse() if (!negotiatedProtocol) { + reportOutcome(telemetry::peer_span::val::upgradeFail); fail("processResponse: Unable to negotiate protocol version"); return; } @@ -434,6 +556,7 @@ ConnectAttempt::processResponse() auto const sharedValue = makeSharedValue(*streamPtr_, journal_); if (!sharedValue) { + reportOutcome(telemetry::peer_span::val::upgradeFail); close(); // makeSharedValue logs return; } @@ -464,6 +587,7 @@ ConnectAttempt::processResponse() overlay_.peerFinder().activate(slot_, publicKey, static_cast(member)); if (result != PeerFinder::Result::Success) { + reportOutcome(telemetry::peer_span::val::upgradeFail); fail("Outbound " + std::string(to_string(result))); return; } @@ -481,9 +605,14 @@ ConnectAttempt::processResponse() overlay_); overlay_.addActive(peer); + + // Only after addActive succeeds is the dial genuinely complete. If + // anything above threw, the catch below reports the failure instead. + reportOutcome(telemetry::peer_span::val::connected); } catch (std::exception const& e) { + reportOutcome(telemetry::peer_span::val::upgradeFail); fail(std::string("Handshake failure (") + e.what() + ")"); return; } diff --git a/src/xrpld/overlay/detail/ConnectAttempt.h b/src/xrpld/overlay/detail/ConnectAttempt.h index f9ba33571fd..3945ac33f2f 100644 --- a/src/xrpld/overlay/detail/ConnectAttempt.h +++ b/src/xrpld/overlay/detail/ConnectAttempt.h @@ -10,12 +10,15 @@ #include #include #include +#include #include #include #include +#include #include #include +#include namespace xrpl { @@ -52,6 +55,37 @@ class ConnectAttempt : public OverlayImpl::Child, std::shared_ptr slot_; request_type req_; + /** + * When the dial began, set at the top of run() before any async + * operation is started. Base for the `overlay_dial_latency_ms` + * measurement. + */ + std::chrono::steady_clock::time_point dialStart_; + + /** + * True once this attempt's outcome has been reported, so the first + * (most specific) terminal path wins and each attempt is counted at + * most once. Not atomic on purpose -- see reportOutcome(). + */ + bool outcomeReported_{false}; + + /** + * Spans this dial: started in run() beside `dialStart_`, ended by + * reportOutcome() on whichever terminal path the state machine takes, and + * by the destructor for an attempt torn down without one. + * + * The counters WP-A1 added answer "how many dials failed, and at which + * stage". This span answers what they cannot: which peer, and how the time + * was spent inside one slow attempt. Both are needed because a dial that + * hangs is invisible in a rate. + * + * Thread-free (a SpanGuard holds no thread-local scope). All the completion + * handlers that end it are bound through `bind_executor(strand_, ...)`, so + * every access after run() is on one strand; run() itself writes it before + * any async operation is started, so those handlers see it safely. + */ + std::optional dialSpan_; + public: ConnectAttempt( Application& app, @@ -98,6 +132,76 @@ class ConnectAttempt : public OverlayImpl::Child, void processResponse(); + /** + * Record how this outbound dial ended, exactly once per attempt. + * + * Every terminal path in the dial state machine funnels here, so the + * emit code and its cached instruments live in one place instead of + * being repeated per branch. The first call wins: later calls return + * immediately, which keeps the reported outcome the most specific one + * (e.g. a "timeout" is not later overwritten by the "tcp_fail" that + * the cancelled socket operation reports). + * + * Emits: + * - `overlay_dial_latency_ms` histogram, no labels + * - `overlay_connect_total` counter, label `outcome` + * - ends the `peer.dial` span with the same `outcome` value plus + * `remote_endpoint` and `duration_ms` + * + * The span shares this funnel rather than being ended per branch, so the + * span's outcome and the counter's label can never disagree, and the + * first-call-wins guard makes the span exactly-once for free. + * + * Dial state machine and where each outcome is reported: + * + * run() ---- dialStart_ = now + * | + * +-- onTimer ................................. "timeout" + * +-- onConnect (connect / local_endpoint) . "tcp_fail" + * +-- onHandshake + * | +-- TLS handshake / shared value ....... "tls_fail" + * | +-- PeerFinder rejects our own address . "self_connection" + * +-- onWrite / onRead / onShutdown ............. "upgrade_fail" + * +-- processResponse + * +-- bad status / protocol / activate ... "upgrade_fail" + * +-- PeerImp created + addActive ........ "connected" + * + * The slot branch is drawn separately from the TLS one because + * `Logic::onConnected` fails for exactly one reason -- the remote address is + * ours -- and that is a local misconfiguration rather than an unreachable + * peer. + * + * @param outcome One of the `peer_span::val` dial-outcome constants: + * `connected`, `tcpFail`, `tlsFail`, `selfConnection`, `upgradeFail`, + * `timeout`. Taken as a string_view over a compile-time constant, so + * no allocation happens on the caller side. The constants are the + * single source for both the counter label and the span attribute, so + * the two cannot drift apart. + * + * @note Per-connection path: one dial per outbound peer, so this is not + * a hot loop. + * @note Thread safety: every completion handler that calls this is bound + * through `bind_executor(strand_, ...)`, so all callers of this + * method run on the same strand and `outcomeReported_` needs no + * atomic or mutex. `dialStart_` is written once in run(), before + * any async operation is initiated, so it is safely visible to + * those handlers even though run() itself may execute on the + * calling thread rather than the strand. + * @note Known limitation: an attempt torn down by overlay shutdown + * mid-dial (stop() -> close(), or the operation_aborted early + * returns) is deliberately not counted -- it has no network + * outcome to attribute. The span is still ended, by the destructor, + * with no `outcome` attribute: a span whose duration is real but + * whose outcome is absent is exactly what "torn down mid-dial" + * means, and dropping it would instead hide the attempt. + * @note MetricsRegistry is already started when this runs: + * ApplicationImp::setup() calls startTelemetry() before + * ApplicationImp::start() calls overlay_->start(). No-op when + * telemetry is compiled out or disabled at runtime. + */ + void + reportOutcome(std::string_view outcome); + template static boost::asio::ip::tcp::endpoint parseEndpoint(std::string const& s, boost::system::error_code& ec) diff --git a/src/xrpld/overlay/detail/Handshake.cpp b/src/xrpld/overlay/detail/Handshake.cpp index a860d2d6040..74e4bb0b385 100644 --- a/src/xrpld/overlay/detail/Handshake.cpp +++ b/src/xrpld/overlay/detail/Handshake.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include #include @@ -226,6 +228,51 @@ buildHandshake( } } +namespace { + +/** + * Count one handshake negotiation failure, then throw as before. + * + * verifyHandshake() rejects a peer by throwing, from more than a dozen + * distinct branches. Routing every one of them through this helper keeps a + * single emit site -- which matters beyond avoiding duplicated code, because + * the metric macro caches its instrument in a function-local `static`: one + * call site means one instrument shared by all reasons, distinguished only by + * the `reason` label. + * + * Emits `handshake_negotiation_fail_total`, label `reason`. + * + * The thrown message is passed through unchanged so callers, which log + * `e.what()`, see exactly the text they saw before. The reason label is a + * separate, low-cardinality machine-readable value; it is never derived from + * the message. + * + * @param app Provides the metrics registry. + * @param reason Short snake_case cause, e.g. "wrong_network". A string + * literal from a fixed set, so cardinality stays bounded. + * @param message The exact message to throw, byte-identical to the previous + * behaviour. + * + * @note Per-connection handshake path -- one call per rejected peer, not a + * hot loop. + * @note No-op when telemetry is compiled out or disabled at runtime; the + * macro carries that guard, so there is no `#ifdef` here. + * @note Always throws; it never returns to its caller. + */ +[[noreturn]] void +throwNegotiationFailure(Application& app, char const* reason, std::string const& message) +{ + XRPL_METRIC_COUNTER_INC_LABELED( + app, + telemetry::metric::handshakeNegotiationFailTotal, + "Peer handshake negotiations rejected, by reason", + {{telemetry::label::reason, std::string(reason)}}); + + throw std::runtime_error(message); +} + +} // namespace + PublicKey verifyHandshake( boost::beast::http::fields const& headers, @@ -238,7 +285,10 @@ verifyHandshake( if (auto const iter = headers.find("Server-Domain"); iter != headers.end()) { if (!isProperlyFormedTomlDomain(iter->value())) - throw std::runtime_error("Invalid server domain"); + { + throwNegotiationFailure( + app, telemetry::lval::handshake_fail::invalidServerDomain, "Invalid server domain"); + } } if (auto const iter = headers.find("Network-ID"); iter != headers.end()) @@ -246,15 +296,25 @@ verifyHandshake( std::uint32_t nid = 0; if (!beast::lexicalCastChecked(nid, iter->value())) - throw std::runtime_error("Invalid peer network identifier"); + { + throwNegotiationFailure( + app, + telemetry::lval::handshake_fail::invalidNetworkId, + "Invalid peer network identifier"); + } if (networkID && nid != *networkID) - throw std::runtime_error("Peer is on a different network"); + { + throwNegotiationFailure( + app, + telemetry::lval::handshake_fail::wrongNetwork, + "Peer is on a different network"); + } } if (auto const iter = headers.find("Network-Time"); iter != headers.end()) { - auto const netTime = [str = iter->value()]() -> TimeKeeper::time_point { + auto const netTime = [str = iter->value(), &app]() -> TimeKeeper::time_point { TimeKeeper::duration::rep val = 0; if (beast::lexicalCastChecked(val, str)) @@ -262,7 +322,10 @@ verifyHandshake( // It's not an error for the header field to not be present but if // it is present and it contains junk data, that is an error. - throw std::runtime_error("Invalid peer clock timestamp"); + throwNegotiationFailure( + app, + telemetry::lval::handshake_fail::invalidClockTimestamp, + "Invalid peer clock timestamp"); }(); using namespace std::chrono; @@ -282,10 +345,13 @@ verifyHandshake( auto const offset = calculateOffset(netTime, ourTime); if (abs(offset) > tolerance) - throw std::runtime_error("Peer clock is too far off"); + { + throwNegotiationFailure( + app, telemetry::lval::handshake_fail::clockSkew, "Peer clock is too far off"); + } } - PublicKey const publicKey = [&headers] { + PublicKey const publicKey = [&headers, &app] { if (auto const iter = headers.find("Public-Key"); iter != headers.end()) { auto pk = parseBase58(TokenType::NodePublic, iter->value()); @@ -293,13 +359,19 @@ verifyHandshake( if (pk) { if (publicKeyType(*pk) != KeyType::Secp256k1) - throw std::runtime_error("Unsupported public key type"); + { + throwNegotiationFailure( + app, + telemetry::lval::handshake_fail::unsupportedKeyType, + "Unsupported public key type"); + } return *pk; } } - throw std::runtime_error("Bad node public key"); + throwNegotiationFailure( + app, telemetry::lval::handshake_fail::badPublicKey, "Bad node public key"); }(); // This check gets two birds with one stone: @@ -312,16 +384,29 @@ verifyHandshake( auto const iter = headers.find("Session-Signature"); if (iter == headers.end()) - throw std::runtime_error("No session signature specified"); + { + throwNegotiationFailure( + app, + telemetry::lval::handshake_fail::noSessionSignature, + "No session signature specified"); + } auto sig = base64Decode(iter->value()); if (!verifyDigest(publicKey, sharedValue, makeSlice(sig), false)) - throw std::runtime_error("Failed to verify session"); + { + throwNegotiationFailure( + app, + telemetry::lval::handshake_fail::sessionVerifyFailed, + "Failed to verify session"); + } } if (publicKey == app.nodeIdentity().first) - throw std::runtime_error("Self connection"); + { + throwNegotiationFailure( + app, telemetry::lval::handshake_fail::selfConnection, "Self connection"); + } if (auto const iter = headers.find("Local-IP"); iter != headers.end()) { @@ -329,11 +414,16 @@ verifyHandshake( auto const localIp = boost::asio::ip::make_address(std::string_view(iter->value()), ec); if (ec) - throw std::runtime_error("Invalid Local-IP"); + { + throwNegotiationFailure( + app, telemetry::lval::handshake_fail::invalidLocalIp, "Invalid Local-IP"); + } if (beast::IP::isPublic(remote) && remote != localIp) { - throw std::runtime_error( + throwNegotiationFailure( + app, + telemetry::lval::handshake_fail::localIpMismatch, "Incorrect Local-IP: " + remote.to_string() + " instead of " + localIp.to_string()); } } @@ -344,7 +434,10 @@ verifyHandshake( auto const remoteIp = boost::asio::ip::make_address(std::string_view(iter->value()), ec); if (ec) - throw std::runtime_error("Invalid Remote-IP"); + { + throwNegotiationFailure( + app, telemetry::lval::handshake_fail::invalidRemoteIp, "Invalid Remote-IP"); + } if (beast::IP::isPublic(remote) && !beast::IP::isUnspecified(publicIp)) { @@ -352,9 +445,11 @@ verifyHandshake( // from some other IP. if (remoteIp != publicIp) { - throw std::runtime_error( + throwNegotiationFailure( + app, + telemetry::lval::handshake_fail::remoteIpMismatch, "Incorrect Remote-IP: " + publicIp.to_string() + " instead of " + - remoteIp.to_string()); + remoteIp.to_string()); } } } diff --git a/src/xrpld/overlay/detail/OverlayImpl.cpp b/src/xrpld/overlay/detail/OverlayImpl.cpp index f9972548d1a..352b3456f7b 100644 --- a/src/xrpld/overlay/detail/OverlayImpl.cpp +++ b/src/xrpld/overlay/detail/OverlayImpl.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -13,6 +14,8 @@ #include #include #include +#include +#include #include #include @@ -78,6 +81,7 @@ #include #include #include +#include #include #include #include @@ -228,18 +232,26 @@ OverlayImpl::onHandoff( JLOG(journal.debug()) << "Peer connection upgrade from " << remoteEndpoint; + // From here on every exit is a terminal outcome for one inbound peer + // attempt, so each reports exactly once. The two returns above are not + // peer attempts at all (a handled HTTP request, and a request that never + // asked to upgrade), which is why they are not counted. error_code ec; auto const localEndpoint(streamPtr->next_layer().socket().local_endpoint(ec)); if (ec) { JLOG(journal.debug()) << remoteEndpoint << " failed: " << ec.message(); + reportAcceptOutcome(telemetry::lval::peer_accept::localEndpointFail); return handoff; } auto consumer = resourceManager_.newInboundEndpoint(beast::IPAddressConversion::fromAsio(remoteEndpoint)); if (consumer.disconnect(journal)) + { + reportAcceptOutcome(telemetry::lval::peer_accept::resourceLimit); return handoff; + } auto const [slot, result] = peerFinder_->newInboundSlot( beast::IPAddressConversion::fromAsio(localEndpoint), @@ -250,6 +262,7 @@ OverlayImpl::onHandoff( // connection refused either IP limit exceeded or self-connect handoff.moved = false; JLOG(journal.debug()) << "Peer " << remoteEndpoint << " refused, " << to_string(result); + reportAcceptOutcome(telemetry::lval::peer_accept::noSlot); return handoff; } @@ -264,6 +277,7 @@ OverlayImpl::onHandoff( handoff.moved = false; handoff.response = makeRedirectResponse(slot, request, remoteEndpoint.address()); handoff.keepAlive = beast::rfc2616::isKeepAlive(request); + reportAcceptOutcome(telemetry::lval::peer_accept::notPeerRequest); return handoff; } } @@ -276,6 +290,7 @@ OverlayImpl::onHandoff( handoff.response = makeErrorResponse( slot, request, remoteEndpoint.address(), "Unable to agree on a protocol version"); handoff.keepAlive = false; + reportAcceptOutcome(telemetry::lval::peer_accept::protocolMismatch); return handoff; } @@ -287,6 +302,7 @@ OverlayImpl::onHandoff( handoff.response = makeErrorResponse(slot, request, remoteEndpoint.address(), "Incorrect security cookie"); handoff.keepAlive = false; + reportAcceptOutcome(telemetry::lval::peer_accept::badCookie); return handoff; } @@ -316,6 +332,7 @@ OverlayImpl::onHandoff( handoff.moved = false; handoff.response = makeRedirectResponse(slot, request, remoteEndpoint.address()); handoff.keepAlive = false; + reportAcceptOutcome(telemetry::lval::peer_accept::slotRefused); return handoff; } } @@ -345,6 +362,10 @@ OverlayImpl::onHandoff( peer->run(); } handoff.moved = true; + + // Only after run() is the peer genuinely accepted. Anything that threw + // above is reported as a handshake error by the catch below instead. + reportAcceptOutcome(telemetry::lval::peer_accept::accepted); return handoff; } catch (std::exception const& e) @@ -356,6 +377,7 @@ OverlayImpl::onHandoff( handoff.moved = false; handoff.response = makeErrorResponse(slot, request, remoteEndpoint.address(), e.what()); handoff.keepAlive = false; + reportAcceptOutcome(telemetry::lval::peer_accept::handshakeError); return handoff; } } @@ -539,9 +561,16 @@ OverlayImpl::start() bootstrapIps.emplace_back("hub.xrpl-commons.org 51235"); } + // Timestamp the resolve request so the handler can report how long the + // whole batch resolve took for each name it reports back. + auto const bootstrapDnsStart = std::chrono::steady_clock::now(); + resolver_.resolve( bootstrapIps, - [this](std::string const& name, std::vector const& addresses) { + [this, bootstrapDnsStart]( + std::string const& name, std::vector const& addresses) { + reportDnsResolve(bootstrapDnsStart, !addresses.empty()); + std::vector ips; ips.reserve(addresses.size()); for (auto const& addr : addresses) @@ -564,9 +593,14 @@ OverlayImpl::start() // Add the ips_fixed from the xrpld.cfg file if (!app_.config().standalone() && !app_.config().ipsFixed.empty()) { + auto const fixedDnsStart = std::chrono::steady_clock::now(); + resolver_.resolve( app_.config().ipsFixed, - [this](std::string const& name, std::vector const& addresses) { + [this, fixedDnsStart]( + std::string const& name, std::vector const& addresses) { + reportDnsResolve(fixedDnsStart, !addresses.empty()); + std::vector ips; ips.reserve(addresses.size()); @@ -593,6 +627,96 @@ OverlayImpl::start() timer->asyncWait(); } +void +OverlayImpl::reportDnsResolve(std::chrono::steady_clock::time_point start, bool resolved) +{ + // Elapsed time is computed inline, not in a named local, so nothing is + // left unused when the macros compile away. Microseconds are converted to + // fractional milliseconds: `duration` would put a + // comma in a non-variadic macro argument, which the preprocessor splits. + XRPL_METRIC_HISTOGRAM_RECORD( + app_, + telemetry::metric::dnsResolveLatencyMs, + "Time taken to resolve a configured peer hostname, in milliseconds", + std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count() / + 1000.0); + + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + telemetry::metric::dnsResolveTotal, + "Peer hostname resolutions, by outcome", + {{telemetry::label::outcome, + std::string( + resolved ? telemetry::lval::dns_resolve::resolved + : telemetry::lval::dns_resolve::empty)}}); +} + +void +OverlayImpl::reportAcceptOutcome(char const* outcome) +{ + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + telemetry::metric::peerAcceptTotal, + "Inbound peer connection attempts, by terminal outcome", + {{telemetry::label::outcome, std::string(outcome)}}); +} + +PeerLedgerSupply +OverlayImpl::getPeerLedgerSupply(std::uint32_t validatedSeq) const +{ + PeerLedgerSupply supply; + + // Tracked separately from supply.supplyMinSeq so the "nothing reported + // yet" case stays distinguishable: a peer set that genuinely serves from + // sequence 0 and a peer set that has said nothing must not both read 0. + // Only a peer that reported anything can lower this. + auto lowest = std::numeric_limits::max(); + + // The next sequence this node must acquire. Widened before the increment + // so it cannot wrap to 0 at the top of the sequence space, which would + // silently turn "needs the next ledger" into "needs the genesis ledger". + // On a node with no validated ledger yet this is 1, which is correct. + auto const neededSeq = static_cast(validatedSeq) + 1; + + // getActivePeers() takes the overlay lock, copies the list and releases + // it, so the per-peer reads below hold no overlay lock. + for (auto const& peer : getActivePeers()) + { + std::uint32_t minSeq = 0; + std::uint32_t maxSeq = 0; + peer->ledgerRange(minSeq, maxSeq); + + // A peer that has not sent mtSTATUS_CHANGE yet reports [0, 0]. It + // supplies nothing, so it must not be counted as serving and must not + // pull the reported window down to zero. + if (maxSeq == 0) + continue; + + ++supply.peersReporting; + + if (validatedSeq >= minSeq && validatedSeq <= maxSeq) + ++supply.peersServingValidated; + + if (neededSeq >= static_cast(minSeq) && + neededSeq <= static_cast(maxSeq)) + { + ++supply.peersServingNext; + } + + lowest = std::min(lowest, minSeq); + supply.supplyMaxSeq = std::max(supply.supplyMaxSeq, static_cast(maxSeq)); + } + + // Convert the sentinel explicitly. Casting the unsigned max would produce + // 4294967295, which a dashboard would plot as a real sequence. + if (supply.peersReporting > 0) + supply.supplyMinSeq = static_cast(lowest); + + return supply; +} + void OverlayImpl::stop() { diff --git a/src/xrpld/overlay/detail/OverlayImpl.h b/src/xrpld/overlay/detail/OverlayImpl.h index cd2c7d630b1..a538044da68 100644 --- a/src/xrpld/overlay/detail/OverlayImpl.h +++ b/src/xrpld/overlay/detail/OverlayImpl.h @@ -428,6 +428,56 @@ class OverlayImpl : public Overlay, public reduce_relay::SquelchHandler return txMetrics_.json(); } + /** + * Aggregates the per-peer advertised ledger ranges into supply counts. + * + * Iterates the active peers once, reading each one's cached + * `ledgerRange()`. Peers advertising [0, 0] have not reported yet and are + * skipped entirely, so they cannot drag `supplyMinSeq` to zero and make a + * healthy peer set look like it offers history from genesis. + * + * @param validatedSeq This node's validated sequence. + * @return The supply counts and the covered sequence window. + * + * @note O(peers). `getActivePeers()` copies the peer list under the + * overlay lock and releases it before the loop runs, so no peer's + * `ledgerRange()` lock is ever taken while holding the overlay lock. + * @note Called from the ~10 s telemetry gauge callback only. + */ + [[nodiscard]] PeerLedgerSupply + getPeerLedgerSupply(std::uint32_t validatedSeq) const override; + + /** + * Forwards the PeerFinder slot census. + * + * @return One consistent snapshot of all nine slot/cache fields. + */ + [[nodiscard]] PeerFinder::SlotCensus + getSlotCensus() override + { + return peerFinder_->getSlotCensus(); + } + + /** + * Emits the inbound-accept outcome counter for one handoff attempt. + * + * Counts the terminal outcome of every inbound connection this node is + * offered. The outbound twin (`overlay_connect_total`) already exists in + * ConnectAttempt, so this closes the in/out split: without it, a node + * refusing every inbound connection looks the same as one nobody dials. + * + * @param outcome Stable slug for the terminal outcome. Drawn from a fixed + * set of literals at the call sites, never from peer-supplied data, + * so label cardinality is bounded by the code. + * + * @note Cold path: one call per inbound handoff, which is a + * connection-rate event, not a message-rate one. + * @note No-op when telemetry is compiled out or disabled; the macro + * carries that guard, so this needs no `#ifdef`. + */ + void + reportAcceptOutcome(char const* outcome); + /** * Add tx reduce-relay metrics. */ @@ -567,6 +617,41 @@ class OverlayImpl : public Overlay, public reduce_relay::SquelchHandler void deleteIdlePeers(); + /** + * Emit the DNS resolve outcome and latency metrics for one resolved name. + * + * Shared by both `resolver_.resolve(...)` completion handlers in + * OverlayImpl::start() (the bootstrap `[ips]` batch and the `[ips_fixed]` + * batch), so the emit code exists exactly once. This matters beyond + * de-duplication: the metric macros cache their instrument in a + * function-local `static`, so keeping a single call site also keeps a + * single shared instrument for both batches. + * + * Emits: + * - `dns_resolve_latency_ms` histogram, no labels + * - `dns_resolve_total` counter, label `outcome` = "resolved" | "empty" + * + * ResolverAsio invokes its handler with the same signature for success and + * failure; on failure the address list is empty. So an empty list is the + * only failure signal available to the caller, and `resolved` must be + * derived from it. + * + * @param start When the resolve request was submitted. The measured latency + * therefore spans the whole batch resolve for that name, not just + * the final DNS round trip. + * @param resolved True when the resolver returned at least one address. + * + * @note Called once per resolved name during overlay startup only + * (bootstrap plus fixed peers) -- this is not a hot loop. + * @note MetricsRegistry is already started when this runs: + * ApplicationImp::setup() calls startTelemetry() before + * ApplicationImp::start() calls overlay_->start(). + * @note No-op when telemetry is compiled out or disabled at runtime; the + * macro carries that guard, so this method needs no `#ifdef`. + */ + void + reportDnsResolve(std::chrono::steady_clock::time_point start, bool resolved); + private: struct TrafficGauges { diff --git a/src/xrpld/overlay/detail/PeerImp.cpp b/src/xrpld/overlay/detail/PeerImp.cpp index 4a8091520fe..c481943161a 100644 --- a/src/xrpld/overlay/detail/PeerImp.cpp +++ b/src/xrpld/overlay/detail/PeerImp.cpp @@ -1,7 +1,3 @@ -// cspell:ignore ISTOGRAM -// The all-caps macro name XRPL_METRIC_HISTOGRAM_RECORD trips cspell's -// compound-word splitter, which emits the subword "ISTOGRAM"; ignore it here. - #include #include @@ -10,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +22,7 @@ #include #include #include +#include #include #include @@ -39,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -234,7 +233,10 @@ PeerImp::run() closed = parseLedgerHash(iter->value()); if (!closed) + { + self->setDisconnectReason(telemetry::lval::disconnect::malformedHandshake); self->fail("Malformed handshake data (1)"); + } } if (auto const iter = self->headers_.find("Previous-Ledger"); iter != self->headers_.end()) @@ -242,11 +244,17 @@ PeerImp::run() previous = parseLedgerHash(iter->value()); if (!previous) + { + self->setDisconnectReason(telemetry::lval::disconnect::malformedHandshake); self->fail("Malformed handshake data (2)"); + } } if (previous && !closed) + { + self->setDisconnectReason(telemetry::lval::disconnect::malformedHandshake); self->fail("Malformed handshake data (3)"); + } { std::scoped_lock const sl(self->recentLock_); @@ -277,6 +285,9 @@ PeerImp::stop() if (!self->socket_.is_open()) return; + // Overlay-wide shutdown, not a fault with this peer. Distinguished so + // a clean restart does not look like a wave of peer failures. + self->setDisconnectReason(telemetry::lval::disconnect::stopping); self->close(); }); } @@ -403,6 +414,10 @@ PeerImp::charge(Resource::Charge const& fee, std::string const& context) expected, true, std::memory_order_acq_rel)) { self->overlay_.incPeerDisconnectCharges(); + // Set inside the latch, so only the one worker that wins the + // exchange writes it. This is the node's own backpressure, not + // a peer or network fault. + self->setDisconnectReason(telemetry::lval::disconnect::chargeResources); self->fail("charge: Resources"); } } @@ -628,9 +643,36 @@ PeerImp::close() socket_.close(ec); // NOLINT(bugprone-unused-return-value) overlay_.incPeerDisconnect(); + + // Emitted right next to incPeerDisconnect() above, and behind the same + // socket-already-closed early return, so this counter's total tracks that + // existing tally rather than being a second, differently-scoped count. + // What it adds is the split: today every disconnect collapses into one + // number, so our-fault backpressure ("large_sendq", "charge_resources") + // cannot be told apart from a topology or network fault ("not_useful", + // "ping_timeout", "read_error"), and the two need opposite responses. + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + telemetry::metric::peerDisconnectTotal, + "Peer disconnects, by cause and connection direction", + {{telemetry::label::reason, std::string(disconnectReason_)}, + {telemetry::label::direction, + std::string(inbound_ ? telemetry::lval::inbound : telemetry::lval::outbound)}}); + JLOG((inbound_ ? journal_.debug() : journal_.info())) << "close: Closed"; } +void +PeerImp::reportServeRefusal(char const* request, char const* reason) +{ + XRPL_METRIC_COUNTER_INC_LABELED( + app_, + telemetry::metric::serveRefusedTotal, + "Peer data requests this node declined to serve, by request kind and cause", + {{telemetry::label::request, std::string(request)}, + {telemetry::label::reason, std::string(reason)}}); +} + void PeerImp::fail(std::string const& reason) { @@ -724,12 +766,16 @@ PeerImp::onTimer(error_code const& ec) // This should never happen JLOG(journal_.error()) << "onTimer: " << ec.message(); + setDisconnectReason(telemetry::lval::disconnect::timerError); close(); return; } if (largeSendq_++ >= Tuning::kSendqIntervals) { + // Our own send queue never drained: this node could not keep up with + // what it owed the peer, so it is local backpressure, not a peer fault. + setDisconnectReason(telemetry::lval::disconnect::largeSendq); fail("Large send queue"); return; } @@ -747,6 +793,9 @@ PeerImp::onTimer(error_code const& ec) (t == Tracking::Unknown && (duration > app_.config().maxUnknownTime))) { overlay_.peerFinder().onFailure(slot_); + // The peer is on a different chain, or we cannot tell: a topology + // signal, not a fault on either side. + setDisconnectReason(telemetry::lval::disconnect::notUseful); fail("Not useful"); return; } @@ -755,6 +804,7 @@ PeerImp::onTimer(error_code const& ec) // Already waiting for PONG if (lastPingSeq_) { + setDisconnectReason(telemetry::lval::disconnect::pingTimeout); fail("Ping Timeout"); return; } @@ -793,6 +843,10 @@ PeerImp::onShutdown(error_code ec) } } + // The TLS shutdown handshake finished. First-wins means the reason set by + // whoever asked for the graceful close is kept; "shutdown" only lands when + // the teardown started here, i.e. a clean close with no earlier cause. + setDisconnectReason(telemetry::lval::disconnect::shutdown); close(); } @@ -808,6 +862,7 @@ PeerImp::doAccept() // the shared value successfully in OverlayImpl if (!sharedValue) { + setDisconnectReason(telemetry::lval::disconnect::sharedValue); fail("makeSharedValue: Unexpected failure"); return; } @@ -857,6 +912,7 @@ PeerImp::doAccept() if (ec == boost::asio::error::operation_aborted) return; + setDisconnectReason(telemetry::lval::disconnect::writeError); fail("onWriteResponse", ec); return; } @@ -866,6 +922,7 @@ PeerImp::doAccept() doProtocolStart(); return; } + setDisconnectReason(telemetry::lval::disconnect::writeError); fail("Failed to write header"); return; })); @@ -940,10 +997,14 @@ PeerImp::onReadMessage(error_code ec, std::size_t bytesTransferred) if (ec == boost::asio::error::eof) { JLOG(journal_.info()) << "EOF"; + // The peer closed its side cleanly. Counted apart from a read + // error because it is normal peer churn, not a fault. + setDisconnectReason(telemetry::lval::disconnect::graceful); gracefulClose(); return; } + setDisconnectReason(telemetry::lval::disconnect::readError); fail("onReadMessage", ec); return; } @@ -973,6 +1034,7 @@ PeerImp::onReadMessage(error_code ec, std::size_t bytesTransferred) if (ec) { + setDisconnectReason(telemetry::lval::disconnect::readError); fail("onReadMessage", ec); return; } @@ -1009,6 +1071,7 @@ PeerImp::onWriteMessage(error_code ec, std::size_t bytesTransferred) if (ec == boost::asio::error::operation_aborted) return; + setDisconnectReason(telemetry::lval::disconnect::writeError); fail("onWriteMessage", ec); return; } @@ -2542,6 +2605,8 @@ PeerImp::onMessage(std::shared_ptr const& m) if (sendQueue_.size() >= Tuning::kDropSendQueue) { JLOG(pJournal_.debug()) << "GetObject: Large send queue"; + reportServeRefusal( + telemetry::lval::serve_request::object, telemetry::lval::serve_refused::sendqFull); return; } @@ -2967,6 +3032,11 @@ PeerImp::doFetchPack(std::shared_ptr const& packet) (app_.getJobQueue().getJobCount(JtPack) > 10)) { JLOG(pJournal_.info()) << "Too busy to make fetch pack"; + // A fetch pack is how a syncing peer catches up in bulk, so refusing + // one directly slows that peer's sync. Counted separately from the + // ledger path because the shed threshold is a different one. + reportServeRefusal( + telemetry::lval::serve_request::fetchpack, telemetry::lval::serve_refused::loadShed); return; } @@ -3466,6 +3536,57 @@ PeerImp::getTxSet(std::shared_ptr const& m) const return shaMap; } +telemetry::SpanGuard +PeerImp::startServeSpan(int itype) const +{ + // A fresh trace root: the request arrives from the wire and is handled on a + // shared JtLedgerReq worker, so freshRoot() stops it inheriting whatever + // unrelated span happens to be active on that worker. + auto span = telemetry::SpanGuard::freshRoot( + telemetry::TraceCategory::Ledger, + telemetry::seg::ledger, + telemetry::ledger_span::op::serve); + if (span) + { + // Which of the four request kinds, from the shared rule so no exit can + // disagree with another about the same request. + span.setAttribute( + telemetry::ledger_span::attr::objectType, + telemetry::ledger_span::serveObjectType(itype)); + // Which peer we are serving. Span-only: one metric series per peer id + // would be unbounded, which is why the paired serve_refused_total + // counter deliberately carries no peer label either. + span.setAttribute(telemetry::peer_span::attr::peerId, static_cast(id())); + } + return span; +} + +void +PeerImp::finishServeSpan( + telemetry::SpanGuard& span, + protocol::TMLedgerData const& ledgerData) noexcept +{ + if (!span) + return; + using namespace telemetry; + // `ledgerData` IS the reply, so its node count is the served-node count and + // is 0 on every refusal path. Reading it here rather than accumulating in + // the assembly loop is what keeps the per-node path untouched, and is also + // what lets one rule cover all eight exits. + auto const served = ledgerData.nodes_size(); + span.setAttribute(ledger_span::attr::servedNodes, static_cast(served)); + span.setAttribute( + ledger_span::attr::outcome, ledger_span::serveOutcome(served, Tuning::kSoftMaxReplyNodes)); + // Which ledger was served. Known only once getLedger() succeeded, so it is + // read here rather than at span start. Span-only: a per-ledger value as a + // metric dimension would mint one series per ledger. + if (ledgerData.has_ledgerseq() && ledgerData.ledgerseq() != 0) + { + span.setAttribute( + ledger_span::attr::ledgerSeq, static_cast(ledgerData.ledgerseq())); + } +} + void PeerImp::processLedgerRequest(std::shared_ptr const& m) { @@ -3480,10 +3601,21 @@ PeerImp::processLedgerRequest(std::shared_ptr const& m) bool fatLeaves{true}; auto const itype{m->itype()}; + // Span this serve, and stamp its result on whichever of the eight exits + // below is taken. The scope-exit guard is what makes that exactly-once + // without repeating an emit per branch, and without any exit -- including + // one added later -- being able to forget. + auto serveSpan = startServeSpan(itype); + ScopeExit const finalizeServeSpan{ + [&serveSpan, &ledgerData]() noexcept { PeerImp::finishServeSpan(serveSpan, ledgerData); }}; + if (itype == protocol::liTS_CANDIDATE) { if (sharedMap = getTxSet(m); !sharedMap) + { + reportServeRefusal(telemetry::lval::serve_request::txset, telemetry::lval::notFound); return; + } map = sharedMap.get(); // Fill out the reply @@ -3501,16 +3633,23 @@ PeerImp::processLedgerRequest(std::shared_ptr const& m) if (sendQueue_.size() >= Tuning::kDropSendQueue) { JLOG(pJournal_.debug()) << "processLedgerRequest: Large send queue"; + reportServeRefusal( + telemetry::lval::serve_request::ledger, telemetry::lval::serve_refused::sendqFull); return; } if (app_.getFeeTrack().isLoadedLocal() && !cluster()) { JLOG(pJournal_.debug()) << "processLedgerRequest: Too busy"; + reportServeRefusal( + telemetry::lval::serve_request::ledger, telemetry::lval::serve_refused::loadShed); return; } if (ledger = getLedger(m); !ledger) + { + reportServeRefusal(telemetry::lval::serve_request::ledger, telemetry::lval::notFound); return; + } // Fill out the reply auto const ledgerHash{ledger->header().hash}; @@ -3541,6 +3680,9 @@ PeerImp::processLedgerRequest(std::shared_ptr const& m) default: // This case should not be possible here JLOG(pJournal_.error()) << "processLedgerRequest: Invalid ledger info type"; + reportServeRefusal( + telemetry::lval::serve_request::ledger, + telemetry::lval::serve_refused::badType); return; } } @@ -3548,6 +3690,8 @@ PeerImp::processLedgerRequest(std::shared_ptr const& m) if (map == nullptr) { JLOG(pJournal_.warn()) << "processLedgerRequest: Unable to find map"; + reportServeRefusal( + telemetry::lval::serve_request::ledger, telemetry::lval::serve_refused::noMap); return; } @@ -3632,7 +3776,16 @@ PeerImp::processLedgerRequest(std::shared_ptr const& m) } if (ledgerData.nodes_size() == 0) + { + // The map was found but produced no nodes to return, so the requester + // gets nothing back and will have to ask someone else. Emitted here, + // after the node loop, rather than inside it -- one call per request. + reportServeRefusal( + itype == protocol::liTS_CANDIDATE ? telemetry::lval::serve_request::txset + : telemetry::lval::serve_request::ledger, + telemetry::lval::serve_refused::emptyReply); return; + } send(std::make_shared(ledgerData, protocol::mtLEDGER_DATA)); } diff --git a/src/xrpld/overlay/detail/PeerImp.h b/src/xrpld/overlay/detail/PeerImp.h index 34f556cae3a..7897ac02c72 100644 --- a/src/xrpld/overlay/detail/PeerImp.h +++ b/src/xrpld/overlay/detail/PeerImp.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -32,6 +33,7 @@ #include #include #include +#include #include #include @@ -52,6 +54,7 @@ #include #include #include +#include #include #include #include @@ -186,6 +189,30 @@ class PeerImp : public Peer, public std::enable_shared_from_this, publi // post duplicate fail() calls) when several queued requests cross // kDropThreshold before the first fail() lands on the strand. std::atomic chargeDisconnectFired_{false}; + + /** + * Why this connection is being torn down, as a stable slug for the + * `peer_disconnect_total` counter's `reason` label. + * + * Set by the site that decides to disconnect and read once by close(), + * which is the single funnel every teardown passes through. Recording at + * close() rather than at each decision site is what makes the count equal + * the real disconnect count: several sites call fail() and then close(), + * and close() itself already self-guards on the socket being open. + * + * First writer wins, so a reason is never overwritten by a later, less + * specific one on the same teardown. Defaults to "unknown" so a path that + * reaches close() without setting a reason still produces a series rather + * than vanishing. + * + * @note Only ever touched on the strand (or, for the charge path, behind + * the chargeDisconnectFired_ latch), so it needs no lock. + * @note The value is always one of a fixed set of literals in this file -- + * never peer-supplied data -- so the label's cardinality is bounded + * by the code. + */ + char const* disconnectReason_{telemetry::lval::disconnect::unknown}; + std::shared_ptr const slot_; boost::beast::multi_buffer readBuffer_; http_request_type request_; @@ -470,6 +497,46 @@ class PeerImp : public Peer, public std::enable_shared_from_this, publi } private: + /** + * Records why this connection is closing, first writer wins. + * + * @param reason Stable slug from the fixed set used in PeerImp.cpp. + * + * @note Not a metric emit. close() does the single emit per teardown; this + * only carries the cause to it, so a site that decides to disconnect + * and a site that performs it stay separate. + */ + void + setDisconnectReason(char const* reason) noexcept + { + // Only the first cause is kept: fail() sites frequently run before + // close(), and a later generic reason must not mask the real one. + if (disconnectReason_ == std::string_view{telemetry::lval::disconnect::unknown}) + disconnectReason_ = reason; + } + + /** + * Emits the serve-refusal counter for one request this node would not + * answer. + * + * This is the supply side of the sync exchange: what this node refuses to + * serve its peers. Nothing measured it before, so a node shedding every + * ledger request looked identical to one being asked for nothing. + * + * @param request Which request kind was refused: "ledger", "txset", + * "object" or "fetchpack". + * @param reason Why it was refused, from the fixed slug set in PeerImp.cpp. + * + * @note Cold relative to the message loop: one call per refused request, + * and never inside the per-tree-node loop of processLedgerRequest. + * @note Both labels are code literals, never peer-supplied data, so + * cardinality is bounded at compile time. + * @note No-op when telemetry is compiled out or disabled; the macro + * carries that guard. + */ + void + reportServeRefusal(char const* request, char const* reason); + void close(); @@ -681,6 +748,49 @@ class PeerImp : public Peer, public std::enable_shared_from_this, publi void processLedgerRequest(std::shared_ptr const& m); + /** + * Start the `ledger.serve` span for one incoming ledger-data request. + * + * This is the supply side of somebody else's sync: this worker is what a + * syncing peer waits on, and nothing measured how long it takes or whether + * it produced anything at all. A fresh trace root, because the request + * arrives from the wire on a shared `JtLedgerReq` worker whose ambient span + * is unrelated. + * + * @param itype The protobuf `TMLedgerInfoType` the peer requested, used to + * stamp `object_type`. Taken as an int so the shared naming rule + * needs no protobuf dependency. + * @return An active span guard, or a null guard when telemetry is disabled + * or the ledger trace category is off. + * + * @note One call per request, never inside the per-tree-node reply loop. + */ + [[nodiscard]] telemetry::SpanGuard + startServeSpan(int itype) const; + + /** + * End the `ledger.serve` span, stamping what the request produced. + * + * Called from a scope-exit guard in processLedgerRequest, which is what + * makes it exactly-once across that function's eight exits without + * repeating an emit per branch — and what stops an exit added later from + * silently forgetting to record one. + * + * Both recorded values are READ from state that already exists at every + * exit rather than accumulated: `ledgerData` is the reply itself, so its + * node count is the served-node count and is zero on all seven refusal + * paths. The per-node assembly loop is therefore untouched. + * + * @param span The span to finish; a null guard is a no-op. + * @param ledgerData The reply as assembled so far, empty on a refusal. + * + * @note noexcept: it runs from a scope-exit guard, so it must not throw + * even when the function body leaves by exception. Every call it + * makes is itself noexcept. + */ + static void + finishServeSpan(telemetry::SpanGuard& span, protocol::TMLedgerData const& ledgerData) noexcept; + /** * Record the OTel metrics for one completed `TMGetObjectByHash` request. * diff --git a/src/xrpld/overlay/detail/PeerSpanNames.h b/src/xrpld/overlay/detail/PeerSpanNames.h index 6212d8fca25..535424ff457 100644 --- a/src/xrpld/overlay/detail/PeerSpanNames.h +++ b/src/xrpld/overlay/detail/PeerSpanNames.h @@ -4,12 +4,27 @@ * Compile-time span name constants for peer overlay tracing. * * Used by PeerImp for peer message handling spans (proposals, - * validations). Built on StaticStr/join() from SpanNames.h. + * validations) and by ConnectAttempt for the outbound dial span. + * Built on StaticStr/join() from SpanNames.h. * * Span hierarchy: * * peer.proposal.receive (PeerImp — incoming proposal) * peer.validation.receive (PeerImp — incoming validation) + * peer.dial (ConnectAttempt — outbound connect attempt) + * + * peer.dial is a trace root: it is the first thing a fresh node does, so + * nothing exists yet to parent it to. + * + * +---------------+ starts +----------------------------+ + * | ConnectAttempt|---------->| span "peer.dial" | + * | ::run() | | outcome / remote_endpoint | + * +---------------+ | duration_ms | + * | +----------------------------+ + * | one terminal path ends it (reportOutcome) + * v + * onTimer / onConnect / onHandshake / onWrite / onRead / + * onShutdown / processResponse */ #include @@ -21,6 +36,7 @@ namespace xrpl::telemetry::peer_span { namespace op { inline constexpr auto proposalReceive = makeStr("proposal.receive"); inline constexpr auto validationReceive = makeStr("validation.receive"); +inline constexpr auto dial = makeStr("dial"); } // namespace op // ===== Attribute keys ======================================================== @@ -40,6 +56,60 @@ using ::xrpl::telemetry::attr::peerId; */ inline constexpr auto proposalTrusted = makeStr("proposal_trusted"); inline constexpr auto validationTrusted = makeStr("validation_trusted"); + +/** + * peer.dial attrs (outbound connect attempt). + * + * `outcome` is the same terminal-reason set the `overlay_connect_total` + * counter already labels with, so the span and the counter can be read + * against each other. `remoteEndpoint` says WHICH peer, which the counter + * deliberately cannot carry: one series per peer address would be unbounded + * cardinality, so it stays span-only and Tempo-searchable instead. + * `durationMs` mirrors the `overlay_dial_latency_ms` histogram value onto the + * individual attempt, so one slow dial is findable rather than only visible + * in an aggregate p95. + */ +inline constexpr auto remoteEndpoint = makeStr("remote_endpoint"); +inline constexpr auto durationMs = makeStr("duration_ms"); +inline constexpr auto outcome = makeStr("outcome"); } // namespace attr +// ===== Attribute values ====================================================== + +namespace val { +/** + * peer.dial outcome values. + * + * The identical slugs `ConnectAttempt::reportOutcome` passes to the + * `overlay_connect_total` counter, defined here so the span and the counter + * cannot drift apart: the dial state machine names its outcome once and both + * signals receive that same value. + * + * - connected: the peer was activated and added to the overlay. + * - tcp_fail: the TCP connect or local-endpoint read failed. + * - tls_fail: the TLS handshake, or the shared-value read taken before + * the HTTP upgrade, failed. + * - self_connection: TLS succeeded and then PeerFinder recognised the remote + * address as one of our own, so we had dialled ourselves. + * - upgrade_fail: TLS succeeded but the HTTP upgrade, protocol negotiation + * or activation was rejected. + * - timeout: the attempt never reached any terminal state in time. + * + * `self_connection` is separate from `tls_fail` because it is a local + * misconfiguration, not an unreachable peer: the node has its own address in + * `[ips_fixed]` or behind its advertised endpoint, and every dial to it is + * wasted. Counting it as a TLS failure made a rising `tls_fail` unreadable -- + * broken peers and a self-dial loop need completely different responses. The + * slug matches `handshake_fail::selfConnection` on + * `handshake_negotiation_fail_total`, so the same fault reads the same way + * whichever signal surfaces it. + */ +inline constexpr auto connected = makeStr("connected"); +inline constexpr auto tcpFail = makeStr("tcp_fail"); +inline constexpr auto tlsFail = makeStr("tls_fail"); +inline constexpr auto selfConnection = makeStr("self_connection"); +inline constexpr auto upgradeFail = makeStr("upgrade_fail"); +inline constexpr auto timeout = makeStr("timeout"); +} // namespace val + } // namespace xrpl::telemetry::peer_span diff --git a/src/xrpld/telemetry/MetricMacros.h b/src/xrpld/telemetry/MetricMacros.h index 6bea92d8e3b..0390a441674 100644 --- a/src/xrpld/telemetry/MetricMacros.h +++ b/src/xrpld/telemetry/MetricMacros.h @@ -1,9 +1,5 @@ #pragma once -// cspell:ignore ISTOGRAM -// The all-caps macro name XRPL_METRIC_HISTOGRAM_RECORD trips cspell's -// compound-word splitter, which emits the subword "ISTOGRAM"; ignore it here. - /** * Call-site OTel metric macros. * diff --git a/src/xrpld/telemetry/MetricNames.h b/src/xrpld/telemetry/MetricNames.h new file mode 100644 index 00000000000..6eb64bc7a98 --- /dev/null +++ b/src/xrpld/telemetry/MetricNames.h @@ -0,0 +1,750 @@ +#pragma once + +/** + * Compile-time OTel metric name constants for the sync-diagnostics signals. + * + * The metric-side counterpart of the `*SpanNames.h` headers: one constant per + * emitted string, so a rename is one edit and a typo is a compile error rather + * than a metric that silently never appears. Each instrument name, label key + * and bounded label value added by the sync-diagnostics work is declared here + * exactly once and referenced from every C++ user of it -- the emit site, the + * gauge registration in MetricsRegistry.cpp, and the unit test. + * + * Layer map -- who references these constants: + * + * +-------------------------------------------------------------+ + * | MetricNames.h (this file, L1-metrics) | + * | namespace metric namespace label namespace lval | + * +-------------------------------------------------------------+ + * ^ ^ ^ + * | | | + * +--------------+ +-------------------+ +---------------------+ + * | emit sites | | MetricsRegistry | | unit test | + * | XRPL_METRIC_ | | Create*Gauge + | | tests/libxrpl/ | + * | macros under | | AddCallback | | telemetry/ | + * | src/xrpld/ | | observe(...) | | MetricMacros.cpp | + * +--------------+ +-------------------+ +---------------------+ + * + * Layers that CANNOT reference a C++ constant -- the collector config, the + * dashboard PromQL, `expected_metrics.json` and the runbook -- are held to + * these same strings by `.github/scripts/otel-naming/check_otel_naming.py` + * instead. + * + * Where this is documented: + * - CONTRIBUTING.md -> "Telemetry metric naming" is the authoritative rule + * list, alongside the sibling span-attribute convention. + * - `.github/scripts/otel-naming/README.md` describes the enforcing rules + * I (no literals), J (suffix conventions) and K (expected_metrics.json). + * - docs/telemetry-runbook.md -> "Adding a New Metric" is the walkthrough for + * adding one, and shows the declare-then-emit pattern. + * + * Naming rules (enforced by the checker's Rule J): + * - Bare `lower_snake_case`. No `xrpld_` prefix in code: the Prometheus + * exporter adds the namespace prefix itself, so writing it here would + * produce `xrpld_xrpld_*` on the wire. + * - A monotonic counter ends in `_total`, so `rate()` over it reads correctly + * and a reader can tell it from a gauge at a glance. + * - A duration carries its unit as the suffix: `_us`, `_ms` or `_seconds`. + * The unit belongs in the name because the OTel `unit` argument is not + * surfaced on the Prometheus metric name. + * - A gauge that is a snapshot of current state takes no suffix + * (`jobq_saturation`, `sync_state`). + * - Label VALUES are declared here only when they come from a fixed set that + * the code itself writes (`namespace lval`), which is what keeps series + * cardinality bounded. A value derived from runtime data -- a site URI, a + * peer address, a ledger hash -- is deliberately NOT declared here and must + * never become a label on a metric. + * + * Why `constexpr char[]` and not the `makeStr`/`StaticStr` DSL that + * `SpanNames.h` uses: the OTel C++ API takes `nostd::string_view`, which in + * this build (`OPENTELEMETRY_STL_VERSION` unset, so the back-ported class is + * used) constructs only from `char const*`, `std::string` or + * `(char const*, size)`. It has NO constructor from `std::string_view`, so + * neither `StaticStr` (which converts to `std::string_view`) nor a + * `constexpr std::string_view` compiles in an instrument-name or label-key + * position -- both were tried and both fail with "no viable conversion". + * A `constexpr char[]` decays to `char const*` and binds directly. This also + * matches the precedent already in MetricsRegistry.cpp + * (`kJobQueuedDurationUs`), which this header absorbs. + * + * Example usage -- a labelled counter at an emit site: + * @code + * XRPL_METRIC_COUNTER_INC_LABELED( + * app_, + * metric::dnsResolveTotal, + * "Peer hostname resolutions, by outcome", + * {{label::outcome, + * std::string( + * resolved ? lval::dns_resolve::resolved : lval::dns_resolve::empty)}}); + * @endcode + * + * Example usage -- an observable gauge and its sub-metric discriminators: + * @code + * syncStateGauge_ = meter_->CreateInt64ObservableGauge( + * metric::syncState, "Sync-pipeline health signals"); + * // ... inside the callback: + * observe(lval::sync_state::ledgersBehind, ops.getLedgersBehindNetwork()); + * @endcode + * + * Example usage -- edge case: a value that must NOT be a constant. The site + * URI is runtime data, so only the KEY is named here; declaring the value + * would imply a bounded set that does not exist. Note the value is built from + * the PARSED url, not the configured string: the config accepts userinfo and a + * query, and either would otherwise ride the label into Prometheus. + * @code + * auto const& url = sites_[siteIdx].loadedResource->pUrl; + * std::string site = url.scheme + "://" + url.domain; + * site += url.path.substr(0, url.path.find_first_of("?#")); + * XRPL_METRIC_COUNTER_INC_LABELED( + * app_, metric::unlFetchTotal, "...", + * {{label::site, site}, {label::outcome, std::string(outcome)}}); + * @endcode + * + * @note Header-only and dependency-free: nothing here includes an OTel or an + * xrpld header, so `src/tests/libxrpl/telemetry/MetricMacros.cpp` can + * include it even though `xrpl_tests` links only `xrpl.libxrpl`. The + * constants are `inline constexpr`, so they contribute no symbol to link + * against. + * @note Not guarded by `XRPL_ENABLE_TELEMETRY`, for the same reason + * `SpanNames.h` is not: call sites name these constants even where the + * macros expand to no-ops, and the compiler elides any constant whose + * only uses are in dead code. + * @note Every constant is a compile-time value with no mutable state, so all + * of them are safe to read concurrently from any thread. + */ + +namespace xrpl::telemetry { + +/** + * Instrument names -- the metric name as it reaches the OTel meter. + * + * Grouped by the subsystem that emits them, matching how the sync-diagnostics + * work was staged. A name is declared here whether it is created lazily by an + * `XRPL_METRIC_*` macro at a call site or eagerly by a `meter_->Create*` call + * in MetricsRegistry.cpp, because the dashboards cannot tell the two apart. + */ +namespace metric { + +// ===== Bootstrap: getting a fresh node its first peers and its first UNL ===== + +/** + * Time to resolve one configured peer hostname. + */ +inline constexpr char dnsResolveLatencyMs[] = "dns_resolve_latency_ms"; +/** + * Peer hostname resolutions, split by whether any address came back. + */ +inline constexpr char dnsResolveTotal[] = "dns_resolve_total"; +/** + * Time from starting an outbound dial to its terminal outcome. + */ +inline constexpr char overlayDialLatencyMs[] = "overlay_dial_latency_ms"; +/** + * Outbound peer connection attempts, by terminal outcome. + */ +inline constexpr char overlayConnectTotal[] = "overlay_connect_total"; +/** + * Peer handshakes this node rejected, by reason. + */ +inline constexpr char handshakeNegotiationFailTotal[] = "handshake_negotiation_fail_total"; +/** + * Validator-list fetch attempts, by site and outcome. + */ +inline constexpr char unlFetchTotal[] = "unl_fetch_total"; +/** + * Trusted UNL key count against the required quorum. + * + * A gauge, not a counter: both series are current state, and the useful read + * is the difference between them. + */ +inline constexpr char unlQuorum[] = "unl_quorum"; +/** + * Network close-time offset from the local clock. + */ +inline constexpr char clockCloseOffsetSeconds[] = "clock_close_offset_seconds"; + +// ===== Sync state: why this node is not FULL yet ============================= + +/** + * Operating-mode transitions, labelled with the mode pair. + */ +inline constexpr char stateChangesTotal[] = "state_changes_total"; +/** + * Four sync-pipeline health signals, split by the `metric` label. + */ +inline constexpr char syncState[] = "sync_state"; +/** + * Main-loop stall episodes. Cumulative, so `rate()` gives episodes/sec. + */ +inline constexpr char serverStallEventsTotal[] = "server_stall_events_total"; + +// ===== Acquire / SHAMap: is ledger data actually arriving? =================== + +/** + * Aggregate ledger-acquire progress across all in-flight acquires. + */ +inline constexpr char syncAcquire[] = "sync_acquire"; +/** + * SHAMap tree-node cache hit rate, 0.0-1.0. + */ +inline constexpr char shamapCacheHitRate[] = "shamap_cache_hit_rate"; +/** + * Ledger acquires by where the data came from (local store vs network). + */ +inline constexpr char syncAcquireSourceTotal[] = "sync_acquire_source_total"; +/** + * Acquire timeouts where not one new node arrived. + */ +inline constexpr char syncAcquireNoProgressTotal[] = "sync_acquire_no_progress_total"; +/** + * SHAMap nodes received during an acquire, by per-node outcome. + */ +inline constexpr char syncAddnodeTotal[] = "sync_addnode_total"; + +// ===== JobQueue: is the worker pool the bottleneck? ========================== + +/** + * Worker-pool saturation: tasks in flight, threads, and jobs queued. + */ +inline constexpr char jobqSaturation[] = "jobq_saturation"; + +// ===== Quorum and publish: can this node accept and publish a ledger? ======= + +/** + * Pre-accept quorum gate and publish lag, split by the `metric` label. + */ +inline constexpr char ledgerQuorumPublish[] = "ledger_quorum_publish"; +/** + * Pre-accept gate rejections for being below quorum. + */ +inline constexpr char ledgerQuorumShortfallTotal[] = "ledger_quorum_shortfall_total"; + +// ===== Back-fill persistence: is history repair making progress? ============= + +/** + * Replay sub-acquires that fell back to a full ledger acquire, by stage. + */ +inline constexpr char ledgerReplayFallbackTotal[] = "ledger_replay_fallback_total"; +/** + * Ledger replay tasks by terminal outcome. + */ +inline constexpr char ledgerReplayOutcomeTotal[] = "ledger_replay_outcome_total"; +/** + * Forced jumps of the last closed ledger to a divergent chain. + */ +inline constexpr char ledgerJumpTotal[] = "ledger_jump_total"; + +// ===== Peer supply: what this node's peers can and will serve =============== + +/** + * Peer coverage of the ledger sequence range this node still needs. + */ +inline constexpr char peerLedgerSupply[] = "peer_ledger_supply"; +/** + * Inbound peer connection attempts, by terminal outcome. + */ +inline constexpr char peerAcceptTotal[] = "peer_accept_total"; +/** + * Peer disconnects, by cause and connection direction. + */ +inline constexpr char peerDisconnectTotal[] = "peer_disconnect_total"; +/** + * Peer data requests this node declined to serve, by kind and cause. + */ +inline constexpr char serveRefusedTotal[] = "serve_refused_total"; +/** + * PeerFinder slots, dials in flight, and address-cache sizes. + */ +inline constexpr char peerfinderSlotCensus[] = "peerfinder_slot_census"; +/** + * Amendment-block warning and the countdown to this node ceasing to validate. + */ +inline constexpr char amendmentBlock[] = "amendment_block"; + +// ===== Consensus ============================================================= + +/** + * Wall-clock duration of a completed consensus round. + */ +inline constexpr char consensusRoundDurationMs[] = "consensus_round_duration_ms"; + +// ===== Sweep: what the periodic cache sweep costs ============================ +// +// The sweep runs every `SizedItem::SweepInterval` seconds (10 s on a tiny node +// through 120 s on a huge one), so these three emit sites are cold: one +// histogram Record and two counter Adds per sweep is free at that cadence. + +/** + * Wall-clock duration of the `malloc_trim` call that ends every cache sweep. + * + * The cost of returning free heap pages to the kernel scales with the resident + * heap, so this is the signal that a node with a large existing database pays a + * per-sweep penalty a fresh node does not. + */ +inline constexpr char sweepMallocTrimUs[] = "sweep_malloc_trim_us"; +/** + * Minor page faults taken *inside* the `malloc_trim` call. + * + * Cumulative, so `rate()` gives faults/sec. Scoped to the trim call only -- see + * the limitation noted on the runbook branch: this proves the trim itself + * faults, not that the trim causes later faults as the caches refill. + */ +inline constexpr char sweepMallocTrimMinorFaultsTotal[] = "sweep_malloc_trim_minor_faults_total"; +/** + * Resident kilobytes the trim actually returned to the kernel. + * + * Cumulative and clamped at zero per sweep: a trim that reclaimed nothing, or + * during which another thread grew the heap faster than the trim shrank it, + * contributes 0 rather than a negative amount. + */ +inline constexpr char sweepMallocTrimReclaimedKbTotal[] = "sweep_malloc_trim_reclaimed_kb_total"; + +// ===== Rotation: the extra writes an online_delete rotation performs ========= + +/** + * Nodes re-stored by `copyNode` because they were missing from both backends. + * + * The genuinely unmeasured extra write of a rotation: a clean node reachable + * from the validated state map whose only on-disk copy lived in a backend an + * earlier rotation removed. Was warn-log-only. + */ +inline constexpr char rotationCopyNodeRestoreTotal[] = "rotation_copy_node_restore_total"; +/** + * Rotation state: whether one is running, and the copy-forward write total. + * + * A gauge, not a counter, because the two readings are polled from the node + * store rather than pushed: `in_flight` is current state and `copy_forward` is a + * cumulative total the nodestore already keeps. Observed from the existing + * `registerNodeStoreGauge` callback, which is how everything else reads the node + * store from xrpld without libxrpl having to know about telemetry. + */ +inline constexpr char rotationState[] = "rotation_state"; + +// ===== Pre-existing instruments pulled in by the family ratchet ============== +// +// These predate the sync-diagnostics work. They are declared here because the +// checker's Rule I enforces literal-freedom per metric FAMILY (first +// underscore segment), and each of these shares a family with a name above -- +// `ledger_`, `nodestore_`, `server_`, `peer_`, `state_`. Leaving them as +// literals would either weaken the rule to per-name (letting a typo'd sibling +// through) or require an exemption list. Declaring them is the honest option: +// no behaviour changes, and the next author editing these families finds the +// constant rather than inventing a second spelling. +// +// The remaining unconverted families are reported as Rule L warnings, so the +// outstanding work stays visible rather than silently accepted. + +/** + * Built-vs-validated ledger mismatches, by reason. + */ +inline constexpr char ledgerHistoryMismatchTotal[] = "ledger_history_mismatch_total"; +/** + * Ledger fee and economy readings. + */ +inline constexpr char ledgerEconomy[] = "ledger_economy"; +/** + * NodeStore I/O counters, queue depth and write load. + */ +inline constexpr char nodestoreState[] = "nodestore_state"; +/** + * Server-level health readings. + */ +inline constexpr char serverInfo[] = "server_info"; +/** + * Peer-network quality readings. + */ +inline constexpr char peerQuality[] = "peer_quality"; +/** + * Node state and operating-mode tracking. + */ +inline constexpr char stateTracking[] = "state_tracking"; + +} // namespace metric + +/** + * Label keys -- the dimension names attached to a metric datapoint. + * + * Every key here is bounded by design: the values it can take are either a + * fixed set declared in `namespace lval` below, or a small enumeration the + * code derives (an operating mode, a job type). A key whose values are + * unbounded runtime data would mint one time series per distinct value, so no + * such key is declared. + */ +namespace label { + +/** + * Sub-metric discriminator on a multi-series gauge. + * + * The pattern every observable gauge in MetricsRegistry.cpp already uses: one + * instrument carries several related readings, told apart by this label rather + * than by being separate instruments. Pre-dates the sync-diagnostics work; + * named here because the new gauges are its heaviest users. + */ +inline constexpr char metric[] = "metric"; +/** + * Job type, as produced by `JobTypes::name()`. + */ +inline constexpr char jobType[] = "job_type"; +/** + * Which producer submitted a job, within its job type. + * + * A job type has several producers (`RcvGetLedger` and `RcvGetObjByHash` both + * run as `JtLedgerReq`), so this is what attributes a latency spike to one of + * them. Bounded by `MetricsRegistry::sanitiseHandler()`, which folds any job + * name that is not all ASCII letters -- the ones embedding a ledger sequence + * -- down to a single `other` value. + */ +inline constexpr char handler[] = "handler"; +/** + * Terminal result of a bounded operation. + */ +inline constexpr char outcome[] = "outcome"; +/** + * Cause of a rejection, refusal or teardown. + */ +inline constexpr char reason[] = "reason"; +/** + * Configured validator-list site URI. The one runtime-valued key here. + */ +inline constexpr char site[] = "site"; +/** + * Operating mode a transition started from. + */ +inline constexpr char from[] = "from"; +/** + * Operating mode a transition ended at. + */ +inline constexpr char to[] = "to"; +/** + * Where acquired ledger data came from. + */ +inline constexpr char source[] = "source"; +/** + * Which stage of a multi-step pipeline the event belongs to. + */ +inline constexpr char stage[] = "stage"; +/** + * Connection direction, inbound or outbound. + */ +inline constexpr char direction[] = "direction"; +/** + * Which kind of peer data request is being described. + */ +inline constexpr char request[] = "request"; + +} // namespace label + +/** + * Bounded label values -- the fixed value sets the code itself writes. + * + * Nested by the instrument (or the gauge) that owns the set, because the same + * word means different things in different sets and a flat namespace would let + * two of them collide. A value is declared here only when the code chooses it + * from a fixed list; anything derived from runtime data stays out. + */ +namespace lval { + +// ===== Shared outcome/direction slugs ======================================= + +/** + * Values shared by more than one instrument. Declared once so two instruments + * that mean the same thing cannot spell it differently. + */ +inline constexpr char timeout[] = "timeout"; +inline constexpr char notFound[] = "not_found"; +inline constexpr char inbound[] = "inbound"; +inline constexpr char outbound[] = "outbound"; + +/** + * `dns_resolve_total` outcomes: did the resolver return any address? + */ +namespace dns_resolve { +inline constexpr char resolved[] = "resolved"; +inline constexpr char empty[] = "empty"; +} // namespace dns_resolve + +/** + * `peer_accept_total` outcomes -- the nine exits of the inbound-accept path. + * + * Every exit records one of these, so the counter's total equals the number of + * inbound attempts and an unexplained gap is impossible. + */ +namespace peer_accept { +inline constexpr char localEndpointFail[] = "local_endpoint_fail"; +inline constexpr char resourceLimit[] = "resource_limit"; +inline constexpr char noSlot[] = "no_slot"; +inline constexpr char notPeerRequest[] = "not_peer_request"; +inline constexpr char protocolMismatch[] = "protocol_mismatch"; +inline constexpr char badCookie[] = "bad_cookie"; +inline constexpr char slotRefused[] = "slot_refused"; +inline constexpr char accepted[] = "accepted"; +inline constexpr char handshakeError[] = "handshake_error"; +} // namespace peer_accept + +/** + * `handshake_negotiation_fail_total` reasons -- one per rejection point in + * the handshake verifier. + * + * These separate a peer misconfiguration this node should tolerate + * (`wrong_network`, `self_connection`) from a local misconfiguration an + * operator must fix (`clock_skew`, `local_ip_mismatch`), which is the whole + * point of splitting the counter by reason. + */ +namespace handshake_fail { +inline constexpr char invalidServerDomain[] = "invalid_server_domain"; +inline constexpr char invalidNetworkId[] = "invalid_network_id"; +inline constexpr char wrongNetwork[] = "wrong_network"; +inline constexpr char invalidClockTimestamp[] = "invalid_clock_timestamp"; +inline constexpr char clockSkew[] = "clock_skew"; +inline constexpr char unsupportedKeyType[] = "unsupported_key_type"; +inline constexpr char badPublicKey[] = "bad_public_key"; +inline constexpr char noSessionSignature[] = "no_session_signature"; +inline constexpr char sessionVerifyFailed[] = "session_verify_failed"; +inline constexpr char selfConnection[] = "self_connection"; +inline constexpr char invalidLocalIp[] = "invalid_local_ip"; +inline constexpr char localIpMismatch[] = "local_ip_mismatch"; +inline constexpr char invalidRemoteIp[] = "invalid_remote_ip"; +inline constexpr char remoteIpMismatch[] = "remote_ip_mismatch"; +} // namespace handshake_fail + +/** + * `unl_fetch_total` outcomes for the transport-level failures. + * + * The success path instead labels with `to_string(ListDisposition)`, whose + * values are owned by the protocol enum and are therefore not restated here -- + * duplicating them would create a second place to update when a disposition is + * added. + */ +namespace unl_fetch { +inline constexpr char fetchError[] = "fetch_error"; +inline constexpr char badStatus[] = "bad_status"; +inline constexpr char parseError[] = "parse_error"; +} // namespace unl_fetch + +/** + * `unl_quorum` sub-metrics: the trusted-key count and the bar it must clear. + */ +namespace unl_quorum { +inline constexpr char trustedKeys[] = "trusted_keys"; +inline constexpr char quorum[] = "quorum"; +// 1 while the validator list has disabled quorum, 0 otherwise. Carries the +// state that used to be encoded by publishing a sentinel value on `quorum` +// itself: a number that large plotted on a shared axis flattens the +// trusted-key line to the baseline, hiding the very failure it marked. +inline constexpr char quorumDisabled[] = "quorum_disabled"; +} // namespace unl_quorum + +/** + * `clock_close_offset_seconds` sub-metric. + */ +namespace clock_offset { +inline constexpr char offset[] = "offset"; +} // namespace clock_offset + +/** + * `sync_state` sub-metrics -- the four "why am I not FULL" signals. + * + * `initial_full_duration_us` reads zero until the node first reaches FULL, + * which is the state this gauge exists to make visible rather than a missing + * value. + */ +namespace sync_state { +inline constexpr char initialFullDurationUs[] = "initial_full_duration_us"; +inline constexpr char networkLedgerGate[] = "network_ledger_gate"; +inline constexpr char serverStallSeconds[] = "server_stall_seconds"; +inline constexpr char ledgersBehind[] = "ledgers_behind"; +} // namespace sync_state + +/** + * `sync_acquire` sub-metrics -- aggregate acquire progress. + * + * The two `missing_*_max` series are the stuck-detector: flat and non-zero + * across collection ticks means the acquire will never finish, while shrinking + * means slow but alive. `in_flight` is the context that tells idle from stuck. + */ +namespace sync_acquire { +inline constexpr char missingStateNodesMax[] = "missing_state_nodes_max"; +inline constexpr char missingTxNodesMax[] = "missing_tx_nodes_max"; +inline constexpr char receivedDataDepth[] = "received_data_depth"; +inline constexpr char inFlight[] = "in_flight"; +} // namespace sync_acquire + +/** + * `shamap_cache_hit_rate` sub-metric: which cache the rate describes. + */ +namespace shamap_cache { +inline constexpr char treenode[] = "treenode"; +} // namespace shamap_cache + +/** + * `sync_acquire_source_total` sources: served locally or fetched from peers. + */ +namespace acquire_source { +inline constexpr char local[] = "local"; +inline constexpr char network[] = "network"; +} // namespace acquire_source + +/** + * `sync_addnode_total` outcomes -- the per-node verdict on received SHAMap data. + * + * The split is what separates real progress (`good`) from wasted bandwidth + * (`duplicate`) and a misbehaving peer (`invalid`); traffic-level metrics show + * all three as healthy throughput. + */ +namespace addnode { +inline constexpr char good[] = "good"; +inline constexpr char duplicate[] = "duplicate"; +inline constexpr char invalid[] = "invalid"; +} // namespace addnode + +/** + * `jobq_saturation` sub-metrics: the numerator, denominator and the backlog. + */ +namespace jobq_saturation { +inline constexpr char runningTasks[] = "running_tasks"; +inline constexpr char workerThreads[] = "worker_threads"; +inline constexpr char totalWaiting[] = "total_waiting"; +} // namespace jobq_saturation + +/** + * `peer_ledger_supply` sub-metrics: who can serve what this node needs. + */ +namespace peer_supply { +inline constexpr char peersReporting[] = "peers_reporting"; +inline constexpr char peersServingValidated[] = "peers_serving_validated"; +inline constexpr char peersServingNext[] = "peers_serving_next"; +inline constexpr char supplyMinSeq[] = "supply_min_seq"; +inline constexpr char supplyMaxSeq[] = "supply_max_seq"; +} // namespace peer_supply + +/** + * `peerfinder_slot_census` sub-metrics -- slots, dials and address caches. + * + * `connecting` non-zero while `out_active` stays under `out_max` is the + * "starting dials and never completing them" case; both caches at zero on a + * fresh node means there is nothing left to dial at all. + */ +namespace slot_census { +inline constexpr char outActive[] = "out_active"; +inline constexpr char outMax[] = "out_max"; +inline constexpr char inActive[] = "in_active"; +inline constexpr char inMax[] = "in_max"; +inline constexpr char connecting[] = "connecting"; +inline constexpr char fixedConfigured[] = "fixed_configured"; +inline constexpr char fixedActive[] = "fixed_active"; +inline constexpr char bootcache[] = "bootcache"; +inline constexpr char livecache[] = "livecache"; +} // namespace slot_census + +/** + * `amendment_block` sub-metrics: the warning flag and the countdown. + */ +namespace amendment_block { +inline constexpr char warned[] = "warned"; +inline constexpr char secondsToBlock[] = "seconds_to_block"; +} // namespace amendment_block + +/** + * `rotation_state` sub-metrics: is a rotation running, and how many extra + * writes have rotations caused. + * + * Read together: a `copy_forward` total that climbs while `in_flight` is 1 is + * the rotation doing its extra writes, which is the expected shape. The same + * total climbing while `in_flight` is 0 would mean the flag leaked, not that + * rotation is cheap. + */ +namespace rotation_state { +inline constexpr char inFlight[] = "in_flight"; +inline constexpr char copyForward[] = "copy_forward"; +} // namespace rotation_state + +/** + * `ledger_quorum_publish` sub-metrics: the gate, and how late publish is. + */ +namespace quorum_publish { +inline constexpr char trustedValidationTally[] = "trusted_validation_tally"; +inline constexpr char quorumTarget[] = "quorum_target"; +inline constexpr char timeToFirstValidatedUs[] = "time_to_first_validated_us"; +inline constexpr char publishLag[] = "publish_lag"; +} // namespace quorum_publish + +/** + * `ledger_quorum_shortfall_total` stage: which gate did the rejecting. + */ +namespace quorum_shortfall { +inline constexpr char preAccept[] = "pre_accept"; +} // namespace quorum_shortfall + +/** + * `ledger_replay_fallback_total` stages: which sub-acquire gave up. + */ +namespace replay_fallback { +inline constexpr char skiplist[] = "skiplist"; +inline constexpr char delta[] = "delta"; +} // namespace replay_fallback + +/** + * `ledger_replay_outcome_total` outcomes -- the four terminal states of a + * replay task. `timeout` is the shared slug above. + */ +namespace replay_outcome { +inline constexpr char success[] = "success"; +inline constexpr char buildFailed[] = "build_failed"; +inline constexpr char parameterFailed[] = "parameter_failed"; +} // namespace replay_outcome + +/** + * `peer_disconnect_total` reasons -- why a peer connection closed. + * + * The split separates our-fault backpressure (`large_sendq`, + * `charge_resources`) from a topology or network fault (`not_useful`, + * `ping_timeout`, `read_error`); the two call for opposite responses. + * `unknown` is the initial value and appears when a teardown path set no + * cause, so an unattributed disconnect is visible rather than absent. + */ +namespace disconnect { +inline constexpr char unknown[] = "unknown"; +inline constexpr char malformedHandshake[] = "malformed_handshake"; +inline constexpr char stopping[] = "stopping"; +inline constexpr char chargeResources[] = "charge_resources"; +inline constexpr char timerError[] = "timer_error"; +inline constexpr char largeSendq[] = "large_sendq"; +inline constexpr char notUseful[] = "not_useful"; +inline constexpr char pingTimeout[] = "ping_timeout"; +inline constexpr char shutdown[] = "shutdown"; +inline constexpr char sharedValue[] = "shared_value"; +inline constexpr char writeError[] = "write_error"; +inline constexpr char graceful[] = "graceful"; +inline constexpr char readError[] = "read_error"; +} // namespace disconnect + +/** + * `serve_refused_total` request kinds: what the peer had asked for. + */ +namespace serve_request { +inline constexpr char object[] = "object"; +inline constexpr char fetchpack[] = "fetchpack"; +inline constexpr char txset[] = "txset"; +inline constexpr char ledger[] = "ledger"; +} // namespace serve_request + +/** + * `serve_refused_total` reasons: why this node would not answer. + * `not_found` is the shared slug above. + * + * `empty_reply` is the subtle one: the map WAS found, but the reply loop + * produced no nodes, so the requester still gets nothing and must ask another + * peer. Counting it as served would make a node that answers every request + * with an empty payload look healthy. + */ +namespace serve_refused { +inline constexpr char sendqFull[] = "sendq_full"; +inline constexpr char loadShed[] = "load_shed"; +inline constexpr char badType[] = "bad_type"; +inline constexpr char noMap[] = "no_map"; +inline constexpr char emptyReply[] = "empty_reply"; +} // namespace serve_refused + +} // namespace lval + +} // namespace xrpl::telemetry diff --git a/src/xrpld/telemetry/MetricsRegistry.cpp b/src/xrpld/telemetry/MetricsRegistry.cpp index 7a9ff556b43..af0e3291ab9 100644 --- a/src/xrpld/telemetry/MetricsRegistry.cpp +++ b/src/xrpld/telemetry/MetricsRegistry.cpp @@ -28,18 +28,23 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include #include +#include #include #include +#include #include +#include #include #include #include @@ -71,6 +76,7 @@ #include #include #include +#include #include #include #include @@ -92,11 +98,18 @@ constexpr char kJobQueuedDurationUs[] = "job_queued_us"; constexpr char kJobRunningDurationUs[] = "job_running_us"; constexpr char kRpcMethodDurationUs[] = "rpc_method_us"; -// Attribute (label) keys for the job instruments. Each is referenced from -// several record sites, and a counter and its histogram must carry exactly -// the same key spelling or the two series cannot be joined in a query. -constexpr char kJobTypeLabel[] = "job_type"; -constexpr char kHandlerLabel[] = "handler"; +// Millisecond-valued duration histogram instrument names. Same +// register-then-create pairing as the microsecond names above, so the same +// reason applies for naming them: the view and the record site must agree. +// +// consensus_round_duration_ms is recorded from RCLConsensus at the call site +// (via XRPL_METRIC_HISTOGRAM_RECORD, which creates the instrument lazily +// there), not created here. Only the VIEW is registered here, because a view +// matches by instrument name and must exist before the instrument is first +// used — the MeterProvider is built with the view registry, and the round +// histogram is not created until the first consensus round completes, well +// after start(). +constexpr char kConsensusRoundDurationMs[] = "consensus_round_duration_ms"; /** * Bucket boundaries for microsecond-valued duration instruments. @@ -123,36 +136,6 @@ constexpr std::array kMicrosecondBoundaries{ 30'000'000.0, 60'000'000.0}; -/** - * Bucket boundaries for latencies that are normally sub-millisecond. - * - * 1 µs, 2 µs, 5 µs, 10 µs, 25 µs, 50 µs, 100 µs, 250 µs, 500 µs, 1 ms, 5 ms, - * 25 ms. - * - * kMicrosecondBoundaries starts at 100 µs, which is above the entire range a - * healthy nodestore read occupies, so every warm read falls in its first - * bucket and the distribution reads as flat. These edges resolve the warm - * range instead, while still reaching far enough to show a cold tail against - * it. - * - * Currently unused: no sub-millisecond histogram instrument exists yet. The - * edges live here so the instrument that records nodestore read latency gets - * a ladder that fits it, rather than silently inheriting the wrong one. - */ -[[maybe_unused]] constexpr std::array kSubMillisecondBoundaries{ - 1.0, - 2.0, - 5.0, - 10.0, - 25.0, - 50.0, - 100.0, - 250.0, - 500.0, - 1'000.0, - 5'000.0, - 25'000.0}; - /** * Register an explicit-bucket histogram view. * @@ -185,10 +168,12 @@ addHistogramView( * Register the microsecond-ladder view for a duration instrument. * * Job wait/run times and RPC latencies routinely exceed the SDK default - * ceiling, so they all share `kMicrosecondBoundaries`. + * ceiling, so they all share `kMicrosecondBoundaries`: 100µs, 500µs, 1ms, 5ms, + * 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 2.5s, 5s, 10s, 30s, 60s — + * sub-millisecond jobs through multi-second stalls, without saturating. * - * @param views The registry to add the view to. - * @param name Instrument name to match. + * @param views The registry to add the view to. + * @param name Instrument name to match (e.g. "job_running_us"). */ void addMicrosecondHistogramView(metric_sdk::ViewRegistry& views, std::string const& name) @@ -196,6 +181,47 @@ addMicrosecondHistogramView(metric_sdk::ViewRegistry& views, std::string const& addHistogramView(views, name, {kMicrosecondBoundaries.begin(), kMicrosecondBoundaries.end()}); } +/** + * Register the explicit-bucket view for a consensus-round duration in + * MILLISECONDS. + * + * The round histogram needs its own boundaries for two reasons. The SDK + * default tops out at 10,000 ms, and a recovering or stalled node routinely + * rounds slower than that — the consensus parameters themselves allow up to + * `ledgerAbandonConsensus` = 120 s — so the default would collapse exactly the + * slow rounds this signal exists to show into one saturated top bucket. And a + * healthy round is about 3-4 s, which the default's coarse spacing near that + * value cannot resolve, so a round drifting from 3 s to 5 s would not move any + * quantile. + * + * Boundaries: 500ms, 1s, 2s, 3s, 4s, 5s, 7.5s, 10s, 15s, 20s, 30s, 60s, 120s. + * Dense across the healthy 2-5 s band, then widening to the 120 s abandon + * limit so a stalled round still lands in a real bucket. + * + * @param views The registry to add the view to. + * @param name Instrument name to match ("consensus_round_duration_ms"). + */ +void +addRoundDurationHistogramView(metric_sdk::ViewRegistry& views, std::string const& name) +{ + addHistogramView( + views, + name, + {500.0, + 1'000.0, + 2'000.0, + 3'000.0, + 4'000.0, + 5'000.0, + 7'500.0, + 10'000.0, + 15'000.0, + 20'000.0, + 30'000.0, + 60'000.0, + 120'000.0}); +} + } // namespace #endif // XRPL_ENABLE_TELEMETRY @@ -269,17 +295,77 @@ MetricsRegistry::initExporterAndProvider(std::string const& endpoint, std::strin attrs[opentelemetry::semconv::service::kServiceInstanceId] = instanceId; auto resourceAttrs = resource::Resource::Create(attrs); - // Build a view registry with explicit microsecond buckets for the - // duration histograms. Without this they use the SDK default buckets - // (max 10,000 = 10 ms), saturating every quantile at 10 ms. + // Build a view registry with explicit buckets for the duration + // histograms. Without this they use the SDK default buckets (max 10,000), + // which saturates every quantile at 10 ms for the µs instruments and at + // 10 s for the round histogram. auto views = std::make_unique(); addMicrosecondHistogramView(*views, kJobQueuedDurationUs); addMicrosecondHistogramView(*views, kJobRunningDurationUs); addMicrosecondHistogramView(*views, kRpcMethodDurationUs); + // Millisecond-scale: recorded at the RCLConsensus call site, so only the + // view is declared here (see the constant's comment). + addRoundDurationHistogramView(*views, kConsensusRoundDurationMs); + // Recorded at its PeerImp.cpp call site, not created here, so the name // comes from the shared constant both sites use. addMicrosecondHistogramView(*views, kGetObjectLookupUs); + // Sweep malloc_trim duration. Shares the microsecond ladder rather than + // getting a bespoke one, and the ladder is what makes it readable: a trim on + // a small heap lands in the tens-of-microseconds buckets, while a trim on a + // multi-gigabyte resident heap runs well past 10 ms -- which is exactly the + // large-existing-database case this signal exists to catch. With the SDK + // default ceiling of 10,000 every one of those would collapse into the + // overflow bucket and p95 would read exactly 10 ms however bad it got. The + // shared ladder's upper reaches (25 ms, 50 ms, 100 ms, 250 ms, 500 ms, 1 s + // and beyond) resolve those, and its lower reaches (100 us, 500 us) resolve + // the healthy fresh-node case, so a per-instrument ladder would add a second + // thing to maintain for no extra resolution. + addMicrosecondHistogramView(*views, metric::sweepMallocTrimUs); + + // Millisecond dial/resolve latencies. Both exceed the SDK default ceiling + // of 10,000: the dial timer is 15 s, so without an explicit ladder every + // timed-out dial lands in the overflow bucket and p95 reads exactly 10 s + // however bad it gets. The 15 s boundary sits on its own so a timeout is + // distinguishable from merely slow. + addHistogramView( + *views, + metric::dnsResolveLatencyMs, + {1.0, + 5.0, + 10.0, + 25.0, + 50.0, + 100.0, + 250.0, + 500.0, + 1'000.0, + 2'500.0, + 5'000.0, + 10'000.0, + 15'000.0, + 20'000.0, + 30'000.0}); + addHistogramView( + *views, + metric::overlayDialLatencyMs, + {1.0, + 5.0, + 10.0, + 25.0, + 50.0, + 100.0, + 250.0, + 500.0, + 1'000.0, + 2'500.0, + 5'000.0, + 10'000.0, + 15'000.0, + 20'000.0, + 30'000.0}); + // The remaining two GetObject histograms are not durations, so the // microsecond ladder above does not fit them. Both still need explicit // boundaries: the SDK default stops at 10,000 and both ranges exceed it. @@ -338,8 +424,10 @@ MetricsRegistry::initSyncInstruments() "validations_sent_total", "Total validations sent by this node"); validationsCheckedCounter_ = meter_->CreateUInt64Counter( "validations_checked_total", "Total network validations received and checked"); - stateChangesCounter_ = - meter_->CreateUInt64Counter("state_changes_total", "Total operating mode changes"); + // state_changes_total is NOT created here. It is emitted at its call site + // (NetworkOPsImp::setMode) through XRPL_METRIC_COUNTER_INC_LABELED so it + // can carry the {from,to} transition labels; a registry-owned instrument + // would only give an unlabelled total. // jq_trans_overflow_total is observed from Overlay's existing cumulative // atomic (Overlay::getJqTransOverflow()) rather than pushed. The overlay // owns the only increment site (PeerImp), so an ObservableCounter reads the @@ -365,7 +453,7 @@ MetricsRegistry::initSyncInstruments() }, this); ledgerHistoryMismatchCounter_ = meter_->CreateUInt64Counter( - "ledger_history_mismatch_total", "Total built-vs-validated ledger mismatches by reason"); + metric::ledgerHistoryMismatchTotal, "Total built-vs-validated ledger mismatches by reason"); txqExpiredCounter_ = meter_->CreateUInt64Counter( "txq_expired_total", "Total transactions expired out of the transaction queue"); txqDroppedCounter_ = meter_->CreateUInt64Counter( @@ -484,8 +572,8 @@ MetricsRegistry::recordJobQueued(std::string_view jobType, std::string_view jobN return; jobQueuedCounter_->Add( 1, - {{kJobTypeLabel, std::string(jobType)}, - {kHandlerLabel, std::string(sanitiseHandler(jobName))}}); + {{label::jobType, std::string(jobType)}, + {label::handler, std::string(sanitiseHandler(jobName))}}); #else (void)jobType; (void)jobName; @@ -505,7 +593,7 @@ MetricsRegistry::recordJobStarted( // Build the attribute pair once: both the counter and the histogram // must carry the identical label set or they cannot be joined. std::string const handler(sanitiseHandler(jobName)); - jobStartedCounter_->Add(1, {{kJobTypeLabel, std::string(jobType)}, {kHandlerLabel, handler}}); + jobStartedCounter_->Add(1, {{label::jobType, std::string(jobType)}, {label::handler, handler}}); if (jobQueuedDurationHistogram_ && queuedDurUs >= 0) { // Guard against negative queued durations: the caller derives this @@ -514,7 +602,7 @@ MetricsRegistry::recordJobStarted( // (logging a warning per call), so skip them rather than spam. jobQueuedDurationHistogram_->Record( static_cast(queuedDurUs), - {{kJobTypeLabel, std::string(jobType)}, {kHandlerLabel, handler}}, + {{label::jobType, std::string(jobType)}, {label::handler, handler}}, opentelemetry::context::Context{}); } #else @@ -535,12 +623,13 @@ MetricsRegistry::recordJobFinished( if (!enabled_ || !jobFinishedCounter_) return; std::string const handler(sanitiseHandler(jobName)); - jobFinishedCounter_->Add(1, {{kJobTypeLabel, std::string(jobType)}, {kHandlerLabel, handler}}); + jobFinishedCounter_->Add( + 1, {{label::jobType, std::string(jobType)}, {label::handler, handler}}); if (jobRunningDurationHistogram_) { jobRunningDurationHistogram_->Record( static_cast(runningDurUs), - {{kJobTypeLabel, std::string(jobType)}, {kHandlerLabel, handler}}, + {{label::jobType, std::string(jobType)}, {label::handler, handler}}, opentelemetry::context::Context{}); } #else @@ -568,6 +657,7 @@ MetricsRegistry::registerAsyncGauges() registerObjectCountGauge(); registerLoadFactorGauge(); registerNodeStoreGauge(); + registerRotationStateGauge(); registerServerInfoGauge(); registerBuildInfoGauge(); registerCompleteLedgersGauge(); @@ -580,6 +670,17 @@ MetricsRegistry::registerAsyncGauges() registerStorageDetailGauge(); registerValidationAgreementGauge(); registerValidationTotalsCounters(); + registerUnlQuorumGauge(); + registerClockSkewGauge(); + registerSyncStateGauge(); + registerStallEventsCounter(); + registerSyncAcquireGauge(); + registerCacheHitRateDetailGauge(); + registerJobQueueSaturationGauge(); + registerPeerLedgerSupplyGauge(); + registerSlotCensusGauge(); + registerAmendmentBlockGauge(); + registerLedgerQuorumPublishGauge(); } void @@ -601,7 +702,7 @@ MetricsRegistry::registerCacheHitRateGauge() auto sleRate = app.getCachedSLEs().rate(); opentelemetry::nostd::get>>(result) - ->Observe(sleRate, {{"metric", "SLE_hit_rate"}}); + ->Observe(sleRate, {{label::metric, "SLE_hit_rate"}}); // Ledger cache hit rate. // TaggedCache::getHitRate() returns 0-100; normalize to @@ -610,40 +711,40 @@ MetricsRegistry::registerCacheHitRateGauge() auto ledgerRate = app.getLedgerMaster().getCacheHitRate() / 100.0; opentelemetry::nostd::get>>(result) - ->Observe(ledgerRate, {{"metric", "ledger_hit_rate"}}); + ->Observe(ledgerRate, {{label::metric, "ledger_hit_rate"}}); // AcceptedLedger cache hit rate (also 0-100 from // TaggedCache; normalize to 0.0-1.0). auto alRate = app.getAcceptedLedgerCache().getHitRate() / 100.0; opentelemetry::nostd::get>>(result) - ->Observe(alRate, {{"metric", "AL_hit_rate"}}); + ->Observe(alRate, {{label::metric, "AL_hit_rate"}}); // TreeNode cache size. auto tnCacheSize = app.getNodeFamily().getTreeNodeCache()->getCacheSize(); opentelemetry::nostd::get>>(result) ->Observe( - static_cast(tnCacheSize), {{"metric", "treenode_cache_size"}}); + static_cast(tnCacheSize), {{label::metric, "treenode_cache_size"}}); // TreeNode track size. auto tnTrackSize = app.getNodeFamily().getTreeNodeCache()->getTrackSize(); opentelemetry::nostd::get>>(result) ->Observe( - static_cast(tnTrackSize), {{"metric", "treenode_track_size"}}); + static_cast(tnTrackSize), {{label::metric, "treenode_track_size"}}); // FullBelow cache size. auto fbSize = app.getNodeFamily().getFullBelowCache()->size(); opentelemetry::nostd::get>>(result) - ->Observe(static_cast(fbSize), {{"metric", "fullbelow_size"}}); + ->Observe(static_cast(fbSize), {{label::metric, "fullbelow_size"}}); // AcceptedLedger cache size (entry count). auto alSize = app.getAcceptedLedgerCache().size(); opentelemetry::nostd::get>>(result) - ->Observe(static_cast(alSize), {{"metric", "AL_size"}}); + ->Observe(static_cast(alSize), {{label::metric, "AL_size"}}); } catch (...) // NOLINT(bugprone-empty-catch) { @@ -672,7 +773,7 @@ MetricsRegistry::registerTxqGauge() auto observe = [&](char const* name, double value) { opentelemetry::nostd::get>>(result) - ->Observe(value, {{"metric", name}}); + ->Observe(value, {{label::metric, name}}); }; observe("txq_count", static_cast(metrics.txCount)); @@ -753,7 +854,7 @@ MetricsRegistry::registerLoadFactorGauge() auto observe = [&](char const* name, double value) { opentelemetry::nostd::get>>(result) - ->Observe(value, {{"metric", name}}); + ->Observe(value, {{label::metric, name}}); }; // Combined load factor (server component). @@ -910,7 +1011,7 @@ MetricsRegistry::registerNodeStoreGauge() // across four helpers, one per domain, to stay inside the per-function // line budget and to keep each domain testable on its own. nodeStoreGauge_ = meter_->CreateInt64ObservableGauge( - "nodestore_state", + metric::nodestoreState, "NodeStore I/O counters, latencies, write-queue depth and acquisition stalls"); nodeStoreGauge_->AddCallback( [](opentelemetry::metrics::ObserverResult result, void* state) { @@ -926,7 +1027,7 @@ MetricsRegistry::registerNodeStoreGauge() ObserveFn const observe = [&](char const* name, std::int64_t value) { opentelemetry::nostd::get>>(result) - ->Observe(value, {{"metric", name}}); + ->Observe(value, {{label::metric, name}}); }; // Qualified because the enclosing lambda captures nothing: @@ -944,12 +1045,76 @@ MetricsRegistry::registerNodeStoreGauge() this); } +void +MetricsRegistry::registerRotationStateGauge() +{ + // --- Sync diagnostics: what an online_delete rotation costs --- + // A rotation performs writes an ordinary fetch would not: the archive + // backend is about to be deleted, so any node body it serves during the + // rotation window has to be rewritten into the writable backend to survive. + // That work scales with the archive, competes with sync I/O, and appears + // ONLY on a populated, already-rotated online_delete database -- which is + // why it never shows up on a fresh node and why it was never measured. The + // count existed as copyForwardCount_ but was log-only and reset per + // rotation. + // + // Polled from the node store rather than pushed, matching + // registerNodeStoreGauge above: DatabaseRotatingImp lives in libxrpl and + // cannot include xrpld/telemetry, so the counters are read through the + // DatabaseRotating accessors on each collection tick instead. + rotationStateGauge_ = meter_->CreateInt64ObservableGauge( + metric::rotationState, "Online-delete rotation state and copy-forward write total"); + rotationStateGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + // Only a rotating store has a rotation to report. On a node + // without online_delete the node store is a DatabaseNodeImp, so + // the cast fails and NO series is published -- deliberately, so + // that an absent series means "rotation is not configured" + // rather than a zero that would read as "rotation is free". + auto* rotating = dynamic_cast(&app.getNodeStore()); + if (rotating == nullptr) + return; + + auto observe = [&](char const* field, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{label::metric, field}}); + }; + + // The window the extra writes happen in. A panel needs this to + // know when the copy-forward total is expected to move. + observe( + lval::rotation_state::inFlight, + static_cast(rotating->isRotationInFlight() ? 1 : 0)); + + // Monotonic, so the panel takes rate() over it. The per-rotation + // tally that rotate() logs is reset on every swap and is + // therefore unusable here. + observe( + lval::rotation_state::copyForward, + static_cast(rotating->copyForwardTotal())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + void MetricsRegistry::registerServerInfoGauge() { // --- Task 9.7a: Server info gauges --- serverInfoGauge_ = - meter_->CreateInt64ObservableGauge("server_info", "Server-level health metrics"); + meter_->CreateInt64ObservableGauge(metric::serverInfo, "Server-level health metrics"); serverInfoGauge_->AddCallback( [](opentelemetry::metrics::ObserverResult result, void* state) { auto* self = static_cast(state); @@ -962,7 +1127,7 @@ MetricsRegistry::registerServerInfoGauge() auto observe = [&](char const* name, int64_t value) { opentelemetry::nostd::get>>(result) - ->Observe(value, {{"metric", name}}); + ->Observe(value, {{label::metric, name}}); }; // Server operating mode (DISCONNECTED=0 .. FULL=4). @@ -1126,7 +1291,7 @@ MetricsRegistry::registerDbMetricsGauge() auto observe = [&](char const* name, int64_t value) { opentelemetry::nostd::get>>(result) - ->Observe(value, {{"metric", name}}); + ->Observe(value, {{label::metric, name}}); }; auto& rdb = app.getRelationalDatabase(); @@ -1165,7 +1330,7 @@ MetricsRegistry::registerValidatorHealthGauge() auto observe = [&](char const* name, double value) { opentelemetry::nostd::get>>(result) - ->Observe(value, {{"metric", name}}); + ->Observe(value, {{label::metric, name}}); }; observe("amendment_blocked", app.getOPs().isAmendmentBlocked() ? 1.0 : 0.0); @@ -1201,7 +1366,7 @@ MetricsRegistry::registerPeerQualityGauge() // Uses Peer::json() to read latency and version since those accessors // are not on the abstract Peer interface (they live on PeerImp). peerQualityGauge_ = - meter_->CreateDoubleObservableGauge("peer_quality", "Peer network quality metrics"); + meter_->CreateDoubleObservableGauge(metric::peerQuality, "Peer network quality metrics"); peerQualityGauge_->AddCallback( [](opentelemetry::metrics::ObserverResult result, void* state) { auto* self = static_cast(state); @@ -1214,7 +1379,7 @@ MetricsRegistry::registerPeerQualityGauge() auto observe = [&](char const* name, double value) { opentelemetry::nostd::get>>(result) - ->Observe(value, {{"metric", name}}); + ->Observe(value, {{label::metric, name}}); }; // Collect latencies, version info, and tracking state from @@ -1321,7 +1486,7 @@ MetricsRegistry::registerReduceRelayGauge() auto observe = [&](char const* name, int64_t value) { opentelemetry::nostd::get>>(result) - ->Observe(value, {{"metric", name}}); + ->Observe(value, {{label::metric, name}}); }; // Each field is a decimal string; emit when present and parseable. @@ -1352,8 +1517,8 @@ void MetricsRegistry::registerLedgerEconomyGauge() { // --- Task 7.11: Ledger economy gauges --- - ledgerEconomyGauge_ = - meter_->CreateDoubleObservableGauge("ledger_economy", "Ledger fee and economy metrics"); + ledgerEconomyGauge_ = meter_->CreateDoubleObservableGauge( + metric::ledgerEconomy, "Ledger fee and economy metrics"); ledgerEconomyGauge_->AddCallback( [](opentelemetry::metrics::ObserverResult result, void* state) { auto* self = static_cast(state); @@ -1366,7 +1531,7 @@ MetricsRegistry::registerLedgerEconomyGauge() auto observe = [&](char const* name, double value) { opentelemetry::nostd::get>>(result) - ->Observe(value, {{"metric", name}}); + ->Observe(value, {{label::metric, name}}); }; // Local fee (drops). @@ -1419,7 +1584,7 @@ MetricsRegistry::registerStateTrackingGauge() { // --- Task 7.12: State tracking gauges --- stateTrackingGauge_ = - meter_->CreateDoubleObservableGauge("state_tracking", "Node state and mode tracking"); + meter_->CreateDoubleObservableGauge(metric::stateTracking, "Node state and mode tracking"); stateTrackingGauge_->AddCallback( [](opentelemetry::metrics::ObserverResult result, void* state) { auto* self = static_cast(state); @@ -1432,7 +1597,7 @@ MetricsRegistry::registerStateTrackingGauge() auto observe = [&](char const* name, double value) { opentelemetry::nostd::get>>(result) - ->Observe(value, {{"metric", name}}); + ->Observe(value, {{label::metric, name}}); }; // State value: 0-4 from OperatingMode, 5=validating, 6=proposing. @@ -1490,7 +1655,7 @@ MetricsRegistry::registerStorageDetailGauge() auto observe = [&](char const* name, int64_t value) { opentelemetry::nostd::get>>(result) - ->Observe(value, {{"metric", name}}); + ->Observe(value, {{label::metric, name}}); }; // Cumulative payload bytes handed to the NodeStore. This is @@ -1545,7 +1710,7 @@ MetricsRegistry::registerValidationAgreementGauge() auto observe = [&](char const* name, double value) { opentelemetry::nostd::get>>(result) - ->Observe(value, {{"metric", name}}); + ->Observe(value, {{label::metric, name}}); }; observe("agreement_pct_1h", self->validationTracker_.agreementPct1h()); @@ -1632,6 +1797,545 @@ MetricsRegistry::registerValidationTotalsCounters() this); } +void +MetricsRegistry::registerUnlQuorumGauge() +{ + // --- Sync diagnostics: trusted UNL size against required quorum --- + // validator_health already exports the quorum on its own; pairing it + // with the trusted-key count in one instrument is what makes the + // "can this node ever validate?" comparison a single query. + unlQuorumGauge_ = meter_->CreateInt64ObservableGauge( + metric::unlQuorum, "Trusted UNL key count vs required quorum"); + unlQuorumGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{label::metric, name}}); + }; + + auto& validators = app.getValidators(); + + // Trusted master keys currently in effect. Zero means no + // usable UNL: quorum can never be met. + observe( + lval::unl_quorum::trustedKeys, + static_cast(validators.trustedKeyCount())); + + // Validations required for a ledger to be fully validated. + // ValidatorList disables quorum by returning SIZE_MAX when too + // many publishers are unavailable. Casting that straight to + // int64_t would wrap to -1 and make the headroom + // (trusted_keys - quorum) read positive on a node that can + // never validate, so report the disabled state as int64 max + // instead: headroom then goes strongly negative, which is the + // truthful signal. + auto const quorum = validators.quorum(); + // A disabled quorum omits the series rather than publishing a + // sentinel. Both consumers of this gauge are timeseries panels + // sharing one axis with trusted_keys, so a huge value would + // flatten the key line to the baseline and hide the outage it + // was meant to signal. The boolean below carries the state, and + // a missing quorum line is itself the visible anomaly. + bool const quorumDisabled = quorum == std::numeric_limits::max(); + if (!quorumDisabled) + observe(lval::unl_quorum::quorum, static_cast(quorum)); + observe(lval::unl_quorum::quorumDisabled, quorumDisabled ? 1 : 0); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerClockSkewGauge() +{ + // --- Sync diagnostics: network close-time offset --- + // A persistent offset shows the local clock disagrees with the + // network, which delays consensus participation. server_info hides + // this below 60 s, so export it continuously instead. + clockSkewGauge_ = meter_->CreateInt64ObservableGauge( + metric::clockCloseOffsetSeconds, + "Network close time offset from the local clock, in seconds"); + clockSkewGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{label::metric, name}}); + }; + + // Negative when the local clock runs ahead of the network. + observe( + lval::clock_offset::offset, + static_cast(app.getTimeKeeper().closeOffset().count())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerSyncStateGauge() +{ + // --- Sync diagnostics: why a fresh node is not FULL yet --- + // Four values that previously lived only in a log line or in server_info + // JSON. All four are cheap reads pulled on the ~10 s reader tick. + syncStateGauge_ = + meter_->CreateInt64ObservableGauge(metric::syncState, "Sync-pipeline health signals"); + syncStateGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{label::metric, name}}); + }; + + auto& ops = app.getOPs(); + + // Time to first FULL. Zero means the node has not synced yet, + // which is exactly the case this signal exists to expose. + observe( + lval::sync_state::initialFullDurationUs, + static_cast(ops.getInitialSyncDurationUs())); + + // 1 = still waiting for a full network ledger. While this is + // set the node refuses transactions and cannot reach FULL. + observe(lval::sync_state::networkLedgerGate, ops.isNeedNetworkLedger() ? 1 : 0); + + // Current main-loop stall duration; 0 when healthy. + observe( + lval::sync_state::serverStallSeconds, + static_cast(app.getLoadManager().getCurrentStallSeconds())); + + // Distance from the network tip, floored at zero by the + // accessor. + observe( + lval::sync_state::ledgersBehind, + static_cast(ops.getLedgersBehindNetwork())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerStallEventsCounter() +{ + // --- Sync diagnostics: stall episode count --- + // Observed rather than pushed: LoadManager's monitor thread already owns + // the cumulative tally, and an ObservableCounter reads it each collection + // cycle without threading a push path through the load-monitor loop. + // Kept out of the sync_state gauge because a cumulative total needs + // counter aggregation for rate() to be meaningful. + stallEventsObservable_ = meter_->CreateInt64ObservableCounter( + metric::serverStallEventsTotal, "Total server main-loop stall episodes"); + stallEventsObservable_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + try + { + opentelemetry::nostd::get>>(result) + ->Observe( + static_cast(self->app_.getLoadManager().getStallEventCount())); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerSyncAcquireGauge() +{ + // --- Sync diagnostics: is ledger acquisition actually progressing? --- + // Aggregated on purpose: a per-ledger label would add one series per ledger + // acquired, which is unbounded. The per-ledger view lives on the + // ledger.acquire span instead. + syncAcquireGauge_ = meter_->CreateInt64ObservableGauge( + metric::syncAcquire, "Aggregate ledger-acquire progress across in-flight acquires"); + syncAcquireGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* name, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{label::metric, name}}); + }; + + // One snapshot feeds all four series, so they are mutually + // consistent rather than read at four different instants. + auto const progress = app.getInboundLedgers().acquireProgress(); + + // Flat and non-zero across ticks = this acquire will never + // finish. Shrinking = slow but alive. + observe( + lval::sync_acquire::missingStateNodesMax, + static_cast(progress.maxMissingStateNodes)); + observe( + lval::sync_acquire::missingTxNodesMax, + static_cast(progress.maxMissingTxNodes)); + + // Deep stash = arriving data outpaces processing. + observe( + lval::sync_acquire::receivedDataDepth, + static_cast(progress.receivedDataDepth)); + + // Context for the three above: zero everywhere with zero + // in-flight acquires is idle, not healthy. + observe(lval::sync_acquire::inFlight, static_cast(progress.inFlight)); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerCacheHitRateDetailGauge() +{ + // --- Sync diagnostics: SHAMap tree-node cache hit rate --- + // The memory layer above the node store: a miss here is what causes a + // node-store read, which the NuDB hit-ratio panel then measures. + shamapCacheHitRateGauge_ = meter_->CreateDoubleObservableGauge( + metric::shamapCacheHitRate, "SHAMap tree-node cache hit rate (0.0-1.0), by cache"); + shamapCacheHitRateGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + // TaggedCache::getHitRate() returns 0-100; normalize to 0.0-1.0 + // so the panel can use Grafana's "percentunit" unit, matching + // how cache_metrics already reports its rates. + auto const rate = app.getNodeFamily().getTreeNodeCache()->getHitRate() / 100.0F; + opentelemetry::nostd::get>>(result) + ->Observe( + static_cast(rate), {{label::metric, lval::shamap_cache::treenode}}); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerJobQueueSaturationGauge() +{ + // --- Sync diagnostics: is the whole worker pool exhausted? --- + // Attributes a broad multi-stage slowdown to the pool once, instead of + // leaving it to look like an independent fault in every subsystem whose + // jobs are queued behind it. + jobQueueSaturationGauge_ = meter_->CreateInt64ObservableGauge( + metric::jobqSaturation, + "Worker-pool saturation: tasks in flight, worker threads, jobs queued"); + jobQueueSaturationGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* field, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{label::metric, field}}); + }; + + // One reading feeds all three series so the ratio and the + // backlog describe the same instant. + auto const saturation = app.getJobQueue().getWorkerSaturation(); + observe(lval::jobq_saturation::runningTasks, saturation.runningTasks); + + // The denominator for the ratio panel. Derived at startup from + // [workers], node size and hardware concurrency, so it cannot + // be hardcoded in a dashboard. + observe(lval::jobq_saturation::workerThreads, saturation.workerThreads); + + // Ratio at 1.0 alone is a busy pool; ratio at 1.0 with a + // non-zero backlog is an exhausted one. + observe(lval::jobq_saturation::totalWaiting, saturation.totalWaiting); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerPeerLedgerSupplyGauge() +{ + // --- Sync diagnostics: can the network even serve what I need? --- + // Each peer advertises its ledger range and the connection caches it, but + // nothing ever compared those ranges, so "no peer holds the sequence I + // want" looked exactly like "my peers are slow" -- two faults with + // completely different fixes. + peerLedgerSupplyGauge_ = meter_->CreateInt64ObservableGauge( + metric::peerLedgerSupply, "Peer coverage of the ledger sequence this node needs"); + peerLedgerSupplyGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* field, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{label::metric, field}}); + }; + + // One pass over the peers, so all five series describe the + // same peer set at the same instant. + auto const supply = app.getOverlay().getPeerLedgerSupply( + app.getLedgerMaster().getValidLedgerIndex()); + + // The denominator. Zero serving out of zero reporting is + // silence; zero out of many is a real supply gap. + observe(lval::peer_supply::peersReporting, supply.peersReporting); + observe(lval::peer_supply::peersServingValidated, supply.peersServingValidated); + + // The verdict: zero here while peers_reporting is non-zero + // means waiting cannot finish the sync. + observe(lval::peer_supply::peersServingNext, supply.peersServingNext); + + // The window the peer set covers, so an operator can tell a + // request for discarded history from one for an unreached tip. + observe(lval::peer_supply::supplyMinSeq, supply.supplyMinSeq); + observe(lval::peer_supply::supplyMaxSeq, supply.supplyMaxSeq); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerSlotCensusGauge() +{ + // --- Sync diagnostics: why can this node not get peers? --- + // All nine numbers already exist inside PeerFinder; only the two active + // counts are exported today, which cannot distinguish "not dialling", + // "dialling and failing" and "nothing to dial". + slotCensusGauge_ = meter_->CreateInt64ObservableGauge( + metric::peerfinderSlotCensus, "PeerFinder slots, connection attempts and address caches"); + slotCensusGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* field, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{label::metric, field}}); + }; + + // One snapshot under one PeerFinder lock acquire, so occupancy + // and capacity can be compared against each other. + auto const census = app.getOverlay().getSlotCensus(); + + observe(lval::slot_census::outActive, census.outActive); + observe(lval::slot_census::outMax, census.outMax); + observe(lval::slot_census::inActive, census.inActive); + observe(lval::slot_census::inMax, census.inMax); + + // Dials in flight. Non-zero while out_active stays under + // out_max is the "starting and never completing" case. + observe(lval::slot_census::connecting, census.connecting); + + // fixed_active below fixed_configured names a configured peer + // that cannot be reached. + observe(lval::slot_census::fixedConfigured, census.fixedConfigured); + observe(lval::slot_census::fixedActive, census.fixedActive); + + // Both at zero on a fresh node means there is nothing to dial. + observe(lval::slot_census::bootcache, census.bootcache); + observe(lval::slot_census::livecache, census.livecache); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerAmendmentBlockGauge() +{ + // --- Sync diagnostics: how long until this node stops validating? --- + // The existing validator_health{metric="amendment_blocked"} reports the + // terminal state, when nothing can be done. This is the window before it. + amendmentBlockGauge_ = meter_->CreateInt64ObservableGauge( + metric::amendmentBlock, + "Amendment-block warning and seconds until the node stops validating"); + amendmentBlockGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* field, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{label::metric, field}}); + }; + + // An unsupported amendment has reached majority. Until now this + // only surfaced as an admin-only server_info warning. + observe(lval::amendment_block::warned, app.getOPs().isAmendmentWarned() ? 1 : 0); + + // Seconds until that amendment activates. -1 means nothing is + // pending: a distinct healthy value rather than an absent + // series, matching validator_health{metric="unl_expiry_days"}. + std::int64_t secondsToBlock = -1; + if (auto const expected = app.getAmendmentTable().firstUnsupportedExpected()) + { + // NetClock's representation is unsigned, so the difference + // is taken in int64_t: subtracting the time_points directly + // would wrap once the activation time has passed. + auto const expectedSecs = + static_cast(expected->time_since_epoch().count()); + auto const nowSecs = static_cast( + app.getTimeKeeper().closeTime().time_since_epoch().count()); + + // Clamped at 0: past due means the block is imminent, not + // overdue by an amount worth charting. + secondsToBlock = std::max(expectedSecs - nowSecs, 0); + } + observe(lval::amendment_block::secondsToBlock, secondsToBlock); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + +void +MetricsRegistry::registerLedgerQuorumPublishGauge() +{ + // --- Sync diagnostics: the quorum gate and the publish pipeline --- + // The last two stages of a fresh sync, and the two whose failures were + // invisible: a node can hold every ledger it needs and still never declare + // one validated (quorum short), or validate correctly and never publish + // (pipeline behind). Both used to be trace-log-only or not derivable at all. + ledgerQuorumPublishGauge_ = meter_->CreateInt64ObservableGauge( + metric::ledgerQuorumPublish, + "Pre-accept quorum gate and publish lag (tally vs quorum, first-validated, lag)"); + ledgerQuorumPublishGauge_->AddCallback( + [](opentelemetry::metrics::ObserverResult result, void* state) { + auto* self = static_cast(state); + if (self->callbacksDetached_.load(std::memory_order_acquire)) + return; + auto& app = self->app_; + + try + { + auto observe = [&](char const* field, int64_t value) { + opentelemetry::nostd::get>>(result) + ->Observe(value, {{label::metric, field}}); + }; + + auto const& ledgerMaster = app.getLedgerMaster(); + + // The pair that separates "slow" from "stuck". A tally climbing + // toward the target will get there; a tally flat below it never + // will, and no acquire or peer panel says which is happening. + observe( + lval::quorum_publish::trustedValidationTally, + ledgerMaster.getTrustedValidationTally()); + + // What the last gate evaluation actually required, as opposed to + // unl_quorum{quorum} which is what the trusted list configures. + // Already clamped against the SIZE_MAX "quorum disabled" + // sentinel by LedgerMaster, so this never wraps negative. + observe(lval::quorum_publish::quorumTarget, ledgerMaster.getQuorumTarget()); + + // One-shot: a value is the time the first ledger took to pass + // the gate, and 0 means it never has. Not a trend. + observe( + lval::quorum_publish::timeToFirstValidatedUs, + ledgerMaster.getTimeToFirstValidatedUs()); + + // Validated but not yet published. pubLedgerSeq_ was never + // exported, so this gap was not derivable from any other series. + observe(lval::quorum_publish::publishLag, ledgerMaster.getPublishLag()); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + // Silently skip if services are not yet ready. + } + }, + this); +} + #endif // XRPL_ENABLE_TELEMETRY // ----------------------------------------------------------------- @@ -1665,15 +2369,6 @@ MetricsRegistry::incrementValidationsChecked() #endif } -void -MetricsRegistry::incrementStateChanges() -{ -#ifdef XRPL_ENABLE_TELEMETRY - if (enabled_ && stateChangesCounter_) - stateChangesCounter_->Add(1); -#endif -} - void MetricsRegistry::incrementLedgerHistoryMismatch(std::string_view reason) { diff --git a/src/xrpld/telemetry/MetricsRegistry.h b/src/xrpld/telemetry/MetricsRegistry.h index 30b8b2ff514..ee77c9c9eea 100644 --- a/src/xrpld/telemetry/MetricsRegistry.h +++ b/src/xrpld/telemetry/MetricsRegistry.h @@ -36,7 +36,6 @@ * | +-- ledgers_closed_total * | +-- validations_sent_total * | +-- validations_checked_total - * | +-- state_changes_total * | +-- ledger_history_mismatch_total{reason} * | +-- txq_expired_total * | +-- txq_dropped_total{reason} @@ -62,7 +61,18 @@ * +-- State tracking (mode value, time in state) * +-- Storage detail (NuDB sizes) * +-- Validation agreement (1h/24h pct, counts) + * +-- UNL quorum (trusted keys vs required quorum) + * +-- Clock close offset (local clock skew) + * +-- Sync state (time to first FULL, network-ledger gate, + * | server stall seconds, ledgers behind network) + * +-- JobQueue saturation (running tasks vs worker threads vs backlog) + * +-- Peer ledger supply (how many peers can serve the needed sequence) + * +-- PeerFinder slot census (slots, attempts, fixed peers, address caches) + * +-- Amendment block (warned flag + seconds until the node stops validating) + * +-- Ledger quorum + publish (validation tally vs quorum target, + * | time to first validated, publish lag) * +-- jq_trans_overflow_total (observed from Overlay) + * +-- server_stall_events_total (observed from LoadManager) * * Control-flow for async gauges: * @@ -546,14 +556,6 @@ class MetricsRegistry void incrementValidationsChecked(); - /** - * Increment the state_changes_total counter. - * Called from NetworkOPsImp::setMode() when the server operating mode - * changes (e.g. CONNECTED -> SYNCING -> TRACKING -> FULL). - */ - void - incrementStateChanges(); - /** * Increment the ledger_history_mismatch_total counter for a reason. * Called from LedgerHistory::handleMismatch() once the mismatch has @@ -624,8 +626,33 @@ class MetricsRegistry /** * Observe the NodeStore I/O totals and the means derived from them. * + * Publishes the four cumulative totals (`node_reads_total`, + * `node_writes`, `node_reads_duration_us`, `node_writes_duration_us`) + * unconditionally, plus `read_mean_us` and `write_mean_us` derived from + * them via scaledMean(). `write_mean_us` is the signal for the "a node + * with a large existing database syncs slower than a fresh one" symptom: + * back-fill is write-bound, so no read-side reading can show it. All + * three concrete store paths time themselves through + * Database::recordStoreDuration(), so the write mean is live on an + * ordinary node. + * + * Gauge rather than histogram, deliberately. A histogram would give true + * percentiles, but it costs one Record() per node object on the + * store/fetch path, and one ledger write walks thousands of SHAMap + * nodes. This reads the existing atomics once per ~10 s tick and adds + * nothing to the hot path. Consequence, stated plainly: p99 is NOT + * obtainable from this signal. A histogram added later would also need an + * explicit-bucket View registered via addMicrosecondHistogramView(), + * because the SDK's default buckets top out at 10,000. + * * @param db NodeStore to read the counters from. * @param observe Sink for one `metric`-labelled value. + * + * @note The totals are monotonic and never reset, so a panel wanting + * current rather than since-boot latency divides the two rates. That is + * why the counts and duration totals are exported beside the means. + * @note A mean is omitted when its count is 0, so a dashboard shows a gap + * rather than a plausible-looking 0 us. */ static void observeNodeStoreTotals(node_store::Database& db, ObserveFn const& observe); @@ -784,10 +811,80 @@ class MetricsRegistry * ledger-acquisition stall counters. */ opentelemetry::nostd::shared_ptr nodeStoreGauge_; + /** + * Observable gauge for online-delete rotation state and its copy-forward + * write total. Publishes nothing on a node without `online_delete`, where + * the node store is not a rotating one. + */ + opentelemetry::nostd::shared_ptr + rotationStateGauge_; /** * Observable gauge for server-level health metrics (state, uptime, peers, etc.). */ opentelemetry::nostd::shared_ptr serverInfoGauge_; + /** + * Observable gauge for trusted UNL key count against the required quorum. + */ + opentelemetry::nostd::shared_ptr unlQuorumGauge_; + /** + * Observable gauge for the network close-time offset (local clock skew). + */ + opentelemetry::nostd::shared_ptr clockSkewGauge_; + /** + * Observable gauge for the sync-pipeline state signals (time to first + * FULL, network-ledger gate, server stall, ledgers behind the network). + */ + opentelemetry::nostd::shared_ptr syncStateGauge_; + /** + * ObservableCounter: server_stall_events_total — observed from + * LoadManager::getStallEventCount() (cumulative episode tally owned by the + * load-monitor thread). + */ + opentelemetry::nostd::shared_ptr + stallEventsObservable_; + /** + * Observable gauge for aggregate ledger-acquire progress (max missing state + * and tx nodes, received-data stash depth, in-flight acquire count). + */ + opentelemetry::nostd::shared_ptr + syncAcquireGauge_; + /** + * Observable gauge for the SHAMap tree-node cache hit rate, which is the + * memory layer above the node store's own hit ratio. + */ + opentelemetry::nostd::shared_ptr + shamapCacheHitRateGauge_; + /** + * Observable gauge for global worker-pool saturation: tasks in flight, + * configured worker threads, and total jobs queued. + */ + opentelemetry::nostd::shared_ptr + jobQueueSaturationGauge_; + /** + * Observable gauge for how much of the needed ledger range the connected + * peer set can actually serve. + */ + opentelemetry::nostd::shared_ptr + peerLedgerSupplyGauge_; + /** + * Observable gauge for PeerFinder slot occupancy, connection attempts, + * fixed peers and address-cache depth. + */ + opentelemetry::nostd::shared_ptr slotCensusGauge_; + /** + * Observable gauge for the amendment-block warning flag and the countdown + * to the amendment activating. + */ + opentelemetry::nostd::shared_ptr + amendmentBlockGauge_; + /** + * Observable gauge for the pre-accept quorum gate and the publish lag: + * the trusted-validation tally against the quorum it must reach, the + * time to the first fully-validated ledger, and how far publishing + * trails validation. + */ + opentelemetry::nostd::shared_ptr + ledgerQuorumPublishGauge_; /** * Observable gauge for build version info (label-based, value=1). */ @@ -862,11 +959,6 @@ class MetricsRegistry */ opentelemetry::nostd::unique_ptr> validationsCheckedCounter_; - /** - * Counter: state_changes_total — incremented on operating mode transitions. - */ - opentelemetry::nostd::unique_ptr> - stateChangesCounter_; /** * ObservableCounter: jq_trans_overflow_total — observed from * Overlay::getJqTransOverflow() (cumulative overflow tally owned by the overlay). @@ -951,6 +1043,8 @@ class MetricsRegistry // the exact `metric` label values it publishes. They read only their // arguments, so exposing them widens no state. + void + registerRotationStateGauge(); // Sync diagnostics: online_delete rotation void registerServerInfoGauge(); // Task 9.7a void @@ -975,6 +1069,344 @@ class MetricsRegistry registerValidationAgreementGauge(); // Task 7.15 void registerValidationTotalsCounters(); // gap-fill: lifetime agree/miss _total + + /** + * Register the `unl_quorum` gauge. + * + * Observes two series under the `metric` attribute: + * `trusted_keys` (ValidatorList::trustedKeyCount()) and `quorum` + * (ValidatorList::quorum()). Both are cheap accessors — one shared + * lock and one atomic load. + * + * `trusted_keys < quorum` means the node can never fully validate a + * ledger, so it will sit in `syncing` until the UNL is fixed. That + * makes this the first place to look when a node never leaves + * `syncing`. + * + * @note Pulled on the OTel reader thread (~10 s tick); does no work + * on any hot path. + */ + void + registerUnlQuorumGauge(); // sync diagnostics: UNL vs quorum + + /** + * Register the `clock_close_offset_seconds` gauge. + * + * Observes one series, `offset`, from + * TimeKeeper::closeOffset(): the seconds this node's notion of + * network close time is displaced from its own wall clock. + * + * The value MAY BE NEGATIVE, meaning the local clock runs ahead of + * the network. Whole-second resolution is all the signal carries, + * since that is the unit TimeKeeper stores. + * + * @note `server_info` only reports this field once |offset| >= 60 s + * (NetworkOPs), so this gauge is the first continuous export of it. + * Pulled on the OTel reader thread (~10 s tick); one atomic load. + */ + void + registerClockSkewGauge(); // sync diagnostics: close-time offset + + /** + * Register the `sync_state` gauge. + * + * One instrument fanning out four series under the `metric` attribute, + * each answering a different "why is this node not FULL yet?" question + * that previously existed only in a log line or in server_info JSON: + * + * `initial_full_duration_us` — microseconds from process start to the + * first FULL transition (NetworkOPs::getInitialSyncDurationUs()). + * Stays 0 until FULL is reached, so a flat 0 IS the "never synced" + * signal; once set it never changes again. + * `network_ledger_gate` — 1 while the node is still waiting to see a + * full network ledger (NetworkOPs::isNeedNetworkLedger()), else 0. A + * persistent 1 blocks transaction submission and FULL. + * `server_stall_seconds` — current main-loop stall duration + * (LoadManager::getCurrentStallSeconds()), 0 when healthy. + * `ledgers_behind` — network tip minus our validated sequence + * (NetworkOPs::getLedgersBehindNetwork()). + * + * The monotonic stall-episode count is a separate instrument + * (`server_stall_events_total`) because a counter and a gauge cannot share + * one instrument: Prometheus would otherwise see a cumulative total under + * last-value aggregation and `rate()` would be meaningless. + * + * @note Pulled on the OTel reader thread (~10 s tick), never on a hot + * path. Three of the four reads are a lock or atomic load; `ledgers_behind` + * additionally walks the connected-peer list, reading each peer's already + * cached ledger range — bounded by peer count and issuing no network I/O. + */ + void + registerSyncStateGauge(); // sync diagnostics: gate, stall, ledgers behind + + /** + * Register the `server_stall_events_total` observable counter. + * + * Observes LoadManager::getStallEventCount(): how many distinct stall + * episodes the monitor thread has reported since process start. Separate + * from `sync_state` because it is cumulative and monotonic, so it needs + * counter (not last-value) aggregation for `rate()` to mean anything. + * + * Read together with `sync_state{metric="server_stall_seconds"}`: a rising + * event count means repeated fresh stalls, while a flat count with a large + * stall-seconds value means one long unresolved stall. + * + * @note Pulled on the OTel reader thread (~10 s tick); one atomic load. + */ + void + registerStallEventsCounter(); // sync diagnostics: stall episode count + + /** + * Register the `sync_acquire` gauge. + * + * One instrument fanning out four series under the `metric` attribute, all + * from a single InboundLedgers::acquireProgress() snapshot: + * + * `missing_state_nodes_max` — largest outstanding account-state node count + * of any in-flight acquire. THE headline stuck-sync signal: flat and + * non-zero across ticks means the acquire will never finish, shrinking + * means it is slow but alive. + * `missing_tx_nodes_max` — the same for the transaction tree. + * `received_data_depth` — peer packets stashed across all acquires waiting + * to be applied. Deep means processing, not peer supply, is the limit. + * `in_flight` — how many acquires are running, so the three values above + * can be read in context: all zero with `in_flight` zero is idle, not + * healthy. + * + * Deliberately aggregated rather than per-ledger. A `ledger_seq` label would + * mint a new time series for every ledger the node ever acquires, which is + * unbounded cardinality; the max/sum keeps the "is it stuck?" answer while + * the per-ledger identity stays on the `ledger.acquire` span, where + * high-cardinality identity belongs. + * + * @note Pulled on the OTel reader thread (~10 s tick), never on a hot path. + * The snapshot takes the acquire-collection lock only to copy shared_ptrs, + * then reads relaxed atomics; the emit sites that feed those atomics all sit + * outside the per-tree-node loops. + */ + void + registerSyncAcquireGauge(); // sync diagnostics: acquire progress + + /** + * Register the `shamap_cache_hit_rate` gauge. + * + * Observes one series, `treenode`, from TreeNodeCache::getHitRate(): the + * percentage of SHAMap tree-node lookups served from memory instead of the + * node store. During a fresh sync a low rate means the node re-reads the + * same subtrees from disk, so sync is disk-bound rather than peer-bound. + * + * Distinct from the `NuDB Cache Hit Ratio` panel on the ledger-data-sync + * dashboard: that one is derived from `nodestore_state` and measures the + * node-store layer (`node_reads_hit / node_reads_total`). This gauge + * measures the in-memory tree-node cache that sits ABOVE it, so a request + * missing here is what produces a node-store read there. + * + * The full-below cache is deliberately NOT reported. It is a KeyCache, whose + * only lookup path is TaggedCache::touchIfExists(), and that method + * increments `stats_.hits`/`stats_.misses` while `getHitRate()` reads the + * separate `hits_`/`misses_` members. Its hit rate is therefore hard-wired + * to 0 regardless of behaviour, so exporting it would ship a permanently + * empty panel; fixing that accounting belongs in a libxrpl change of its own. + * + * @note Pulled on the OTel reader thread (~10 s tick). Takes the cache's + * mutex for two integer reads and a divide; no hot-path cost. + */ + void + registerCacheHitRateDetailGauge(); // sync diagnostics: treenode cache + + /** + * Register the `jobq_saturation` gauge. + * + * Three series under the `metric` attribute, from one + * JobQueue::getWorkerSaturation() reading: + * + * `running_tasks` — worker threads currently executing a job. + * `worker_threads` — threads the pool is configured to run, the + * denominator that makes `running_tasks` legible. Exported rather + * than hardcoded in the dashboard because it is derived at startup + * from `[workers]`, node size and hardware concurrency. + * `total_waiting` — jobs queued across all types. + * + * The reason this is separate from the per-job-type gauges JobQueue + * itself publishes (`jobq__waiting` / `_running` / `_deferred`): + * when the pool itself is exhausted, every subsystem waiting behind it + * looks independently slow, and each per-type panel invites the wrong + * conclusion. A `running_tasks / worker_threads` ratio at 1.0 with a + * non-zero `total_waiting` attributes the whole slowdown to pool + * exhaustion once. Those per-type gauges carry no capacity term at all, + * so no reading there can say whether the pool is the cause. + * + * @note Pulled on the OTel reader thread (~10 s tick). One atomic load, + * one plain int read, and one pass over the per-type counters under the + * JobQueue mutex. + */ + void + registerJobQueueSaturationGauge(); // sync diagnostics: pool saturation + + /** + * Register the `peer_ledger_supply` gauge. + * + * Five series under the `metric` attribute, from one + * Overlay::getPeerLedgerSupply() pass over the active peers: + * + * `peers_reporting` — peers that have advertised a ledger range at all. + * The denominator that makes the rest readable. + * `peers_serving_validated` — peers whose range covers this node's + * validated sequence. + * `peers_serving_next` — **the signal this gauge exists for.** Peers + * whose range covers validated + 1, the next ledger this node must + * acquire. Zero here with a non-zero `peers_reporting` means no + * connected peer holds what this node needs, so no amount of waiting + * will finish the sync; the peer set has to change. + * `supply_min_seq` / `supply_max_seq` — the sequence window the peer set + * covers, so an operator can see whether the node is asking for + * history nobody kept or for a tip nobody has reached. + * + * Each peer already caches the range it advertises in mtSTATUS_CHANGE + * (`PeerImp::minLedger_` / `maxLedger_`, read via `Peer::ledgerRange()`), + * but those ranges were never compared against each other, so "no peer has + * what I need" was indistinguishable from "my peers are slow" — the two + * faults with completely different fixes. + * + * Distinct from what already exists. `server_info{metric="peers"}` is a + * bare connection count with no notion of what those peers hold. + * `sync_state{metric="ledgers_behind"}` uses the same per-peer maxima but + * collapses them to a single distance-to-tip number, which cannot say how + * many peers can serve that distance or whether the range has a hole. + * `peer_quality{metric="peers_insane_count"}` counts peers on a different + * chain, which is a correctness signal, not an availability one. + * + * Peers advertising [0, 0] have not reported yet and are excluded from + * every field, so they cannot make a healthy peer set appear to serve from + * genesis. When nothing has reported, both window fields read 0, which is + * why `peers_reporting` must be read alongside them. + * + * @note Pulled on the OTel reader thread (~10 s tick), never on a message + * path. O(peers): `getActivePeers()` copies the peer list under the overlay + * lock and releases it, then each peer's cached range is read under that + * peer's own short-lived lock. + */ + void + registerPeerLedgerSupplyGauge(); // sync diagnostics: peer range coverage + + /** + * Register the `peerfinder_slot_census` gauge. + * + * Nine series under the `metric` attribute, from one + * Overlay::getSlotCensus() snapshot: `out_active`, `out_max`, `in_active`, + * `in_max`, `connecting`, `fixed_configured`, `fixed_active`, `bootcache` + * and `livecache`. + * + * All nine are already computed inside PeerFinder (`Counts`, `Bootcache`, + * `Livecache`, the fixed-peer map) and only two of them are exported + * today, as the legacy beast::insight gauges + * `peer_finder_active_inbound_peers` and + * `peer_finder_active_outbound_peers`. Those two carry no capacity, + * attempt or cache term, which leaves the three most common bootstrap + * failures invisible: + * + * - `connecting` non-zero while `out_active` stays below `out_max` — + * dials are being started and never completing. Without the attempt + * count this looks the same as a node that is not dialling at all. + * - `bootcache` at 0 — no seed addresses to dial in the first place. + * - `fixed_active` below `fixed_configured` — a peer named in the + * configuration is unreachable. + * + * The nine fields come from a single acquire of the PeerFinder lock, so + * they are mutually consistent and share one label set. The two legacy + * gauges are read at unrelated instants and cannot be joined with each + * other, let alone with a capacity term. + * + * @note Pulled on the OTel reader thread (~10 s tick). One lock acquire, + * then integer and container-size reads. + */ + void + registerSlotCensusGauge(); // sync diagnostics: peerfinder slot census + + /** + * Register the `amendment_block` gauge. + * + * Two series under the `metric` attribute: + * + * `warned` — 1 once an unsupported amendment has reached majority, from + * NetworkOPs::isAmendmentWarned(). + * `seconds_to_block` — **the leading indicator.** Seconds until that + * amendment activates, derived from + * `AmendmentTable::firstUnsupportedExpected()` against the network + * close time. `-1` when nothing is pending, matching the sentinel + * `validator_health{metric="unl_expiry_days"}` already uses, so the + * healthy state is a distinct value rather than a missing series. + * Clamped at 0 rather than going negative, because past-due means the + * block is imminent, not overdue by some amount. + * + * Amendment-blocked is a terminal sync blocker: the node stops validating + * and never resumes without a software upgrade. The existing + * `validator_health{metric="amendment_blocked"}` reports that state after + * it has happened, when nothing can be done about it. This gauge is the + * window before it, which is the only actionable part. + * + * The blocking amendment's identity is deliberately NOT a label. The + * network can vote on an arbitrary 256-bit amendment id — the set is not + * drawn from this build's known features — so an id label would be + * unbounded cardinality and would mint a permanent new series per + * amendment. The id is already logged by + * `AmendmentTableImpl::doValidatedLedger` ("Unsupported amendment + * reached majority at ..."), so it is available through logs, correlated + * to this series by node and time. + * + * @note Pulled on the OTel reader thread (~10 s tick). One mutex acquire + * inside the amendment table plus one clock read. + * @note The subtraction is done in `std::int64_t`, not in NetClock's + * unsigned representation, so a past-due activation cannot wrap to a huge + * positive count. + */ + void + registerAmendmentBlockGauge(); // sync diagnostics: amendment countdown + + /** + * Register the `ledger_quorum_publish` gauge. + * + * Four series under the `metric` attribute, read from LedgerMaster: + * + * `trusted_validation_tally` — trusted validations counted at the last + * pre-accept gate in `LedgerMaster::checkAccept`. + * `quorum_target` — validations that gate required. **The pair is the + * signal.** The tally alone cannot separate a node accumulating + * validations toward quorum (slow, will finish) from one whose tally + * plateaus below the target (stuck, never will); with the target + * beside it, the two shapes are unmistakable. + * `time_to_first_validated_us` — how long the node took to get its + * first ledger through that gate. One-shot, like the + * `sync_state{initial_full_duration_us}` milestone: a value means it + * happened and this is how long it took, 0 means it never has. + * `publish_lag` — validated sequence minus published sequence. Non-zero + * and growing means validation is fine and the publish pipeline is + * behind, which no other signal distinguishes. + * + * All four are grouped under one instrument because they answer one + * question in sequence — did enough validations arrive, did the gate pass, + * how long did that take, and did the result reach clients — so an + * operator reads them from a single consistent poll. + * + * Distinct from what already exists. `unl_quorum{quorum}` is the quorum + * the validator list *configures*, a static property of the trusted set; + * `quorum_target` is what an actual gate evaluation *required*, and the + * tally beside it is the live count that must reach it — neither existed + * anywhere before. `server_info{validated_ledger_seq}` publishes the + * validated sequence but nothing published the pubLedgerSeq_ counterpart, + * so the lag between them was not derivable at all. + * + * @note `quorum_target` reports int64 max when the validator list has + * switched quorum off (`ValidatorList::quorum()` returns SIZE_MAX). The + * clamp lives in `LedgerMaster::checkAccept`, so the wrap to -1 that would + * invert a tally-versus-target panel cannot happen here. + * @note Pulled on the OTel reader thread (~10 s tick). Five relaxed atomic + * loads through lock-free LedgerMaster accessors: no lock is taken, which + * is what keeps an OTel callback from ever contending with, or inverting + * lock order against, the LedgerMaster mutex held by the emit path. + */ + void + registerLedgerQuorumPublishGauge(); // sync diagnostics: quorum + publish #endif // XRPL_ENABLE_TELEMETRY };