Skip to content

feat: support log sampling and remove unused method #76

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jaronoff97 merged 2 commits into from
Jul 7, 2026
Merged

feat: support log sampling and remove unused method #76

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
41e6aff
support log sampling
jaronoff97 Jul 7, 2026
e439ea7
remove old accessor
jaronoff97 Jul 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 21 additions & 10 deletions proto/tero/policy/v1/log.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ message LogTarget {
// Valid values:
// "all" - Keep everything (default, can be omitted)
// "none" - Drop everything
// "N%" - Keep N percent (0-100), e.g. "50%"
// "N%" - Keep approximately N percent (0-100), e.g. "50%"
// "N/s" - Keep at most N per second (shorthand for "N/1s"), e.g. "100/s"
// "N/m" - Keep at most N per minute (shorthand for "N/1m"), e.g. "1000/m"
// "N/Ds" - Keep at most N per D-second window, e.g. "1/5s"
Expand All @@ -41,6 +41,11 @@ message LogTarget {
// keep/drop decision. Use for lifecycle events (request_id, trace_id, job_id)
// to avoid sampling individual log lines independently.
//
// For percentage sampling (N%), omitting sample_key means each matching log is
// sampled independently using a random value generated at evaluation time.
// That default gives the expected approximate distribution, but is not
// idempotent across repeated evaluation of the same log sequence.
//
// Only applies when keep is a sampling value (N%, N/s, N/m, N/Ds, N/Dm).
// Example: sample_key = log_attribute["request_id"] with keep = "10%" means
// 10% of requests are kept, with all logs from each kept request preserved.
Expand Down Expand Up @@ -79,8 +84,8 @@ enum LogField {
LOG_FIELD_BODY = 1;
LOG_FIELD_SEVERITY_TEXT = 2;
// trace_id and span_id are bytes. They are authored as lowercase hex and
// matched as raw bytes (exact/equals), or as their hex rendering for string
// match types. See the Value message and the spec's
// matched as raw bytes with equals.hex_value, or as their hex rendering for
// string pattern match types. See the Value message and the spec's
// "Bytes and Identifier Fields" section.
LOG_FIELD_TRACE_ID = 3;
LOG_FIELD_SPAN_ID = 4;
Expand Down Expand Up @@ -124,12 +129,17 @@ message LogMatcher {

// Match type. Exactly one must be set.
//
// The string match types (exact, regex, starts_with, ends_with, contains)
// operate only on string field values. To match non-string values, use the
// typed equals matcher or the numeric comparison matchers (gt, gte, lt, lte).
// Use the typed equals matcher for equality. The exact field is deprecated and
// will move to reserved in a future version after wire-format users migrate to
// equals.string_value.
//
// The string pattern match types (regex, starts_with, ends_with, contains)
// operate only on string field values. Use the numeric comparison matchers
// (gt, gte, lt, lte) for ranges.
oneof match {
// Exact string match (string field values only)
string exact = 10;
// Deprecated: use equals.string_value instead. This field will move to
// reserved in a future version.
string exact = 10 [deprecated = true];

// Regular expression match (string field values only)
string regex = 11;
Expand All @@ -146,7 +156,7 @@ message LogMatcher {
// Literal substring match (string field values only)
string contains = 15;

// Typed equality for non-string field values (bool, int, double, bytes)
// Typed equality for string, bool, int, double, or bytes field values.
Value equals = 22;

// Numeric greater-than comparison (int/double field values)
Expand All @@ -165,7 +175,8 @@ message LogMatcher {
// If true, inverts the match result
bool negate = 20;

// If true, applies case-insensitive matching to all match types
// If true, applies case-insensitive matching to equals.string_value and the
// string pattern match types.
bool case_insensitive = 21;
}

Expand Down
20 changes: 13 additions & 7 deletions proto/tero/policy/v1/metric.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,12 +100,17 @@ message MetricMatcher {
// Match type. Exactly one must be set.
// Note: For metric_type field, only exists is valid (type equality is implicit).
//
// The string match types (exact, regex, starts_with, ends_with, contains)
// operate only on string field values. To match non-string values, use the
// typed equals matcher or the numeric comparison matchers (gt, gte, lt, lte).
// Use the typed equals matcher for equality. The exact field is deprecated and
// will move to reserved in a future version after wire-format users migrate to
// equals.string_value.
//
// The string pattern match types (regex, starts_with, ends_with, contains)
// operate only on string field values. Use the numeric comparison matchers
// (gt, gte, lt, lte) for ranges.
oneof match {
// Exact string match (string field values only)
string exact = 10;
// Deprecated: use equals.string_value instead. This field will move to
// reserved in a future version.
string exact = 10 [deprecated = true];

// Regular expression match (string field values only)
string regex = 11;
Expand All @@ -122,7 +127,7 @@ message MetricMatcher {
// Literal substring match (string field values only)
string contains = 15;

// Typed equality for non-string field values (bool, int, double, bytes)
// Typed equality for string, bool, int, double, or bytes field values.
Value equals = 22;

// Numeric greater-than comparison (int/double field values)
Expand All @@ -141,6 +146,7 @@ message MetricMatcher {
// If true, inverts the match result
bool negate = 20;

// If true, applies case-insensitive matching to all match types
// If true, applies case-insensitive matching to equals.string_value and the
// string pattern match types.
bool case_insensitive = 21;
}
11 changes: 6 additions & 5 deletions proto/tero/policy/v1/shared.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,7 @@ message AttributePath {
repeated string path = 1;
}

// Value carries a typed, non-string scalar for the `equals` matcher. String
// equality is expressed with the `exact` match type, so this message
// intentionally has no string variant.
// Value carries a typed scalar for the `equals` matcher.
//
// `equals` matches when the field value has the same type and value as the set
// variant. Integer and floating-point values are compared in a single numeric
Expand All @@ -68,14 +66,16 @@ message AttributePath {
// equals: true -> bool_value
// equals: 200 -> int_value
// equals: 0.5 -> double_value
// equals: "foo" -> string_value
//
// Bytes are authored either as proto-native base64 (`bytes_value`) or, more
// readably, as a lowercase-hex string (`hex_value`). The two are equivalent —
// hex_value is decoded to bytes at policy-compile time and yields the same bytes
// as the corresponding bytes_value — but hex_value keeps identifiers
// (trace/span ids) readable in the canonical proto/JSON form instead of base64.
// A bare string literal (e.g. `equals: "foo"`) MUST be rejected — use `exact`
// for strings.
//
// String equality SHOULD be authored with string_value. The older matcher-level
// `exact` field is deprecated and will move to reserved in a future version.
message Value {
// Exactly one must be set.
oneof value {
Expand All @@ -88,6 +88,7 @@ message Value {
// input, canonically lowercase). Decoded to bytes at policy-compile time;
// an invalid hex string MUST be rejected.
string hex_value = 5;
string string_value = 6;
}
}

Expand Down
26 changes: 16 additions & 10 deletions proto/tero/policy/v1/trace.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,9 @@ enum TraceField {
// Span fields
TRACE_FIELD_NAME = 1;
// trace_id, span_id, and parent_span_id are bytes. They are authored as
// lowercase hex and matched as raw bytes (exact/equals), or as their hex
// rendering for string match types. See the Value message and the spec's
// "Bytes and Identifier Fields" section.
// lowercase hex and matched as raw bytes with equals.hex_value, or as their
// hex rendering for string pattern match types. See the Value message and the
// spec's "Bytes and Identifier Fields" section.
TRACE_FIELD_TRACE_ID = 2;
TRACE_FIELD_SPAN_ID = 3;
TRACE_FIELD_PARENT_SPAN_ID = 4;
Expand Down Expand Up @@ -119,12 +119,17 @@ message TraceMatcher {
// Match type. Exactly one must be set.
// Note: For span_kind and span_status fields, only exists is valid (equality is implicit).
//
// The string match types (exact, regex, starts_with, ends_with, contains)
// operate only on string field values. To match non-string values, use the
// typed equals matcher or the numeric comparison matchers (gt, gte, lt, lte).
// Use the typed equals matcher for equality. The exact field is deprecated and
// will move to reserved in a future version after wire-format users migrate to
// equals.string_value.
//
// The string pattern match types (regex, starts_with, ends_with, contains)
// operate only on string field values. Use the numeric comparison matchers
// (gt, gte, lt, lte) for ranges.
oneof match {
// Exact string match (string field values only)
string exact = 10;
// Deprecated: use equals.string_value instead. This field will move to
// reserved in a future version.
string exact = 10 [deprecated = true];

// Regular expression match (string field values only)
string regex = 11;
Expand All @@ -141,7 +146,7 @@ message TraceMatcher {
// Literal substring match (string field values only)
string contains = 15;

// Typed equality for non-string field values (bool, int, double, bytes)
// Typed equality for string, bool, int, double, or bytes field values.
Value equals = 22;

// Numeric greater-than comparison (int/double field values)
Expand All @@ -160,7 +165,8 @@ message TraceMatcher {
// If true, inverts the match result
bool negate = 20;

// If true, applies case-insensitive matching to all match types
// If true, applies case-insensitive matching to equals.string_value and the
// string pattern match types.
bool case_insensitive = 21;
}

Expand Down
19 changes: 10 additions & 9 deletions src/bench/main.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ const TraceFieldRef = policy_zig.TraceFieldRef;
const LogAccessor = policy_zig.LogAccessor;
const MetricAccessor = policy_zig.MetricAccessor;
const TraceAccessor = policy_zig.TraceAccessor;
const TypedValue = policy_zig.TypedValue;
const NoopEventBus = o11y.NoopEventBus;

const COUNTS = [_]usize{ 1, 10, 100, 1000 };
Expand All @@ -24,38 +25,38 @@ const COUNTS = [_]usize{ 1, 10, 100, 1000 };

const BenchLog = struct {
body: []const u8,
fn access(ctx: *const anyopaque, field: FieldRef) ?[]const u8 {
fn access(ctx: *const anyopaque, field: FieldRef) ?TypedValue {
const self: *const BenchLog = @ptrCast(@alignCast(ctx));
return switch (field) {
.log_field => |lf| if (lf == .LOG_FIELD_BODY) self.body else null,
.log_field => |lf| if (lf == .LOG_FIELD_BODY) .{ .string = self.body } else null,
else => null,
};
}
const accessor: LogAccessor = .{ .value = access };
const accessor: LogAccessor = .{ .typed_value = access };
};

const BenchMetric = struct {
name: []const u8,
fn access(ctx: *const anyopaque, field: MetricFieldRef) ?[]const u8 {
fn access(ctx: *const anyopaque, field: MetricFieldRef) ?TypedValue {
const self: *const BenchMetric = @ptrCast(@alignCast(ctx));
return switch (field) {
.metric_field => |mf| if (mf == .METRIC_FIELD_NAME) self.name else null,
.metric_field => |mf| if (mf == .METRIC_FIELD_NAME) .{ .string = self.name } else null,
else => null,
};
}
const accessor: MetricAccessor = .{ .value = access };
const accessor: MetricAccessor = .{ .typed_value = access };
};

const BenchTrace = struct {
name: []const u8,
fn access(ctx: *const anyopaque, field: TraceFieldRef) ?[]const u8 {
fn access(ctx: *const anyopaque, field: TraceFieldRef) ?TypedValue {
const self: *const BenchTrace = @ptrCast(@alignCast(ctx));
return switch (field) {
.trace_field => |tf| if (tf == .TRACE_FIELD_NAME) self.name else null,
.trace_field => |tf| if (tf == .TRACE_FIELD_NAME) .{ .string = self.name } else null,
else => null,
};
}
const accessor: TraceAccessor = .{ .value = access };
const accessor: TraceAccessor = .{ .typed_value = access };
};

// --- JSON builders (allocator passed per-call, Zig 0.16 style) ---
Expand Down
25 changes: 15 additions & 10 deletions src/policy/log_transform.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,10 +202,9 @@ pub fn applyRemove(
///
/// When `options.compiled` is null, replaces the entire field value with
/// `rule.replacement`. When non-null, runs the pre-compiled regex over the
/// textual representation returned by `accessor.value` and substitutes each
/// match using the pre-parsed replacement template. If the accessor returns
/// null (field missing or non-string), or the regex finds no match, the
/// operation is a no-op.
/// `.string` value returned by `accessor.typed_value` and substitutes each
/// match using the pre-parsed replacement template. If the field is missing
/// or non-string, or the regex finds no match, the operation is a no-op.
///
/// Returns true if the field was redacted.
pub fn applyRedact(
Expand All @@ -220,7 +219,9 @@ pub fn applyRedact(
if (options.compiled) |c| {
const allocator = options.scratch orelse return false;
const io = options.io orelse return false;
const value = accessor.value(ctx, field_ref) orelse return false;
const tv = accessor.typed_value(ctx, field_ref) orelse return false;

@macroscopeapp macroscopeapp Bot Jul 7, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Medium policy/log_transform.zig:222

Regex redaction in applyRedact now bails out when accessor.typed_value returns a non-.string value, so a regex redact rule targeting a field exposed as .bytes (e.g. LOG_FIELD_TRACE_ID / LOG_FIELD_SPAN_ID) always returns false and leaves the field unchanged. Previously accessor.value supplied a textual representation that the regex could still match. Consider coercing non-string typed values to text before running the regex, or document that regex redaction intentionally excludes .bytes fields.

🚀 Reply "fix it for me" or copy this AI Prompt for your agent: 
In file @src/policy/log_transform.zig around line 222:

Regex redaction in `applyRedact` now bails out when `accessor.typed_value` returns a non-`.string` value, so a regex redact rule targeting a field exposed as `.bytes` (e.g. `LOG_FIELD_TRACE_ID` / `LOG_FIELD_SPAN_ID`) always returns `false` and leaves the field unchanged. Previously `accessor.value` supplied a textual representation that the regex could still match. Consider coercing non-string typed values to text before running the regex, or document that regex redaction intentionally excludes `.bytes` fields.

if (tv != .string) return false; // regex redact no-ops on non-string values
const value = tv.string;

var out: std.ArrayList(u8) = .empty;
// No deinit: `out.items` is handed to the accessor.set by reference
Expand All @@ -243,7 +244,7 @@ pub fn applyRedact(
}

// Whole-value redact: only fires if the field exists.
if (accessor.value(ctx, field_ref) == null) return false;
if (!accessor.callExists(ctx, field_ref)) return false;

accessor.set.?(ctx, field_ref, rule.replacement);
return true;
Expand Down Expand Up @@ -414,8 +415,12 @@ const TestContext = struct {
self.allocator.free(removed.value);
}

fn accessorTypedValue(ctx: *const anyopaque, field: FieldRef) ?types.TypedValue {
return .{ .string = accessorValue(ctx, field) orelse return null };
}

const accessor: LogAccessor = .{
.value = accessorValue,
.typed_value = accessorTypedValue,
.set = accessorSet,
.delete = accessorDelete,
.move = accessorMove,
Expand Down Expand Up @@ -568,10 +573,10 @@ test "applyRedact: regex output outlives applyRedact return" {

stored: ?[]const u8 = null,

fn valueFn(ctx: *const anyopaque, field: FieldRef) ?[]const u8 {
fn valueFn(ctx: *const anyopaque, field: FieldRef) ?types.TypedValue {
_ = field;
const self: *const Self = @ptrCast(@alignCast(ctx));
return self.stored;
return .{ .string = self.stored orelse return null };
}

fn setFn(ctx: *anyopaque, field: FieldRef, value: []const u8) void {
Expand All @@ -585,7 +590,7 @@ test "applyRedact: regex output outlives applyRedact return" {
}
fn moveFn(_: *anyopaque, _: FieldRef, _: []const u8) void {}
const accessor: LogAccessor = .{
.value = valueFn,
.typed_value = valueFn,
.set = setFn,
.delete = deleteFn,
.move = moveFn,
Expand Down
Loading
Loading