[codex] Harden observability evaluation validators

.agents/settings.json

@@ -0,0 +1,5 @@
+{
+  "enabledPlugins": {
+    "railway@claude-plugins-official": true
+  }
+}

evals/Qyl.Observability.Evaluation.Tests/CardinalitySafetyEvaluatorTests.cs

@@ -0,0 +1,61 @@
+using Microsoft.Extensions.AI;
+using Microsoft.Extensions.AI.Evaluation;
+using Qyl.Observability.Evaluation.Evaluators;
+
+namespace Qyl.Observability.Evaluation.Tests;
+
+public sealed class CardinalitySafetyEvaluatorTests
+{
+    [Fact]
+    public void Analyze_PassesSafeBoundedAttributes()
+    {
+        var record = EvaluatorTestData.Record(
+            telemetry: [EvaluatorTestData.Span("span", "trace-1", "span-1", attributesJson: """{"http.route":"/checkout","error.type":"TimeoutException"}""")]);
+
+        AnalysisResult result = CardinalitySafetyEvaluator.Analyze(record);
+
+        Assert.True(result.Passed, result.Reason);
+    }
+
+    [Fact]
+    public void Analyze_FailsBlockedAttributeKey()
+    {
+        var record = EvaluatorTestData.Record(
+            telemetry: [EvaluatorTestData.Span("span", "trace-1", "span-1", attributesJson: """{"qyl.prompt.raw":"diagnose alice@example.com"}""")]);
+
+        AnalysisResult result = CardinalitySafetyEvaluator.Analyze(record);
+
+        Assert.False(result.Passed);
+        Assert.Contains("span:qyl.prompt.raw", result.Reason, StringComparison.Ordinal);
+    }
+
+    [Theory]
+    [InlineData("""{"user.id":"alice@example.com"}""")]
+    [InlineData("""{"auth.header":"Bearer token"}""")]
+    [InlineData("""{"openai.key":"sk-test"}""")]
+    public void Analyze_FailsSensitiveAttributeValues(string attributesJson)
+    {
+        var record = EvaluatorTestData.Record(
+            telemetry: [EvaluatorTestData.Span("span", "trace-1", "span-1", attributesJson: attributesJson)]);
+
+        AnalysisResult result = CardinalitySafetyEvaluator.Analyze(record);
+
+        Assert.False(result.Passed);
+    }
+
+    [Fact]
+    public async Task EvaluateAsync_ReturnsFailedMetricForUnsafeAttribute()
+    {
+        var record = EvaluatorTestData.Record(
+            telemetry: [EvaluatorTestData.Span("span", "trace-1", "span-1", attributesJson: """{"message.raw":"secret"}""")]);
+
+        EvaluationResult result = await new CardinalitySafetyEvaluator(record).EvaluateAsync(
+            [],
+            new ChatResponse(new ChatMessage(ChatRole.Assistant, "")),
+            cancellationToken: TestContext.Current.CancellationToken);
+
+        BooleanMetric metric = result.Get<BooleanMetric>("qyl.cardinality.safety");
+        Assert.False(metric.Value);
+        Assert.True(metric.Interpretation?.Failed);
+    }
+}

evals/Qyl.Observability.Evaluation.Tests/EvaluatorTestData.cs

@@ -0,0 +1,100 @@
+using System.Text.Json;
+using Qyl.Observability.Evaluation.Models;
+
+namespace Qyl.Observability.Evaluation.Tests;
+
+internal static class EvaluatorTestData
+{
+    public static ObservabilityEvaluationRecord Record(
+        string finalResponse = "",
+        IReadOnlyList<ToolCallRecord>? toolCalls = null,
+        IReadOnlyList<ExpectedToolCallRecord>? expectedToolCalls = null,
+        IReadOnlyList<TelemetryEvidenceRecord>? telemetry = null,
+        IReadOnlyList<string>? requiredEvidenceIds = null,
+        IReadOnlyList<string>? forbiddenClaims = null,
+        IReadOnlyList<string>? expectedFailedMetrics = null,
+        bool shouldPass = true)
+    {
+        return new ObservabilityEvaluationRecord
+        {
+            Id = "record",
+            Source = "tests",
+            Scenario = "scenario",
+            Agent = new AgentInfo
+            {
+                Name = "QylIncidentAgent",
+                ModelProvider = "fixture",
+                ModelName = "deterministic",
+                Instructions = "Use qyl observability tools.",
+                Tools = ["qyl.query.traces", "qyl.query.metrics", "qyl.query.logs"]
+            },
+            UserInput = "diagnose incident",
+            ToolCalls = toolCalls ?? [],
+            FinalResponse = finalResponse,
+            Telemetry = telemetry ?? [],
+            RequiredEvidenceIds = requiredEvidenceIds ?? [],
+            ForbiddenClaims = forbiddenClaims ?? [],
+            ExpectedToolCalls = expectedToolCalls ?? [],
+            ExpectedFailedMetrics = expectedFailedMetrics ?? [],
+            ShouldPass = shouldPass
+        };
+    }
+
+    public static ToolCallRecord ToolCall(string name, string argumentsJson = "{}")
+    {
+        return new ToolCallRecord
+        {
+            Name = name,
+            Arguments = ParseObject(argumentsJson),
+            ResultSummary = "ok"
+        };
+    }
+
+    public static ExpectedToolCallRecord ExpectedToolCall(string name, string argumentsJson = "{}")
+    {
+        return new ExpectedToolCallRecord
+        {
+            Name = name,
+            Arguments = ParseObject(argumentsJson)
+        };
+    }
+
+    public static TelemetryEvidenceRecord Span(
+        string id,
+        string? traceId,
+        string? spanId,
+        string? parentSpanId = null,
+        string attributesJson = "{}")
+    {
+        return new TelemetryEvidenceRecord
+        {
+            Id = id,
+            SignalType = "span",
+            ServiceName = "checkout-api",
+            Operation = "POST /checkout",
+            TraceId = traceId,
+            SpanId = spanId,
+            ParentSpanId = parentSpanId,
+            Attributes = ParseObject(attributesJson)
+        };
+    }
+
+    public static TelemetryEvidenceRecord Metric(string id)
+    {
+        return new TelemetryEvidenceRecord
+        {
+            Id = id,
+            SignalType = "metric",
+            ServiceName = "checkout-api",
+            Operation = "http.server.duration",
+            Attributes = ParseObject("""{"metric":"http.server.duration"}""")
+        };
+    }
+
+    private static IReadOnlyDictionary<string, JsonElement> ParseObject(string json)
+    {
+        using JsonDocument document = JsonDocument.Parse(json);
+        return document.RootElement.EnumerateObject()
+            .ToDictionary(static property => property.Name, static property => property.Value.Clone(), StringComparer.Ordinal);
+    }
+}

evals/Qyl.Observability.Evaluation.Tests/Qyl.Observability.Evaluation.Tests.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="ANcpLua.NET.Sdk.Test">
+
+  <PropertyGroup>
+    <RootNamespace>Qyl.Observability.Evaluation.Tests</RootNamespace>
+    <OutputType>Exe</OutputType>
+    <GenerateDocumentationFile>false</GenerateDocumentationFile>
+    <NoWarn>$(NoWarn);CS1591;CA1707</NoWarn>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Qyl.Observability.Evaluation\Qyl.Observability.Evaluation.csproj"/>
+  </ItemGroup>
+
+</Project>

evals/Qyl.Observability.Evaluation.Tests/RunnerAndLoaderTests.cs

@@ -0,0 +1,91 @@
+using Microsoft.Extensions.AI.Evaluation;
+using Qyl.Observability.Evaluation;
+using Qyl.Observability.Evaluation.Evaluators;
+
+namespace Qyl.Observability.Evaluation.Tests;
+
+public sealed class RunnerAndLoaderTests
+{
+    [Fact]
+    public void ScenarioRunResult_FailsWhenPassRecordDeclaresExpectedFailures()
+    {
+        var record = EvaluatorTestData.Record(expectedFailedMetrics: ["qyl.trace.correlation"], shouldPass: true);
+
+        ScenarioRunResult result = ScenarioRunResult.Create(record, [PassedMetric("qyl.trace.correlation")]);
+
+        Assert.False(result.Passed);
+        Assert.Contains("pass record declares expected failed metrics", result.Mismatches);
+    }
+
+    [Fact]
+    public void ScenarioRunResult_FailsWhenFailRecordDeclaresNoExpectedFailures()
+    {
+        var record = EvaluatorTestData.Record(shouldPass: false);
+
+        ScenarioRunResult result = ScenarioRunResult.Create(record, [FailedMetric("qyl.trace.correlation")]);
+
+        Assert.False(result.Passed);
+        Assert.Contains("fail record declares no expected failed metrics", result.Mismatches);
+    }
+
+    [Fact]
+    public void ScenarioRunResult_FailsWhenActualFailedMetricsDifferFromExpected()
+    {
+        var record = EvaluatorTestData.Record(expectedFailedMetrics: ["qyl.tool.call.accuracy"], shouldPass: false);
+
+        ScenarioRunResult result = ScenarioRunResult.Create(record, [FailedMetric("qyl.trace.correlation")]);
+
+        Assert.False(result.Passed);
+        Assert.Contains("failed metrics [qyl.trace.correlation] != expected [qyl.tool.call.accuracy]", result.Mismatches);
+    }
+
+    [Fact]
+    public async Task EvaluationRunner_MatchesExpectedFailedMetrics()
+    {
+        var records = new[]
+        {
+            EvaluatorTestData.Record(
+                finalResponse: "Evidence ev-metric supports this.",
+                telemetry: [EvaluatorTestData.Metric("ev-metric")],
+                requiredEvidenceIds: ["ev-metric"],
+                shouldPass: true),
+            EvaluatorTestData.Record(
+                finalResponse: "No citation.",
+                requiredEvidenceIds: ["ev-missing"],
+                expectedFailedMetrics: ["qyl.telemetry.evidence"],
+                shouldPass: false)
+        };
+
+        IReadOnlyList<ScenarioRunResult> results = await EvaluationRunner.RunAsync(records, TestContext.Current.CancellationToken);
+
+        Assert.All(results, static result => Assert.True(result.Passed, string.Join("; ", result.Mismatches)));
+    }
+
+    [Fact]
+    public void ScenarioLoader_LoadsJsonlAndSkipsBlankLines()
+    {
+        string path = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid():N}.jsonl");
+        File.WriteAllText(path, """
+{"id":"fixture","source":"tests","scenario":"load","agent":{"name":"agent","modelProvider":"fixture","modelName":"model","instructions":"instructions","tools":[]},"userInput":"input","toolCalls":[],"finalResponse":"response","telemetry":[],"requiredEvidenceIds":[],"forbiddenClaims":[],"expectedToolCalls":[],"expectedFailedMetrics":[],"shouldPass":true}
+
+""");
+
+        try
+        {
+            var records = ScenarioLoader.LoadJsonl(path);
+
+            Assert.Single(records);
+            Assert.Equal("fixture", records[0].Id);
+        }
+        finally
+        {
+            File.Delete(path);
+        }
+    }
+
+    private static BooleanMetric PassedMetric(string name)
+        => EvaluationMetricFactory.CreateBoolean(name, true, "passed");
+
+    private static BooleanMetric FailedMetric(string name)
+        => EvaluationMetricFactory.CreateBoolean(name, false, "failed");
+}

evals/Qyl.Observability.Evaluation.Tests/TelemetryEvidenceEvaluatorTests.cs

@@ -0,0 +1,57 @@
+using Microsoft.Extensions.AI;
+using Microsoft.Extensions.AI.Evaluation;
+using Qyl.Observability.Evaluation.Evaluators;
+
+namespace Qyl.Observability.Evaluation.Tests;
+
+public sealed class TelemetryEvidenceEvaluatorTests
+{
+    [Fact]
+    public void Analyze_PassesWhenEvidenceExistsIsCitedAndForbiddenClaimsAreAbsent()
+    {
+        var record = EvaluatorTestData.Record(
+            finalResponse: "Evidence ev-metric-latency explains the spike.",
+            telemetry: [EvaluatorTestData.Metric("ev-metric-latency")],
+            requiredEvidenceIds: ["ev-metric-latency"],
+            forbiddenClaims: ["database saturation"]);
+
+        AnalysisResult result = TelemetryEvidenceEvaluator.Analyze(record);
+
+        Assert.True(result.Passed, result.Reason);
+    }
+
+    [Fact]
+    public void Analyze_FailsMissingTelemetryMissingCitationAndForbiddenClaim()
+    {
+        var record = EvaluatorTestData.Record(
+            finalResponse: "The issue is database saturation.",
+            telemetry: [EvaluatorTestData.Metric("ev-present")],
+            requiredEvidenceIds: ["ev-missing", "ev-present"],
+            forbiddenClaims: ["database saturation"]);
+
+        AnalysisResult result = TelemetryEvidenceEvaluator.Analyze(record);
+
+        Assert.False(result.Passed);
+        Assert.Contains("missing telemetry: ev-missing", result.Reason, StringComparison.Ordinal);
+        Assert.Contains("missing citations: ev-missing, ev-present", result.Reason, StringComparison.Ordinal);
+        Assert.Contains("forbidden claims: database saturation", result.Reason, StringComparison.Ordinal);
+    }
+
+    [Fact]
+    public async Task EvaluateAsync_ReturnsPassedMetricForGroundedAnswer()
+    {
+        var record = EvaluatorTestData.Record(
+            finalResponse: "Evidence ev-log-error supports the conclusion.",
+            telemetry: [EvaluatorTestData.Metric("ev-log-error")],
+            requiredEvidenceIds: ["ev-log-error"]);
+
+        EvaluationResult result = await new TelemetryEvidenceEvaluator(record).EvaluateAsync(
+            [],
+            new ChatResponse(new ChatMessage(ChatRole.Assistant, record.FinalResponse)),
+            cancellationToken: TestContext.Current.CancellationToken);
+
+        BooleanMetric metric = result.Get<BooleanMetric>("qyl.telemetry.evidence");
+        Assert.True(metric.Value);
+        Assert.False(metric.Interpretation?.Failed);
+    }
+}