Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion packages/opencode/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@
"@ai-sdk/togetherai": "2.0.41",
"@ai-sdk/vercel": "2.0.39",
"@ai-sdk/xai": "3.0.82",
"@altimateai/altimate-core": "0.5.1",
"@altimateai/altimate-core": "0.7.0",
"@altimateai/drivers": "workspace:*",
"@aws-sdk/credential-providers": "3.1057.0",
"@clack/prompts": "1.0.0-alpha.1",
Expand Down
34 changes: 23 additions & 11 deletions packages/opencode/src/altimate/native/altimate-core.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
*/

import * as core from "@altimateai/altimate-core"
import { dialectHint } from "./engine-coerce"
import { register } from "./dispatcher"
import { schemaOrEmpty, resolveSchema } from "./schema-resolver"
import type { AltimateCoreResult } from "./types"
Expand Down Expand Up @@ -181,10 +182,20 @@ export function registerAll(): void {
)
: core.lint(params.sql, schema)
const safety = core.scanSql(params.sql)
// PII exposure for the composite check — the tool has always rendered a
// PII section; previously nothing populated it. Additive: a PII failure
// must not fail the whole composite.
let pii: Record<string, unknown> = {}
try {
pii = toData(core.checkQueryPii(params.sql, schema))
Comment thread
anandgupta42 marked this conversation as resolved.
} catch {
Comment thread
anandgupta42 marked this conversation as resolved.
Outdated
// validation/lint above already report unparseable SQL
}
const data: Record<string, unknown> = {
validation: toData(validation),
lint: toData(lintResult),
safety: toData(safety),
pii,
Comment thread
anandgupta42 marked this conversation as resolved.
}
return ok(true, data)
} catch (e) {
Expand Down Expand Up @@ -323,10 +334,10 @@ export function registerAll(): void {
// Pass the optional dialect hint so dialect-specific compiled warehouse SQL
// (e.g. Snowflake semi-structured `col:field`) parses and the pair is
// decidable instead of abstaining on a syntax error. Supported since
// altimate-core@0.5.1. Use `|| undefined` (not `??`) so the default empty
// string from ReviewConfig.dialect coerces to "no hint": the engine throws
// on an unknown dialect "", and "" must mean auto-detect, not a real dialect.
const raw = await core.checkEquivalence(params.sql1, params.sql2, schema, params.dialect || undefined)
// altimate-core@0.5.1. dialectHint coerces "" (the ReviewConfig default)
// to undefined: the engine throws on an unknown dialect "", and "" must
// mean auto-detect, not a real dialect.
const raw = await core.checkEquivalence(params.sql1, params.sql2, schema, dialectHint(params.dialect))
const data = toData(raw)
return ok(true, data)
} catch (e) {
Expand All @@ -337,8 +348,9 @@ export function registerAll(): void {
// 12. altimate_core.migration
register("altimate_core.migration", async (params) => {
try {
// Build schema from old_ddl, analyze new_ddl against it
const schema = core.Schema.fromDdl(params.old_ddl, params.dialect ?? undefined)
// Build schema from old_ddl, analyze new_ddl against it. dialectHint
// coerces "" to auto-detect (the engine throws on an unknown dialect "").
const schema = core.Schema.fromDdl(params.old_ddl, dialectHint(params.dialect))
const raw = core.analyzeMigration(params.new_ddl, schema)
const data = toData(raw)
return ok(true, data)
Expand Down Expand Up @@ -432,7 +444,7 @@ export function registerAll(): void {
register("altimate_core.column_lineage", async (params) => {
try {
const schema = resolveSchema(params.schema_path, params.schema_context)
const raw = core.columnLineage(params.sql, params.dialect ?? undefined, schema ?? undefined)
const raw = core.columnLineage(params.sql, dialectHint(params.dialect), schema ?? undefined)
return ok(true, toData(raw))
} catch (e) {
return fail(e)
Expand All @@ -453,7 +465,7 @@ export function registerAll(): void {
// 22. altimate_core.format
register("altimate_core.format", async (params) => {
try {
const raw = core.formatSql(params.sql, params.dialect ?? undefined)
const raw = core.formatSql(params.sql, dialectHint(params.dialect))
const data = toData(raw)
return ok(true, data)
} catch (e) {
Expand All @@ -464,7 +476,7 @@ export function registerAll(): void {
// 23. altimate_core.metadata
register("altimate_core.metadata", async (params) => {
try {
const raw = core.extractMetadata(params.sql, params.dialect ?? undefined)
const raw = core.extractMetadata(params.sql, dialectHint(params.dialect))
return ok(true, toData(raw))
} catch (e) {
return fail(e)
Expand All @@ -474,7 +486,7 @@ export function registerAll(): void {
// 24. altimate_core.compare
register("altimate_core.compare", async (params) => {
try {
const raw = core.compareQueries(params.left_sql, params.right_sql, params.dialect ?? undefined)
const raw = core.compareQueries(params.left_sql, params.right_sql, dialectHint(params.dialect))
return ok(true, toData(raw))
} catch (e) {
return fail(e)
Expand Down Expand Up @@ -528,7 +540,7 @@ export function registerAll(): void {
// 29. altimate_core.import_ddl — returns Schema, must serialize
register("altimate_core.import_ddl", async (params) => {
try {
const schema = core.importDdl(params.ddl, params.dialect ?? undefined)
const schema = core.importDdl(params.ddl, dialectHint(params.dialect))
const jsonObj = schema.toJson()
return ok(true, { success: true, schema: toData(jsonObj) })
} catch (e) {
Expand Down
35 changes: 35 additions & 0 deletions packages/opencode/src/altimate/native/engine-coerce.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/**
* Shared coercions for altimate-core engine values.
*
* The engine's napi surface has two recurring foot-guns for consumers:
* - `PiiClassification` is a string OR `{ Custom: string }` — naive string
* interpolation renders `[object Object]`.
* - dialect parameters throw on the empty string (`unknown dialect ''`);
* `""` must mean "auto-detect" and be passed as undefined.
*/

/** PiiClassification is 'Email' | … | { Custom: string } | 'None'. */
export function classificationToString(c: unknown, fallback = "PII"): string {
Comment thread
anandgupta42 marked this conversation as resolved.
Outdated
if (typeof c === "string") return c
if (c && typeof c === "object" && typeof (c as { Custom?: unknown }).Custom === "string") {
return (c as { Custom: string }).Custom
}
return fallback
}

/** Coerce an optional dialect to an engine-safe hint: "" and null mean auto-detect. */
export function dialectHint(dialect: string | undefined | null): string | undefined {
return dialect || undefined
}

/** Map the engine's numeric confidence (0..1) to a string band. */
export function bandConfidence(c: unknown): "high" | "medium" | "low" {
Comment thread
anandgupta42 marked this conversation as resolved.
Comment thread
anandgupta42 marked this conversation as resolved.
if (typeof c === "string") {
const s = c.toLowerCase()
if (s === "high" || s === "medium" || s === "low") return s
}
const n = typeof c === "number" ? c : NaN
if (n >= 0.8) return "high"
if (n >= 0.5) return "medium"
return "low"
}
67 changes: 41 additions & 26 deletions packages/opencode/src/altimate/native/schema/pii-detector.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
*/

import * as core from "@altimateai/altimate-core"
import { bandConfidence, classificationToString } from "../engine-coerce"
import { getCache } from "./cache"
import * as Registry from "../connections/registry"
import type {
Expand All @@ -12,6 +13,19 @@ import type {
PiiFinding,
} from "../types"

/**
* Extract the real PII rows from an engine PiiReport.
*
* The engine returns `{ columns, pii_count, risk_level, total_columns }` with
* a row for EVERY column — classification "None" means not PII. (The previous
* consumer read a nonexistent `findings` field, making detection a silent
* no-op.) Exported for tests.
*/
export function piiColumnsFromReport(piiData: unknown): Array<Record<string, any>> {
const columns = ((piiData as Record<string, any>)?.columns ?? []) as Array<Record<string, any>>
return columns.filter((c) => c.classification !== "None")
}

/**
* Detect PII in cached schema metadata by running altimate-core's
* classifyPii() on column names and types.
Expand Down Expand Up @@ -72,19 +86,20 @@ export async function detectPii(params: PiiDetectParams): Promise<PiiDetectResul
const result = core.classifyPii(schema)
const piiData = JSON.parse(JSON.stringify(result))

if (piiData && piiData.findings && piiData.findings.length > 0) {
for (const finding of piiData.findings) {
findings.push({
warehouse: col.warehouse,
schema: col.schema_name,
table: col.table,
column: col.name,
data_type: col.data_type,
pii_category: finding.category || finding.pii_type || "UNKNOWN",
confidence: finding.confidence || "medium",
})
tablesWithPii.add(`${col.warehouse}.${col.schema_name}.${col.table}`)
}
// Engine PiiReport: { columns, pii_count, risk_level, total_columns };
// every column gets a row — classification "None" means not PII.
const piiColumns = piiColumnsFromReport(piiData)
for (const piiCol of piiColumns) {
findings.push({
warehouse: col.warehouse,
schema: col.schema_name,
table: col.table,
column: col.name,
data_type: col.data_type,
pii_category: classificationToString(piiCol.classification, "UNKNOWN"),
confidence: bandConfidence(piiCol.confidence),
Comment thread
anandgupta42 marked this conversation as resolved.
Outdated
})
tablesWithPii.add(`${col.warehouse}.${col.schema_name}.${col.table}`)
}
} catch {
// classifyPii may not find PII — that is expected
Expand Down Expand Up @@ -165,19 +180,19 @@ async function detectPiiLive(params: PiiDetectParams): Promise<PiiDetectResult>
const result = core.classifyPii(schema)
const piiData = JSON.parse(JSON.stringify(result))

if (piiData?.findings) {
for (const finding of piiData.findings) {
findings.push({
warehouse: params.warehouse!,
schema: schemaName,
table: tableInfo.name,
column: finding.column || "",
data_type: finding.data_type,
pii_category: finding.category || finding.pii_type || "UNKNOWN",
confidence: finding.confidence || "medium",
})
tablesWithPii.add(`${params.warehouse}.${schemaName}.${tableInfo.name}`)
}
// Engine PiiReport: { columns, … } with a row per column; "None" = not PII.
const piiColumns = piiColumnsFromReport(piiData)
for (const piiCol of piiColumns) {
findings.push({
warehouse: params.warehouse!,
schema: schemaName,
table: tableInfo.name,
column: piiCol.column || "",
data_type: columns.find((c) => c.name === piiCol.column)?.data_type,
pii_category: classificationToString(piiCol.classification, "UNKNOWN"),
confidence: bandConfidence(piiCol.confidence),
})
tablesWithPii.add(`${params.warehouse}.${schemaName}.${tableInfo.name}`)
}
} catch {
// ignore
Expand Down
8 changes: 6 additions & 2 deletions packages/opencode/src/altimate/review/runner.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { Dispatcher } from "../native"
import { classificationToString } from "../native/engine-coerce"
import { parseManifest } from "../native/dbt/manifest"
import type { CheckResult, EquivalenceResult, GradeResult, ImpactResult, ReviewRunner } from "./orchestrate"
import { buildReviewSchemaContext, type SchemaContext } from "./schema-context"
Expand Down Expand Up @@ -227,7 +228,9 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun
// RAW issues (which still carry column/target/name) — the normalized
// `issues` drop those fields, so mapping over them would always miss.
const piiColumns = [
...asArray<any>(data.pii).map(piiColumnOf),
// data.pii is the engine PiiQueryResult ({ pii_columns, … }); the
// legacy array shape is kept as a fallback.
...asArray<any>((data.pii as any)?.pii_columns ?? data.pii).map(piiColumnOf),
Comment thread
anandgupta42 marked this conversation as resolved.
Outdated
...rawIssues
.filter((i: any) => /pii|sensitive/i.test(String(i.category ?? i.rule ?? i.code ?? i.kind ?? "")))
.map(piiColumnOf),
Expand Down Expand Up @@ -400,7 +403,8 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun
.filter((c) => c?.classification && c.classification !== "None")
.map((c) => ({
column: String(c.column ?? ""),
classification: String(c.classification ?? ""),
// classification can be { Custom: string } — String() would emit "[object Object]"
classification: classificationToString(c.classification, ""),
confidence: typeof c.confidence === "number" ? c.confidence : 0,
masking: c.suggested_masking ?? undefined,
}))
Expand Down
23 changes: 16 additions & 7 deletions packages/opencode/src/altimate/tools/altimate-core-check.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import z from "zod"
import { Tool } from "../../tool/tool"
import { Dispatcher } from "../native"
import { classificationToString } from "../native/engine-coerce"
import type { Telemetry } from "../telemetry"

export const AltimateCoreCheckTool = Tool.define("altimate_core_check", {
Expand Down Expand Up @@ -30,9 +31,9 @@ export const AltimateCoreCheckTool = Tool.define("altimate_core_check", {
findings.push({ category: f.rule ?? "lint" })
}
for (const t of data.safety?.threats ?? []) {
findings.push({ category: t.type ?? "safety_threat" })
findings.push({ category: t.rule ?? t.type ?? "safety_threat" })
}
for (const p of data.pii?.findings ?? []) {
for (const p of data.pii?.pii_columns ?? data.pii?.findings ?? []) {
findings.push({ category: "pii_detected" })
}
// altimate_change end
Expand Down Expand Up @@ -62,7 +63,7 @@ export function formatCheckTitle(data: Record<string, any>): string {
if (!data.validation?.valid) parts.push("validation errors")
if (!data.lint?.clean) parts.push(`${data.lint?.findings?.length ?? 0} lint findings`)
if (!data.safety?.safe) parts.push("safety threats")
if (data.pii?.findings?.length) parts.push("PII detected")
if (data.pii?.pii_columns?.length || data.pii?.findings?.length) parts.push("PII detected")
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Outdated
return parts.length ? parts.join(", ") : "PASS"
}

Expand Down Expand Up @@ -93,16 +94,24 @@ export function formatCheck(data: Record<string, any>): string {
lines.push("Safe — no threats.")
} else {
for (const t of data.safety?.threats ?? []) {
lines.push(` [${t.severity ?? "warning"}] ${t.type ?? "safety"}: ${t.description ?? ""}`)
lines.push(` [${t.severity ?? "warning"}] ${t.rule ?? t.type ?? "safety"}: ${t.message ?? t.description ?? ""}`)
}
}

lines.push("\n=== PII ===")
if (!data.pii?.findings?.length) {
// Engine PiiQueryResult: { accesses_pii, pii_columns, risk_level, parse_error? }
const piiCols = (data.pii?.pii_columns ?? data.pii?.findings ?? []) as any[]
if (data.pii?.parse_error) {
// Abstention, not a clean verdict — the engine could not parse the query.
lines.push(`PII check skipped: ${data.pii.parse_error}`)
} else if (!piiCols.length) {
lines.push("No PII detected.")
} else {
for (const p of data.pii?.findings ?? []) {
lines.push(` ${p.column ?? "unknown"}: ${p.category ?? "PII"} (${p.confidence ?? "unknown"} confidence)`)
for (const p of piiCols) {
const cls = classificationToString(p.classification ?? p.category)
const where = [p.table, p.column ?? "unknown"].filter(Boolean).join(".")
const via = Array.isArray(p.query_targets) && p.query_targets.length ? ` exposed via: ${p.query_targets.join(", ")}` : ""
lines.push(` ${where}: ${cls}${via}${p.suggested_masking ? ` (masking: ${p.suggested_masking})` : ""}`)
}
}

Expand Down
Loading
Loading