feat(catalog)!: rename required_field_filter_paths -> required_filters (CNF)

Port the breaking VGI wire-protocol change (vgi-python 0.15.0, vgi C++
extension) to the Go SDK. The TableInfo trailing field is renamed from
required_field_filter_paths to required_filters, and its type changes from a
flat AND-only []string to conjunctive normal form [][]string: an outer AND of
inner OR-groups of dotted-path column references. A group is satisfied when any
one of its member paths has a WHERE filter; every group must be satisfied. So
[["accession_number"], ["ticker","cik"]] means "accession_number AND one of
(ticker, cik)". No backward compatibility — the old name is removed entirely.

- Wire type: list<utf8> -> list<list<utf8>> (arrow.ListOf(arrow.ListOf(String))),
  serialized via the nested list-builder pattern used by unique_constraints /
  primary_key_constraints.
- TableInfo.RequiredFilters and CatalogTable.RequiredFilters are now [][]string.
- Add validateRequiredFilters (mirrors vgi-python's descriptor): reject empty
  groups and empty strings; the leading dotted segment of each path must name a
  real column.
- Convert example fixtures to singleton groups to preserve prior AND meaning and
  add rff_or, a genuine OR-group ([["a","b"]]) backing required_filters_or_group.
- Add unit tests for CNF serialization round-trip and validation.

Verified: go build ./..., go vet ./..., go test ./... (green), and the full
Query-farm/vgi sqllogictest integration suite (make test: 9909 assertions,
249 cases, 0 failures), including required_filters_or_group.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: rustyconover

cmd/vgi-example-worker/main.go

@@ -603,50 +603,60 @@ func main() {
 		},
 		StatisticsCacheMaxAgeSeconds: &statsTTL3600,
 	})
-	// required_field_filter_paths fixtures — back the
-	// required_field_filter_paths_*.test sqllogictest matrix. The C++ optimizer
-	// extension enforces the declared paths at bind time.
+	// required_filters fixtures — back the required_filters_*.test
+	// sqllogictest matrix. The C++ optimizer extension enforces the declared
+	// CNF groups at bind time. Each group is an OR of member paths; every group
+	// must be satisfied. Prior flat single-path requirements become singleton
+	// groups to preserve their AND meaning.
 	w.RegisterCatalogTable("data", vgi.CatalogTable{
-		Name:                     "rff_simple",
-		Comment:                  "rff_simple — requires a filter referencing column 'a'.",
-		Columns:                  table.RffSimpleSchema,
-		RequiredFieldFilterPaths: []string{"a"},
+		Name:            "rff_simple",
+		Comment:         "rff_simple — requires a filter referencing column 'a'.",
+		Columns:         table.RffSimpleSchema,
+		RequiredFilters: [][]string{{"a"}},
 	})
 	w.RegisterCatalogTable("data", vgi.CatalogTable{
-		Name:                     "rff_struct",
-		Comment:                  "rff_struct — requires filters on both struct subfields s.a and s.b.",
-		Columns:                  table.RffStructSchema,
-		RequiredFieldFilterPaths: []string{"s.a", "s.b"},
+		Name:            "rff_struct",
+		Comment:         "rff_struct — requires filters on both struct subfields s.a and s.b.",
+		Columns:         table.RffStructSchema,
+		RequiredFilters: [][]string{{"s.a"}, {"s.b"}},
 	})
 	w.RegisterCatalogTable("data", vgi.CatalogTable{
-		Name:                     "rff_nested",
-		Comment:                  "rff_nested — requires a filter on the 3-deep nested path wrapper.mid.leaf.",
-		Columns:                  table.RffNestedSchema,
-		RequiredFieldFilterPaths: []string{"wrapper.mid.leaf"},
+		Name:            "rff_nested",
+		Comment:         "rff_nested — requires a filter on the 3-deep nested path wrapper.mid.leaf.",
+		Columns:         table.RffNestedSchema,
+		RequiredFilters: [][]string{{"wrapper.mid.leaf"}},
 	})
 	w.RegisterCatalogTable("data", vgi.CatalogTable{
-		Name:                     "rff_multi",
-		Comment:                  "rff_multi — mixed top-level + struct subfield requirements.",
-		Columns:                  table.RffMultiSchema,
-		RequiredFieldFilterPaths: []string{"top", "s.a"},
+		Name:            "rff_multi",
+		Comment:         "rff_multi — mixed top-level + struct subfield requirements (top AND s.a).",
+		Columns:         table.RffMultiSchema,
+		RequiredFilters: [][]string{{"top"}, {"s.a"}},
+	})
+	// rff_or — genuine OR-group: a single CNF group satisfied by a filter on
+	// EITHER a or b (i.e. [["a","b"]] means "one of (a, b)").
+	w.RegisterCatalogTable("data", vgi.CatalogTable{
+		Name:            "rff_or",
+		Comment:         "rff_or — requires a filter on either column a OR column b (single OR-group).",
+		Columns:         table.RffNoneSchema,
+		RequiredFilters: [][]string{{"a", "b"}},
 	})
 	w.RegisterCatalogTable("data", vgi.CatalogTable{
 		Name:    "rff_none",
-		Comment: "rff_none — control table with no required_field_filter_paths (opt-out fast path).",
+		Comment: "rff_none — control table with no required_filters (opt-out fast path).",
 		Columns: table.RffNoneSchema,
 	})
 	// rff_rowid — row_id virtual column + required bbox.* filters. The rowid
 	// table_filter is keyed by a sentinel >> column count, which the optimizer
-	// must skip. See required_field_filter_paths_rowid.test.
+	// must skip. See required_filters_rowid.test.
 	w.RegisterCatalogTable("data", vgi.CatalogTable{
-		Name:                     "rff_rowid",
-		Comment:                  "rff_rowid — row_id virtual column + required bbox.* filters.",
-		Columns:                  table.RffRowidSchema,
-		RequiredFieldFilterPaths: []string{"bbox.xmin", "bbox.xmax", "bbox.ymin", "bbox.ymax"},
+		Name:            "rff_rowid",
+		Comment:         "rff_rowid — row_id virtual column + required bbox.* filters.",
+		Columns:         table.RffRowidSchema,
+		RequiredFilters: [][]string{{"bbox.xmin"}, {"bbox.xmax"}, {"bbox.ymin"}, {"bbox.ymax"}},
 	})
 	// rff_parquet / rff_hive / rff_hive_mixed — native read_parquet delegation
 	// with bbox.* required filters (mirrors Overture transportation.segment).
-	// See required_field_filter_paths_native.test.
+	// See required_filters_native.test.
 	rffBboxType := arrow.StructOf(
 		arrow.Field{Name: "xmin", Type: arrow.PrimitiveTypes.Float32},
 		arrow.Field{Name: "ymin", Type: arrow.PrimitiveTypes.Float32},
@@ -660,7 +670,7 @@ func main() {
 			{Name: "bbox", Type: rffBboxType},
 			{Name: "other", Type: arrow.PrimitiveTypes.Int64},
 		}, nil),
-		RequiredFieldFilterPaths: []string{"bbox.xmin", "bbox.xmax", "bbox.ymin", "bbox.ymax"},
+		RequiredFilters: [][]string{{"bbox.xmin"}, {"bbox.xmax"}, {"bbox.ymin"}, {"bbox.ymax"}},
 	})
 	rffHiveColumns := arrow.NewSchema([]arrow.Field{
 		{Name: "id", Type: arrow.BinaryTypes.String},
@@ -671,16 +681,16 @@ func main() {
 		{Name: "type", Type: arrow.BinaryTypes.String},
 	}, nil)
 	w.RegisterCatalogTable("data", vgi.CatalogTable{
-		Name:                     "rff_hive",
-		Comment:                  "rff_hive — native read_parquet over Hive glob with bbox.* required filters.",
-		Columns:                  rffHiveColumns,
-		RequiredFieldFilterPaths: []string{"bbox.xmin", "bbox.xmax", "bbox.ymin", "bbox.ymax"},
+		Name:            "rff_hive",
+		Comment:         "rff_hive — native read_parquet over Hive glob with bbox.* required filters.",
+		Columns:         rffHiveColumns,
+		RequiredFilters: [][]string{{"bbox.xmin"}, {"bbox.xmax"}, {"bbox.ymin"}, {"bbox.ymax"}},
 	})
 	w.RegisterCatalogTable("data", vgi.CatalogTable{
-		Name:                     "rff_hive_mixed",
-		Comment:                  "rff_hive_mixed — native read_parquet, top-level 'id' + bbox.* required filters.",
-		Columns:                  rffHiveColumns,
-		RequiredFieldFilterPaths: []string{"id", "bbox.xmin", "bbox.xmax", "bbox.ymin", "bbox.ymax"},
+		Name:            "rff_hive_mixed",
+		Comment:         "rff_hive_mixed — native read_parquet, top-level 'id' + bbox.* required filters.",
+		Columns:         rffHiveColumns,
+		RequiredFilters: [][]string{{"id"}, {"bbox.xmin"}, {"bbox.xmax"}, {"bbox.ymin"}, {"bbox.ymax"}},
 	})
 	// filter_echo_table — catalog table echoing pushed-down filters, backs
 	// filter_pushdown_through_view.test.
@@ -946,6 +956,10 @@ func main() {
 				return &vgi.ScanFunctionResult{FunctionName: "rff_multi_scan"}, nil
 			case "rff_none":
 				return &vgi.ScanFunctionResult{FunctionName: "rff_none_scan"}, nil
+			case "rff_or":
+				// rff_or reuses the (a, b) rff_none dataset; only its
+				// required_filters (a single OR-group) differ.
+				return &vgi.ScanFunctionResult{FunctionName: "rff_none_scan"}, nil
 			case "rff_rowid":
 				return &vgi.ScanFunctionResult{FunctionName: "rff_rowid_scan"}, nil
 			case "filter_echo_table":

examples/all/all.go

@@ -191,7 +191,7 @@ func registerTables(w *vgi.Worker) {
 	w.RegisterTable(table.NewProductsScanFunction())
 	w.RegisterTable(table.NewProjectsScanFunction())
 	// rff_* scan functions back the Tables exercised by the
-	// required_field_filter_paths_*.test sqllogictest matrix.
+	// required_filters_*.test sqllogictest matrix.
 	w.RegisterTable(table.NewRffSimpleScanFunction())
 	w.RegisterTable(table.NewRffStructScanFunction())
 	w.RegisterTable(table.NewRffNestedScanFunction())

examples/table/required_filters.go

@@ -14,10 +14,10 @@ import (
 )
 
 // Scan functions backing the rff_* Tables exercised by the
-// required_field_filter_paths_*.test sqllogictest matrix. They mirror
+// required_filters_*.test sqllogictest matrix. They mirror
 // vgi-python's vgi/_test_fixtures/table/required_filters.py.
 //
-// The C++ VGI optimizer extension enforces Table.required_field_filter_paths
+// The C++ VGI optimizer extension enforces Table.required_filters
 // at bind/optimize time — rejecting any scan that lacks the declared WHERE
 // filters before a single worker byte is read. Once a scan passes that check,
 // DuckDB applies the actual filter itself, so these functions simply emit the
@@ -40,7 +40,7 @@ func (f *RffSimpleScanFunction) Name() string { return "rff_simple_scan" }
 
 func (f *RffSimpleScanFunction) Metadata() vgi.FunctionMetadata {
 	return vgi.FunctionMetadata{
-		Description: "rff_simple — flat columns (a, b) for required_field_filter_paths tests",
+		Description: "rff_simple — flat columns (a, b) for required_filters tests",
 		Stability:   vgi.StabilityConsistent,
 		Categories:  []string{"generator", "testing"},
 	}
@@ -90,7 +90,7 @@ func (f *RffNoneScanFunction) Name() string { return "rff_none_scan" }
 
 func (f *RffNoneScanFunction) Metadata() vgi.FunctionMetadata {
 	return vgi.FunctionMetadata{
-		Description: "rff_none — control table with no required_field_filter_paths",
+		Description: "rff_none — control table with no required_filters",
 		Stability:   vgi.StabilityConsistent,
 		Categories:  []string{"generator", "testing"},
 	}
@@ -145,7 +145,7 @@ func (f *RffStructScanFunction) Name() string { return "rff_struct_scan" }
 
 func (f *RffStructScanFunction) Metadata() vgi.FunctionMetadata {
 	return vgi.FunctionMetadata{
-		Description: "rff_struct — STRUCT(s.a, s.b) + other for required_field_filter_paths tests",
+		Description: "rff_struct — STRUCT(s.a, s.b) + other for required_filters tests",
 		Stability:   vgi.StabilityConsistent,
 		Categories:  []string{"generator", "testing"},
 	}
@@ -216,7 +216,7 @@ func (f *RffNestedScanFunction) Name() string { return "rff_nested_scan" }
 
 func (f *RffNestedScanFunction) Metadata() vgi.FunctionMetadata {
 	return vgi.FunctionMetadata{
-		Description: "rff_nested — nested STRUCT(wrapper.mid.leaf) for required_field_filter_paths tests",
+		Description: "rff_nested — nested STRUCT(wrapper.mid.leaf) for required_filters tests",
 		Stability:   vgi.StabilityConsistent,
 		Categories:  []string{"generator", "testing"},
 	}
@@ -339,7 +339,7 @@ func NewRffMultiScanFunction() vgi.TableFunction {
 // Because of the virtual rowid column this fixture needs projection_pushdown:
 // under projection the emitted batch must match the *projected* output schema,
 // so it builds only the requested columns. See
-// required_field_filter_paths_rowid.test.
+// required_filters_rowid.test.
 // ---------------------------------------------------------------------------
 
 // rffRowidMetadata marks the row_id virtual column (hidden from SELECT *).

vgi/catalog.go

@@ -6,6 +6,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"strings"
 
 	"github.com/Query-farm/vgi-rpc-go/vgirpc"
 	"github.com/apache/arrow-go/v18/arrow"
@@ -1775,6 +1776,43 @@ func resolveColumnGroupIndices(columns *arrow.Schema, groups [][]string) [][]int
 	return result
 }
 
+// validateRequiredFilters checks a table's CNF required-filter groups. Each
+// OR-group must be non-empty and contain no empty strings, and the leading
+// dotted segment of every path must name a real column on the table. Returns
+// nil when groups is empty (the no-enforcement fast path) or columns is nil.
+func validateRequiredFilters(tableName string, columns *arrow.Schema, groups [][]string) error {
+	if len(groups) == 0 {
+		return nil
+	}
+	columnNames := map[string]struct{}{}
+	if columns != nil {
+		for i := 0; i < columns.NumFields(); i++ {
+			columnNames[columns.Field(i).Name] = struct{}{}
+		}
+	}
+	for _, group := range groups {
+		if len(group) == 0 {
+			return fmt.Errorf("table %q: required_filters must not contain empty groups", tableName)
+		}
+		for _, path := range group {
+			if path == "" {
+				return fmt.Errorf("table %q: required_filters must not contain empty strings", tableName)
+			}
+			if columns == nil {
+				continue
+			}
+			head := path
+			if idx := strings.IndexByte(path, '.'); idx >= 0 {
+				head = path[:idx]
+			}
+			if _, ok := columnNames[head]; !ok {
+				return fmt.Errorf("table %q: required_filters path %q references unknown column %q", tableName, path, head)
+			}
+		}
+	}
+	return nil
+}
+
 // serializeCatalogTable converts a CatalogTable into serialized TableInfo bytes.
 func (w *Worker) serializeCatalogTable(schemaName string, ct *CatalogTable) ([]byte, error) {
 	// Resolve columns: if Function is set but Columns is nil, derive via OnBind
@@ -1824,6 +1862,14 @@ func (w *Worker) serializeCatalogTable(schemaName string, ct *CatalogTable) ([]b
 	unique := resolveColumnGroupIndices(columns, ct.Unique)
 	primaryKey := resolveColumnGroupIndices(columns, ct.PrimaryKey)
 
+	// Validate required_filters (CNF): each OR-group must be non-empty and
+	// contain no empty strings, and the leading dotted segment of each path
+	// must be a real column on this table. Mirrors vgi-python's descriptor
+	// validation; struct-subfield validity is left to DuckDB's binder.
+	if err := validateRequiredFilters(ct.Name, columns, ct.RequiredFilters); err != nil {
+		return nil, err
+	}
+
 	// Serialize FOREIGN KEY constraints
 	var foreignKeys [][]byte
 	for _, fk := range ct.ForeignKey {
@@ -1848,7 +1894,7 @@ func (w *Worker) serializeCatalogTable(schemaName string, ct *CatalogTable) ([]b
 		SupportsColumnStatistics: len(ct.Statistics) > 0,
 		CardinalityEstimate:      ct.CardinalityEstimate,
 		CardinalityMax:           ct.CardinalityMax,
-		RequiredFieldFilterPaths: ct.RequiredFieldFilterPaths,
+		RequiredFilters:          ct.RequiredFilters,
 	}
 
 	// Inline the scan function for function-backed tables so the C++ extension