diff --git a/README.md b/README.md index eddf26f..5d3e09f 100644 --- a/README.md +++ b/README.md @@ -354,6 +354,26 @@ testfixtures.New( ### PostgreSQL / TimescaleDB / CockroachDB +#### Restricting table scanning by `search_path` + +By default, this package discovers tables, sequences, and constraints across all +non-system schemas. To limit fixture operations to the schemas in PostgreSQL's +effective `search_path`, enable `RestrictTableScanningBySearchPath` on the +loader: + +```go +testfixtures.New( + testfixtures.Database(db), + testfixtures.Dialect("postgres"), + testfixtures.RestrictTableScanningBySearchPath(), + testfixtures.Directory("testdata/fixtures"), +) +``` + +Configure `search_path` in the connection string so every connection in the +pool uses it. This allows parallel tests to share a database safely when each +test uses a separate schema. + This package has three approaches to disable foreign keys while importing fixtures for PostgreSQL databases: diff --git a/cmd/testfixtures/testfixtures.go b/cmd/testfixtures/testfixtures.go index fdcb4e9..7ed8db3 100644 --- a/cmd/testfixtures/testfixtures.go +++ b/cmd/testfixtures/testfixtures.go @@ -21,18 +21,19 @@ func main() { log.SetOutput(os.Stderr) var ( - versionFlag bool - dialect string - connString string - dir string - files []string - paths []string - useDropContraint bool - useAlterContraint bool - skipResetSequences bool - resetSequencesTo int64 - skipTestDatabaseCheck bool - dumpFlag bool + versionFlag bool + dialect string + connString string + dir string + files []string + paths []string + useDropContraint bool + useAlterContraint bool + restrictTableScanningBySearchPath bool + skipResetSequences bool + resetSequencesTo int64 + skipTestDatabaseCheck bool + dumpFlag bool ) pflag.BoolVar(&versionFlag, "version", false, "show testfixtures version") @@ -43,6 +44,7 @@ func main() { pflag.StringSliceVarP(&paths, "paths", "p", nil, "a list of fixture paths to load (directory or file)") pflag.BoolVar(&useDropContraint, "drop-constraint", false, "use ALTER CONSTRAINT to disable referential integrity (CockroachDB only)") pflag.BoolVar(&useAlterContraint, "alter-constraint", false, "use ALTER CONSTRAINT to disable referential integrity (PostgreSQL only)") + pflag.BoolVar(&restrictTableScanningBySearchPath, "restrict-table-scanning-by-search-path", false, "limit fixture operations to schemas in search_path (PostgreSQL only)") pflag.BoolVar(&skipResetSequences, "no-reset-sequences", false, "skip reset of sequences after loading (PostgreSQL and MySQL/MariaDB only)") pflag.Int64Var(&resetSequencesTo, "reset-sequences-to", 0, "sets the number sequences will be reset after loading fixtures (PostgreSQL and MySQL/MariaDB only, defaults to 10000)") pflag.BoolVar(&skipTestDatabaseCheck, "dangerous-no-test-database-check", false, `skips check for "test" in database name (use with caution)`) @@ -119,6 +121,9 @@ func main() { testfixtures.Database(db), testfixtures.Dialect(dialect), } + if restrictTableScanningBySearchPath { + options = append(options, testfixtures.RestrictTableScanningBySearchPath()) + } if dir != "" { options = append(options, testfixtures.Directory(dir)) } diff --git a/dbtests/postgresql_test.go b/dbtests/postgresql_test.go index ec2a16e..c59f938 100644 --- a/dbtests/postgresql_test.go +++ b/dbtests/postgresql_test.go @@ -1,9 +1,14 @@ package dbtests import ( + "database/sql" + "fmt" + "net/url" + "strings" "testing" "github.com/go-testfixtures/testfixtures/v3" + "github.com/google/uuid" _ "github.com/jackc/pgx/v4/stdlib" _ "github.com/lib/pq" ) @@ -23,6 +28,10 @@ func TestPostgreSQL(t *testing.T) { t.Run("WithDropConstraint", func(t *testing.T) { testPostgreSQL(t, connStr, testfixtures.UseDropConstraint()) }) + + t.Run("RestrictTableScanningBySearchPath", func(t *testing.T) { + testPostgreSQLRestrictTableScanningBySearchPath(t, connStr) + }) } func testPostgreSQL(t *testing.T, connStr string, additionalOptions ...func(*testfixtures.Loader) error) { @@ -40,3 +49,112 @@ func testPostgreSQL(t *testing.T, connStr string, additionalOptions ...func(*tes }) } } + +func testPostgreSQLRestrictTableScanningBySearchPath(t *testing.T, connStr string) { + t.Helper() + + constraintModes := []struct { + name string + option func(*testfixtures.Loader) error + }{ + {name: "disable_triggers"}, + {name: "alter_constraints", option: testfixtures.UseAlterConstraint()}, + {name: "drop_constraints", option: testfixtures.UseDropConstraint()}, + } + + for _, dialect := range []string{"postgres", "pgx"} { + t.Run(dialect, func(t *testing.T) { + for _, constraintMode := range constraintModes { + t.Run(constraintMode.name, func(t *testing.T) { + testPostgreSQLRestrictTableScanningBySearchPathConstraintMode(t, dialect, connStr, constraintMode.option) + }) + } + }) + } +} + +func testPostgreSQLRestrictTableScanningBySearchPathConstraintMode( + t *testing.T, + dialect string, + connStr string, + constraintOption func(*testfixtures.Loader) error, +) { + t.Helper() + + suffix := strings.ReplaceAll(uuid.NewString(), "-", "_") + schema := "testfixtures_search_path_" + suffix + otherSchema := schema + "_other" + + adminDB := openDB(t, dialect, connStr) + createSearchPathTestSchema(t, adminDB, schema) + createSearchPathTestSchema(t, adminDB, otherSchema) + t.Cleanup(func() { + _, _ = adminDB.Exec(fmt.Sprintf(`DROP SCHEMA IF EXISTS %q CASCADE`, schema)) + _, _ = adminDB.Exec(fmt.Sprintf(`DROP SCHEMA IF EXISTS %q CASCADE`, otherSchema)) + }) + + db := openDB(t, dialect, withSearchPath(t, connStr, schema)) + options := []func(*testfixtures.Loader) error{ + testfixtures.Database(db), + testfixtures.Dialect(dialect), + testfixtures.RestrictTableScanningBySearchPath(), + testfixtures.Directory("testdata/fixtures_search_path"), + } + if constraintOption != nil { + options = append(options, constraintOption) + } + + loader, err := testfixtures.New(options...) + if err != nil { + t.Fatalf("failed to create loader: %v", err) + } + + if _, err := adminDB.Exec(fmt.Sprintf(`DROP SCHEMA %q CASCADE`, otherSchema)); err != nil { + t.Fatalf("failed to drop unrelated schema: %v", err) + } + if err := loader.Load(); err != nil { + t.Fatalf("failed to load fixtures: %v", err) + } + + var name string + if err := db.QueryRow("SELECT name FROM widgets WHERE id = 1").Scan(&name); err != nil { + t.Fatalf("failed to query loaded fixture: %v", err) + } + if name != "scoped widget" { + t.Fatalf("unexpected fixture name: %q", name) + } +} + +func createSearchPathTestSchema(t *testing.T, db *sql.DB, schema string) { + t.Helper() + + query := fmt.Sprintf(` + CREATE SCHEMA %q; + CREATE TABLE %q.parents ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name TEXT NOT NULL + ); + CREATE TABLE %q.widgets ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + parent_id BIGINT NOT NULL REFERENCES %q.parents(id), + name TEXT NOT NULL + ); + `, schema, schema, schema, schema) + if _, err := db.Exec(query); err != nil { + t.Fatalf("failed to create search_path test schema: %v", err) + } +} + +func withSearchPath(t *testing.T, connStr, schema string) string { + t.Helper() + + parsed, err := url.Parse(connStr) + if err == nil && parsed.Scheme != "" { + query := parsed.Query() + query.Set("search_path", schema) + parsed.RawQuery = query.Encode() + return parsed.String() + } + + return connStr + " search_path=" + schema +} diff --git a/dbtests/testdata/fixtures_search_path/parents.yml b/dbtests/testdata/fixtures_search_path/parents.yml new file mode 100644 index 0000000..369c07d --- /dev/null +++ b/dbtests/testdata/fixtures_search_path/parents.yml @@ -0,0 +1,2 @@ +- id: 1 + name: scoped parent diff --git a/dbtests/testdata/fixtures_search_path/widgets.yml b/dbtests/testdata/fixtures_search_path/widgets.yml new file mode 100644 index 0000000..d9f3b78 --- /dev/null +++ b/dbtests/testdata/fixtures_search_path/widgets.yml @@ -0,0 +1,3 @@ +- id: 1 + parent_id: 1 + name: scoped widget diff --git a/postgresql.go b/postgresql.go index d7b45b0..b2d18da 100644 --- a/postgresql.go +++ b/postgresql.go @@ -15,10 +15,11 @@ import ( type postgreSQL struct { baseHelper - useAlterConstraint bool - useDropConstraint bool - skipResetSequences bool - resetSequencesTo int64 + useAlterConstraint bool + useDropConstraint bool + restrictTableScanningBySearchPathEnabled bool + skipResetSequences bool + resetSequencesTo int64 tables []string sequences []string @@ -30,6 +31,10 @@ type postgreSQL struct { tablesHasIdentityColumn map[string]bool } +func (h *postgreSQL) restrictTableScanningBySearchPath() { + h.restrictTableScanningBySearchPathEnabled = true +} + type pgConstraint struct { tableName string constraintName string @@ -95,9 +100,10 @@ func (h *postgreSQL) tableNames(q shared.Queryable) ([]string, error) { WHERE pg_class.relkind = 'r' AND pg_namespace.nspname NOT IN ('pg_catalog', 'information_schema', 'crdb_internal', 'pg_extension') AND pg_namespace.nspname NOT LIKE 'pg_toast%' - AND pg_namespace.nspname NOT LIKE '\_timescaledb%'; + AND pg_namespace.nspname NOT LIKE '\_timescaledb%' + AND (NOT $1 OR pg_namespace.nspname = ANY (current_schemas(false))); ` - rows, err := q.Query(sql) + rows, err := q.Query(sql, h.restrictTableScanningBySearchPathEnabled) if err != nil { return nil, err } @@ -125,9 +131,10 @@ func (h *postgreSQL) getSequences(q shared.Queryable) ([]string, error) { INNER JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace WHERE pg_class.relkind = 'S' AND pg_namespace.nspname NOT LIKE '\_timescaledb%' + AND (NOT $1 OR pg_namespace.nspname = ANY (current_schemas(false))); ` - rows, err := q.Query(sql) + rows, err := q.Query(sql, h.restrictTableScanningBySearchPathEnabled) if err != nil { return nil, err } @@ -149,7 +156,7 @@ func (h *postgreSQL) getSequences(q shared.Queryable) ([]string, error) { return sequences, nil } -func (*postgreSQL) getNonDeferrableConstraints(q shared.Queryable) ([]pgConstraint, error) { +func (h *postgreSQL) getNonDeferrableConstraints(q shared.Queryable) ([]pgConstraint, error) { var constraints []pgConstraint const sql = ` @@ -159,8 +166,9 @@ func (*postgreSQL) getNonDeferrableConstraints(q shared.Queryable) ([]pgConstrai AND is_deferrable = 'NO' AND table_schema <> 'crdb_internal' AND table_schema NOT LIKE '\_timescaledb%' - ` - rows, err := q.Query(sql) + AND (NOT $1 OR table_schema = ANY (current_schemas(false))); + ` + rows, err := q.Query(sql, h.restrictTableScanningBySearchPathEnabled) if err != nil { return nil, err } @@ -191,9 +199,10 @@ func (h *postgreSQL) getConstraints(q shared.Queryable) ([]pgConstraint, error) WHERE contype = 'f' AND pg_namespace.nspname NOT IN ('pg_catalog', 'information_schema', 'crdb_internal') AND pg_namespace.nspname NOT LIKE 'pg_toast%' - AND pg_namespace.nspname NOT LIKE '\_timescaledb%'; - ` - rows, err := q.Query(sql) + AND pg_namespace.nspname NOT LIKE '\_timescaledb%' + AND (NOT $1 OR pg_namespace.nspname = ANY (current_schemas(false))); + ` + rows, err := q.Query(sql, h.restrictTableScanningBySearchPathEnabled) if err != nil { return nil, err } @@ -464,10 +473,11 @@ func (h *postgreSQL) buildTableHasIdentityColumn(q shared.Queryable) (map[string table_schema NOT IN ('pg_catalog', 'information_schema', 'crdb_internal') AND table_schema NOT LIKE 'pg_toast%' AND table_schema NOT LIKE '\_timescaledb%' AND - is_identity = 'YES' + is_identity = 'YES' AND + (NOT $1 OR table_schema = ANY (current_schemas(false))) GROUP BY table_name;` - rows, err := q.Query(query) + rows, err := q.Query(query, h.restrictTableScanningBySearchPathEnabled) if err != nil { return nil, err } diff --git a/testfixtures.go b/testfixtures.go index e0d696a..ebff00d 100644 --- a/testfixtures.go +++ b/testfixtures.go @@ -171,6 +171,29 @@ func Dialect(dialect string, opts ...DialectOptions) func(*Loader) error { } } +type tableScanningBySearchPathRestricter interface { + restrictTableScanningBySearchPath() +} + +// RestrictTableScanningBySearchPath limits database metadata discovery and +// referential integrity operations to schemas in the connection's effective +// search_path. +// +// The search_path must be configured for every connection in the pool, +// typically through the connection string. PostgreSQL currently supports this +// option. +func RestrictTableScanningBySearchPath() func(*Loader) error { + return func(l *Loader) error { + h, ok := l.helper.(tableScanningBySearchPathRestricter) + if !ok { + return fmt.Errorf("testfixtures: RestrictTableScanningBySearchPath is not supported for this database") + } + + h.restrictTableScanningBySearchPath() + return nil + } +} + func helperForDialect(dialect string) (helper, error) { switch dialect { case "postgres", "postgresql", "timescaledb", "pgx": diff --git a/testfixtures_test.go b/testfixtures_test.go index 39b1f91..c5a501e 100644 --- a/testfixtures_test.go +++ b/testfixtures_test.go @@ -42,6 +42,37 @@ func TestRequiredOptions(t *testing.T) { }) } +func TestRestrictTableScanningBySearchPath(t *testing.T) { + t.Run("postgresql", func(t *testing.T) { + loader := &Loader{} + if err := Dialect("postgresql")(loader); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if err := RestrictTableScanningBySearchPath()(loader); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + pgHelper, ok := loader.helper.(*postgreSQL) + if !ok { + t.Fatal("expected PostgreSQL helper") + } + if !pgHelper.restrictTableScanningBySearchPathEnabled { + t.Error("expected table scanning to be restricted by search_path") + } + }) + + t.Run("non_postgresql", func(t *testing.T) { + loader := &Loader{} + if err := Dialect("sqlite")(loader); err != nil { + t.Fatalf("unexpected error: %v", err) + } + err := RestrictTableScanningBySearchPath()(loader) + if err == nil { + t.Fatal("expected an error") + } + }) +} + func TestQuoteKeyword(t *testing.T) { tests := []struct { helper helper