Merge pull request #912 from phdoerfler/feat/generated-test-data

Share the test data as CSV, one directory per dataset

Author: milessabin

.github/workflows/ci.yml

@@ -75,6 +75,9 @@ jobs:
       - name: Check Headers
         run: sbt 'project ${{ matrix.project }}' '++ ${{ matrix.scala }}' headerCheckAll
 
+      - name: Check and build the test data
+        run: sbt 'project ${{ matrix.project }}' '++ ${{ matrix.scala }}' checkTestData genTestData
+
       - name: Start up test databases
         run: docker compose up --force-recreate -d --wait --quiet-pull
 
@@ -310,6 +313,9 @@ jobs:
         if: matrix.java == 'temurin@11' && steps.setup-java-temurin-11.outputs.cache-hit == 'false'
         run: sbt +update
 
+      - name: Build the test data
+        run: sbt genTestData
+
       - name: Start up test databases
         run: docker compose up --force-recreate -d --wait --quiet-pull
 

build.sbt

@@ -56,7 +56,13 @@ ThisBuild / githubWorkflowBuild ~= { steps =>
       commands = List("headerCheckAll"),
       name = Some("Check Headers")
     ),
+    WorkflowStep.Sbt(
+      commands = List("checkTestData", "genTestData"),
+      name = Some("Check and build the test data")
+    ),
     WorkflowStep.Run(
+      // The scripts have to exist before this: compose mounts target/testdata into the
+      // containers, and a database only reads its init directory on a first start.
       commands = List("docker compose up --force-recreate -d --wait --quiet-pull"),
       name = Some("Start up test databases")
     )
@@ -91,6 +97,10 @@ ThisBuild / githubWorkflowAddedJobs +=
     sbtStepPreamble = Nil,
     steps = githubWorkflowJobSetup.value.toList ++
       List(
+        WorkflowStep.Sbt(
+          commands = List("genTestData"),
+          name = Some("Build the test data")
+        ),
         WorkflowStep.Run(
           commands = List("docker compose up --force-recreate -d --wait --quiet-pull"),
           name = Some("Start up test databases")
@@ -102,6 +112,11 @@ ThisBuild / githubWorkflowAddedJobs +=
 
 ThisBuild / tlSitePublishBranch := Some("main")
 
+lazy val genTestData =
+  taskKey[Unit]("Build the container init scripts from the shared test data")
+lazy val checkTestData =
+  taskKey[Unit]("Check every dataset's scripts and CSVs against each other")
+lazy val newDataset = inputKey[Unit]("Create an empty dataset directory: newDataset <name>")
 lazy val allUp = taskKey[Unit]("Start all docker compose services")
 lazy val allStop = taskKey[Unit]("Stop all docker compose services")
 lazy val pgUp = taskKey[Unit]("Start Postgres")
@@ -111,18 +126,50 @@ lazy val oracleStop = taskKey[Unit]("Stop Oracle")
 lazy val mssqlUp = taskKey[Unit]("Start SQL Server")
 lazy val mssqlStop = taskKey[Unit]("Stop SQL Server")
 
-ThisBuild / allUp := runDocker("docker compose up -d --wait --quiet-pull")
-ThisBuild / allStop := runDocker("docker compose stop")
-ThisBuild / pgUp := runDocker("docker compose up -d --wait --quiet-pull postgres")
-ThisBuild / pgStop := runDocker("docker compose stop postgres")
-ThisBuild / oracleUp := runDocker("docker compose up -d --wait --quiet-pull oracle")
-ThisBuild / oracleStop := runDocker("docker compose stop oracle")
-ThisBuild / mssqlUp := runDocker("docker compose up -d --wait --quiet-pull mssql")
-ThisBuild / mssqlStop := runDocker("docker compose stop mssql")
-
-def runDocker(cmd: String): Unit = {
-  require(cmd.! == 0, s"docker indicated an error")
+ThisBuild / genTestData := GenTestData(buildRoot)
+ThisBuild / checkTestData := {
+  val problems = GenTestData.check(buildRoot)
+  val log = streams.value.log
+  problems.foreach(log.error(_))
+  if (problems.nonEmpty) sys.error(s"${problems.size} problems in testdata")
 }
+ThisBuild / newDataset := NewDataset(buildRoot, Def.spaceDelimited("<name>").parsed)
+
+// An input task is evaluated once per aggregated project, so without this newDataset creates the
+// directory and then fails three times saying it exists. Plain tasks resolve to one scoped key
+// and run once already.
+ThisBuild / newDataset / aggregate := false
+ThisBuild / allUp := dockerUp()
+ThisBuild / allStop := dockerStop()
+ThisBuild / pgUp := dockerUp("postgres")
+ThisBuild / pgStop := dockerStop("postgres")
+ThisBuild / oracleUp := dockerUp("oracle")
+ThisBuild / oracleStop := dockerStop("oracle")
+ThisBuild / mssqlUp := dockerUp("mssql")
+ThisBuild / mssqlStop := dockerStop("mssql")
+
+// The compose file is named relatively, so docker is already run from the build root; the
+// generated init scripts are written relative to the same place.
+def buildRoot: File = file(".").getAbsoluteFile
+
+/**
+ * Starts the named services, or every service when named none.
+ */
+def dockerUp(services: String*): Unit = {
+  // A container only reads its init scripts the first time it starts, so they have to be
+  // current before anything brings one up.
+  GenTestData(buildRoot)
+  runDocker("docker compose up -d --wait --quiet-pull", services)
+}
+
+/**
+ * Stops the named services, or every service when named none.
+ */
+def dockerStop(services: String*): Unit =
+  runDocker("docker compose stop", services)
+
+def runDocker(cmd: String, services: Seq[String]): Unit =
+  require((cmd +: services).mkString(" ").! == 0, s"docker indicated an error")
 
 lazy val commonSettings = Seq(
   // scalacOptions --= Seq("-Wunused:params", "-Wunused:imports", "-Wunused:patvars", "-Wdead-code", "-Wunused:locals", "-Wunused:privates", "-Wunused:implicits"),
@@ -295,8 +342,7 @@ lazy val doobiepg = project
     name := "grackle-doobie-pg",
     Test / fork := true,
     Test / parallelExecution := false,
-    Test / testOptions += Tests
-      .Setup(_ => runDocker("docker compose up -d --wait --quiet-pull postgres")),
+    Test / testOptions += Tests.Setup(_ => dockerUp("postgres")),
     libraryDependencies ++= Seq(
       "org.typelevel" %% "doobie-postgres-circe" % doobieVersion,
       // Pin transitive Postgres JDBC driver to >= 42.7.11 (CVE-2026-42198 / GHSA-98qh-xjc8-98pq)
@@ -314,8 +360,7 @@ lazy val doobieoracle = project
     name := "grackle-doobie-oracle",
     Test / fork := true,
     Test / parallelExecution := false,
-    Test / testOptions += Tests
-      .Setup(_ => runDocker("docker compose up -d --wait --quiet-pull oracle")),
+    Test / testOptions += Tests.Setup(_ => dockerUp("oracle")),
     libraryDependencies ++= Seq(
       "com.oracle.database.jdbc" % "ojdbc8" % oracleDriverVersion
     )
@@ -334,8 +379,7 @@ lazy val doobiemssql = project
     // mssql-jdbc binds a zone-naive java.sql.Timestamp using the ambient JVM zone, so MSSQL
     // datetime tests fail off-UTC unless pinned.
     Test / javaOptions += "-Duser.timezone=UTC",
-    Test / testOptions += Tests
-      .Setup(_ => runDocker("docker compose up -d --wait --quiet-pull mssql")),
+    Test / testOptions += Tests.Setup(_ => dockerUp("mssql")),
     libraryDependencies ++= Seq(
       "com.microsoft.sqlserver" % "mssql-jdbc" % mssqlDriverVersion
     )
@@ -359,8 +403,7 @@ lazy val skunk = crossProject(JVMPlatform, JSPlatform, NativePlatform)
   )
   .jvmSettings(
     Test / fork := true,
-    Test / testOptions += Tests.Setup(_ =>
-      runDocker("docker compose up -d --wait --quiet-pull postgres")),
+    Test / testOptions += Tests.Setup(_ => dockerUp("postgres")),
     libraryDependencies ++= Seq(
       "ch.qos.logback" % "logback-classic" % logbackVersion % "test"
     )

docker-compose.yml

@@ -8,7 +8,7 @@ services:
       - POSTGRES_USER=test
       - POSTGRES_PASSWORD=test
     volumes:
-      - ./testdata/pg/:/docker-entrypoint-initdb.d/
+      - ./target/testdata/pg/:/docker-entrypoint-initdb.d/
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 5s
@@ -22,7 +22,7 @@ services:
     environment:
       ORACLE_PASSWORD: test
     volumes:
-      - ./testdata/oracle/:/grackle-initdb.d/
+      - ./target/testdata/oracle/:/grackle-initdb.d/
       - ./modules/doobie-oracle/src/test/resources/scripts/:/container-entrypoint-initdb.d/
     healthcheck:
       test: bash -c "[ -f /tmp/healthy ]"
@@ -42,7 +42,7 @@ services:
       ACCEPT_EULA: Y
       MSSQL_TCP_PORT: 1433
     volumes:
-      - ./testdata/mssql/:/grackle-initdb.d/
+      - ./target/testdata/mssql/:/grackle-initdb.d/
       - ./modules/doobie-mssql/src/test/resources/scripts/:/container-entrypoint-initdb.d/
     entrypoint: ["/bin/bash", "/container-entrypoint-initdb.d/entrypoint.sh"]
     healthcheck:

project/Column.scala

@@ -0,0 +1,35 @@
+// Copyright (c) 2016-2025 Association of Universities for Research in Astronomy, Inc. (AURA)
+// Copyright (c) 2016-2025 Grackle Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * A column, and how the dialects disagree about writing its values, if they do.
+ */
+case class Column(name: String, kind: Kind, sqlType: String)
+
+object Column {
+
+  /**
+   * `nextshowing:timestamptz` is a timestamp column; a bare `title` is a plain one. The type
+   * comes from the dialect's own schema, which is where Oracle's array constructor lives.
+   */
+  def parse(header: String, sqlTypeOf: String => String): Column = {
+    val (name, kind) = header.split(":", -1) match {
+      case Array(name) => (name, Kind.Plain: Kind)
+      case Array(name, kind) => (name, Kind.named(kind))
+      case _ => sys.error(s"malformed column header '$header'")
+    }
+    Column(name, kind, sqlTypeOf(name))
+  }
+}

project/Dialect.scala

@@ -0,0 +1,106 @@
+// Copyright (c) 2016-2025 Association of Universities for Research in Astronomy, Inc. (AURA)
+// Copyright (c) 2016-2025 Grackle Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import java.time.OffsetDateTime
+import java.time.format.DateTimeFormatter
+
+import Dialect._
+import fs2.{Fallible, Stream}
+import fs2.data.csv.lowlevel
+
+/**
+ * How one database spells the values the dialects disagree about.
+ */
+sealed abstract class Dialect(val name: String) {
+  def terminator: String = ";"
+  def date(value: String): String = literal(value)
+  def time(value: String): String = literal(value)
+  def timestamp(value: String): String = literal(value)
+  def boolean(value: String): String = literal(value.toUpperCase)
+  def array(elements: List[String], sqlType: String): String
+
+  /**
+   * `\N` is the CSV's null; a plain value is a string literal for the database to coerce.
+   */
+  final def value(column: Column, cell: String): String =
+    if (cell == "\\N") "NULL"
+    else
+      column.kind match {
+        case Kind.Plain => literal(cell)
+        case Kind.Array => array(elements(cell), column.sqlType)
+        case Kind.Date => date(cell)
+        case Kind.Time => time(cell)
+        case Kind.Timestamp => timestamp(cell)
+        case Kind.Boolean => boolean(cell)
+      }
+
+  final def literal(value: String): String = s"'${value.replace("'", "''")}'"
+}
+
+object Postgres extends Dialect("pg") {
+  def array(elements: List[String], sqlType: String): String =
+    literal(elements.map(quoted).mkString("{", ",", "}"))
+}
+
+object Oracle extends Dialect("oracle") {
+  override def date(value: String): String = s"DATE ${literal(value)}"
+  override def time(value: String): String = s"INTERVAL '0 $value' DAY TO SECOND (0)"
+  override def timestamp(value: String): String = s"TIMESTAMP ${literal(sqlTimestamp(value))}"
+
+  /**
+   * A VARRAY value is built by calling the type, so the column's type is the constructor.
+   */
+  def array(elements: List[String], sqlType: String): String = {
+    require(sqlType.nonEmpty, "an array column needs a collection type in Oracle's schema")
+    elements.map(literal).mkString(s"$sqlType(", ", ", ")")
+  }
+}
+
+object SqlServer extends Dialect("mssql") {
+  override def terminator: String = ";\nGO"
+  override def timestamp(value: String): String = literal(sqlTimestamp(value))
+  override def boolean(value: String): String = if (value.toBoolean) "1" else "0"
+
+  /**
+   * SQL Server has no array type; the mappings read a JSON array out of a string column.
+   */
+  def array(elements: List[String], sqlType: String): String =
+    literal(elements.map(quoted).mkString("[", ", ", "]"))
+}
+
+object Dialect {
+
+  /**
+   * An array's elements are comma separated, quoted the way any other CSV field would be.
+   */
+  def elements(cell: String): List[String] =
+    if (cell.isEmpty) Nil
+    else
+      Stream
+        .emit(cell)
+        .through(lowlevel.rows[Fallible, String](','))
+        .compile
+        .toList
+        .fold(throw _, _.head.values.toList)
+
+  def quoted(element: String): String =
+    "\"" + element.replace("\\", "\\\\").replace("\"", "\\\"") + "\""
+
+  /**
+   * ISO-8601 in the CSV; `2020-05-22 19:35:00 +00:00` is what Oracle and SQL Server read.
+   */
+  def sqlTimestamp(value: String): String =
+    OffsetDateTime.parse(value).format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss xxx"))
+}