feat: multiple versions for the timescaledb-apache extension

.gitignore

@@ -25,3 +25,5 @@ result*
 
 db/schema.sql
 common-nix.vars.pkr.hcl
+
+.envrc

flake.nix

@@ -112,7 +112,7 @@
           /*"postgis"*/
         ];
 
-        #FIXME for now, timescaledb is not included in the orioledb version of supabase extensions, as there is an issue
+        # FIXME for now, timescaledb is not included in the orioledb version of supabase extensions, as there is an issue
         # with building timescaledb with the orioledb patched version of postgresql
         orioledbPsqlExtensions = [
           /* pljava */
@@ -131,7 +131,6 @@
         ourExtensions = [
           ./nix/ext/rum.nix
           ./nix/ext/timescaledb.nix
-          ./nix/ext/timescaledb-2.9.1.nix
           ./nix/ext/pgroonga.nix
           ./nix/ext/index_advisor.nix
           ./nix/ext/wal2json.nix
@@ -162,16 +161,13 @@
           ./nix/ext/plv8.nix
         ];
 
-        #Where we import and build the orioledb extension, we add on our custom extensions
-        # plus the orioledb option
-        #we're not using timescaledb or plv8 in the orioledb-17 version or pg 17 of supabase extensions
+        # Where we import and build the orioledb extension, we add on our
+        # custom extensions plus the orioledb option. We're not using
+        # timescaledb or plv8 in the orioledb-17 version or pg 17 of supabase
+        # extensions
         orioleFilteredExtensions = builtins.filter
-          (
-            x:
-            x != ./nix/ext/timescaledb.nix &&
-            x != ./nix/ext/timescaledb-2.9.1.nix &&
-            x != ./nix/ext/plv8.nix
-        ) ourExtensions;
+          (x: x != ./nix/ext/timescaledb.nix && x != ./nix/ext/plv8.nix)
+          ourExtensions;
 
         orioledbExtensions = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ];
         dbExtensions17 = orioleFilteredExtensions;
@@ -1377,6 +1373,7 @@
         # The list of exported 'checks' that are run with every run of 'nix
         # flake check'. This is run in the CI system, as well.
         checks = {
+          timescaledb = import ./nix/tests/timescaledb.nix { inherit self; inherit pkgs; };
           psql_15 = makeCheckHarness basePackages.psql_15.bin;
           psql_17 = makeCheckHarness basePackages.psql_17.bin;
           psql_orioledb-17 = makeCheckHarness basePackages.psql_orioledb-17.bin;

nix/ext/timescaledb.nix

@@ -1,42 +1,97 @@
-{ lib, stdenv, fetchFromGitHub, cmake, postgresql, openssl, libkrb5 }:
+{ pkgs, lib, stdenv, fetchFromGitHub, cmake, postgresql, openssl, libkrb5 }:
 
-stdenv.mkDerivation rec {
-  pname = "timescaledb-apache";
-  version = "2.16.1";
+let
+  pname = "timescaledb";
+  build = version: hash: revision:
+    stdenv.mkDerivation rec {
+      inherit pname version;
 
-  nativeBuildInputs = [ cmake ];
-  buildInputs = [ postgresql openssl libkrb5 ];
+      nativeBuildInputs = [ cmake ];
+      buildInputs = [ postgresql openssl libkrb5 ];
 
-  src = fetchFromGitHub {
-    owner = "timescale";
-    repo = "timescaledb";
-    rev = version;
-    hash = "sha256-sLxWdBmih9mgiO51zLLxn9uwJVYc5JVHJjSWoADoJ+w=";
-  };
+      src = fetchFromGitHub {
+        owner = "timescale";
+        repo = "timescaledb";
+        rev = version;
+        inherit hash;
+      };
 
-  cmakeFlags = [ "-DSEND_TELEMETRY_DEFAULT=OFF" "-DREGRESS_CHECKS=OFF" "-DTAP_CHECKS=OFF" "-DAPACHE_ONLY=1" ]
-    ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ];
-
-  # Fix the install phase which tries to install into the pgsql extension dir,
-  # and cannot be manually overridden. This is rather fragile but works OK.
-  postPatch = ''
-    for x in CMakeLists.txt sql/CMakeLists.txt; do
-      substituteInPlace "$x" \
-        --replace 'DESTINATION "''${PG_SHAREDIR}/extension"' "DESTINATION \"$out/share/postgresql/extension\""
-    done
-
-    for x in src/CMakeLists.txt src/loader/CMakeLists.txt tsl/src/CMakeLists.txt; do
-      substituteInPlace "$x" \
-        --replace 'DESTINATION ''${PG_PKGLIBDIR}' "DESTINATION \"$out/lib\""
-    done
-  '';
+      cmakeFlags = [
+        "-DSEND_TELEMETRY_DEFAULT=OFF"
+        "-DREGRESS_CHECKS=OFF"
+        "-DTAP_CHECKS=OFF"
+        "-DAPACHE_ONLY=1"
+      ] ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ];
+
+      postPatch = ''
+        for x in CMakeLists.txt sql/CMakeLists.txt; do
+          if [ -f "$x" ]; then
+            substituteInPlace "$x" \
+              --replace 'DESTINATION "''${PG_SHAREDIR}/extension"' "DESTINATION \"$out/share/postgresql/extension\""
+          fi
+        done
+
+        for x in src/CMakeLists.txt src/loader/CMakeLists.txt tsl/src/CMakeLists.txt; do
+          if [ -f "$x" ]; then
+            substituteInPlace "$x" \
+              --replace 'DESTINATION ''${PG_PKGLIBDIR}' "DESTINATION \"$out/lib\""
+          fi
+        done
+      '';
 
-  meta = with lib; {
-    description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space";
-    homepage = "https://www.timescale.com/";
-    changelog = "https://github.com/timescale/timescaledb/blob/${version}/CHANGELOG.md";
-    platforms = postgresql.meta.platforms;
-    license = licenses.asl20;
-    broken = versionOlder postgresql.version "13";
+      postInstall = ''
+        if [ -f $out/lib/timescaledb.so ]; then
+          mv $out/lib/timescaledb.so $out/lib/timescaledb-${version}.so
+        fi
+        if [ -f $out/share/postgresql/extension/timescaledb.control ]; then
+          mv $out/share/postgresql/extension/timescaledb.control $out/share/postgresql/extension/timescaledb--${version}.control
+        fi
+      '';
+
+      meta = with lib; {
+        description =
+          "Scales PostgreSQL for time-series data via automatic partitioning across time and space";
+        homepage = "https://www.timescale.com/";
+        changelog =
+          "https://github.com/timescale/timescaledb/blob/${version}/CHANGELOG.md";
+        license = licenses.postgresql;
+        inherit (postgresql.meta) platforms;
+      };
+    };
+
+  allVersions =
+    (builtins.fromJSON (builtins.readFile ./versions.json)).timescaledb;
+  supportedVersions = lib.filterAttrs (_: value:
+    builtins.elem (lib.versions.major postgresql.version) value.postgresql)
+    allVersions;
+  versions = lib.naturalSort (lib.attrNames supportedVersions);
+  latestVersion = lib.last versions;
+  numberOfVersions = builtins.length versions;
+  packages = builtins.attrValues
+    (lib.mapAttrs (name: value: build name value.hash (value.revision or name))
+      supportedVersions);
+in pkgs.buildEnv {
+  name = pname;
+  paths = packages;
+  postBuild = ''
+    {
+      echo "default_version = '${latestVersion}'"
+      cat $out/share/postgresql/extension/${pname}--${latestVersion}.control
+    } > $out/share/postgresql/extension/${pname}.control
+    ln -sfn ${pname}-${latestVersion}${postgresql.dlSuffix} $out/lib/${pname}${postgresql.dlSuffix}
+
+    # checks
+    (set -x
+       test "$(ls -A $out/lib/${pname}*${postgresql.dlSuffix} | wc -l)" = "${
+         toString (numberOfVersions + 1)
+       }"
+    )
+  '';
+  pathsToLink = [ "/lib" "/share/postgresql/extension" ];
+  passthru = {
+    inherit versions numberOfVersions;
+    pname = "${pname}-all";
+    version = "multi-" + lib.concatStringsSep "-"
+      (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
   };
 }

nix/ext/versions.json

@@ -0,0 +1,16 @@
+{
+  "timescaledb": {
+    "2.9.1": {
+      "postgresql": [
+        "15"
+      ],
+      "hash": "sha256-fvVSxDiGZAewyuQ2vZDb0I6tmlDXl6trjZp8+qDBtb8="
+    },
+    "2.16.1": {
+      "postgresql": [
+        "15"
+      ],
+      "hash": "sha256-sLxWdBmih9mgiO51zLLxn9uwJVYc5JVHJjSWoADoJ+w="
+    }
+  }
+}

nix/tests/timescaledb.nix

@@ -0,0 +1,83 @@
+{ self, pkgs }:
+let
+  inherit (pkgs) lib;
+  installedExtension = postgresMajorVersion:
+    self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/timescaledb-all";
+  versions = (installedExtension "15").versions;
+  firstVersion = lib.head versions;
+  latestVersion = lib.last versions;
+  postgresqlWithExtension = postgresql:
+    let
+      majorVersion = lib.versions.major postgresql.version;
+      pkg = pkgs.buildEnv {
+        name = "postgresql-${majorVersion}-timescaledb";
+        paths = [ postgresql postgresql.lib (installedExtension majorVersion) ];
+        passthru = {
+          inherit (postgresql) version psqlSchema;
+          lib = pkg;
+          withPackages = _: pkg;
+        };
+        nativeBuildInputs = [ pkgs.makeWrapper ];
+        pathsToLink = [ "/" "/bin" "/lib" ];
+        postBuild = ''
+          wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib
+          wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib
+          wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib
+        '';
+      };
+    in pkg;
+in self.inputs.nixpkgs.lib.nixos.runTest {
+  name = "timescaledb";
+  hostPkgs = pkgs;
+  nodes.server = { config, ... }: {
+    virtualisation = {
+      forwardPorts = [{
+        from = "host";
+        host.port = 13022;
+        guest.port = 22;
+      }];
+    };
+    services.openssh = { enable = true; };
+    users.users.root.openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIArkmq6Th79Z4klW6Urgi4phN8yq769/l/10jlE00tU9"
+    ];
+
+    services.postgresql = {
+      enable = true;
+      package =
+        postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15;
+      settings = { shared_preload_libraries = "timescaledb"; };
+    };
+
+    specialisation.postgresql15.configuration = {
+      services.postgresql = {
+        package = lib.mkForce
+          (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15);
+      };
+    };
+  };
+  testScript = { nodes, ... }: ''
+    def run_sql(query):
+        return server.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip()
+
+    def check_upgrade_path():
+        with subtest("Check timescaledb upgrade path"):
+            server.succeed("sudo -u postgres psql -c 'DROP EXTENSION IF EXISTS timescaledb;'")
+            run_sql(r"""CREATE EXTENSION timescaledb WITH VERSION \"${firstVersion}\";""")
+            installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = 'timescaledb';""")
+            assert installed_version == "${firstVersion}", f"Expected timescaledb version ${firstVersion}, but found {installed_version}"
+            for version in [${
+              lib.concatStringsSep ", " (map (s: ''"${s}"'') versions)
+            }][1:]:
+                run_sql(f"""ALTER EXTENSION timescaledb UPDATE TO '{version}';""")
+                installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = 'timescaledb';""")
+                assert installed_version == version, f"Expected timescaledb version {version}, but found {installed_version}"
+
+        start_all()
+
+        server.wait_for_unit("multi-user.target")
+        server.wait_for_unit("postgresql.service")
+
+        check_upgrade_path()
+  '';
+}