dynamically generate Nix manual version overview

CONTRIBUTING.md

@@ -27,33 +27,10 @@ Adapted from the [Contributor Covenant] and [The Carpentries Code of Conduct]:
 With the current setup, the Nix manual hosted on nix.dev does not get updated automatically with new releases.
 The following manual steps are required:
 
-- Regularly update the inputs to use the latest versions of the Nix release branches with `nix shell --run "niv update"`
-
-  To avoid long build times, make sure Nix can be fetched from the cache.
-  If it doesn't, find the latest commit that is [built by Hydra](https://hydra.nixos.org/project/nix). For example, to pin Nix 2.18:
-
-  ```bash
-  niv update nix_2-18 -r f5f4de6a550327b4b1a06123c2e450f1b92c73b6
-  ```
-
-- On each new Nix release:
-
-  1. Add the latest version in [`default.nix`](./default.nix).
-     For example, to add Nix 2.19:
-
-     ```bash
-     niv add nixos/nix -n nix_2-19 -b 2.19-maintenance
-     ```
-
-  2. Reference the latest version in [`source/reference/nix-manual.md`](./source/reference/nix-manual.md).
-
-- If an unstable or stable release of Nixpkgs adopt a new version of Nix, update the corresponding references here.
-
-  Also update URLs to the the Nix manual to the version used by Nixpkgs unstable.
-  For example, if one wants to move from 2.18 to 2.19:
-  ```bash
-  sed -i 's#https://nix.dev/manual/nix/2.18/#https://nix.dev/manual/nix/2.19/#g' $(ls **/*.md)
-  ```
+```shell-session
+nix-shell --run update-nixpkgs-releases
+nix-shell --run update-nix-releases
+```
 
 ## Contributor guides
 

default.nix

@@ -1,67 +1,68 @@
-{ inputs ? import ./nix/sources.nix
+{ inputs ? import ./nix/inputs.nix
 , system ? builtins.currentSystem
 ,
 }:
 let
-  pkgs = import inputs.nixpkgs {
+  pkgs = import inputs.nixpkgs."23.05" {
     config = { };
-    overlays = [ (import ./overlay.nix) ];
+    overlays = [ (import ./nix/overlay.nix) ];
     inherit system;
   };
 
-  nix-dev = pkgs.stdenv.mkDerivation {
-    name = "nix-dev";
-    src = ./.;
-    nativeBuildInputs = with pkgs.python310.pkgs; [
-      linkify-it-py
-      myst-parser
-      sphinx
-      sphinx-book-theme
-      sphinx-copybutton
-      sphinx-design
-      sphinx-notfound-page
-      sphinx-sitemap
-    ];
-    buildPhase = ''
-      make html
-    '';
-    installPhase =
-      let
-        # Various versions of the Nix manuals, grep for (nix-manual)= to find where they are displayed
-        # FIXME: This requires human interaction to update! See ./CONTRIBUTING.md for details.
-        releases = {
-          latest = "2.19";
-          rolling = "2.18";
-          stable = "2.18";
-          prev-stable = "2.13";
-        };
-        inputName = version: pkgs.lib.strings.replaceStrings [ "." ] [ "-" ] version;
-        src = version: inputs."nix_${inputName version}";
-        manual = version: (import (src version)).default.doc;
-        copy = version: ''
-          cp -Rf ${manual version}/share/doc/nix/manual/* $out/manual/nix/${version}
-        '';
-        # add upstream page redirects of the form `<from> <to> <status>`, excluding comment lines and empty
-        redirects = version: ''
-          sed '/^#/d;/^$/d;s#^\(.*\) \(.*\) #/manual/nix/${version}\1 /manual/nix/${version}\2 #g' ${src version}/doc/manual/_redirects >> $out/_redirects
+  lib = pkgs.lib;
+  releases = import ./nix/releases.nix { inherit lib inputs system; };
+
+  nix-dev =
+    pkgs.stdenv.mkDerivation {
+      name = "nix-dev";
+      src = ./.;
+      nativeBuildInputs = with pkgs.python310.pkgs; [
+        linkify-it-py
+        myst-parser
+        sphinx
+        sphinx-book-theme
+        sphinx-copybutton
+        sphinx-design
+        sphinx-notfound-page
+        sphinx-sitemap
+        pkgs.perl
+      ];
+      buildPhase =
+        let
+          substitutedNixManualReference = pkgs.substitute {
+            src = ./source/reference/nix-manual.md;
+            replacements = lib.concatLists (lib.mapAttrsToList (from: to: [ "--subst-var-by" from to ]) releases.substitutions);
+          };
+        in
+        ''
+          cp -f ${substitutedNixManualReference} source/reference/nix-manual.md
+          make html
         '';
-        shortlink = release: version: ''
-          echo /manual/nix/${release} /manual/nix/${version}/ 302 >> $out/_redirects
+      installPhase =
+        let
+          # Various versions of the Nix manuals, grep for (nix-manual)= to find where they are displayed.
+          # FIXME: This requires human interaction to update! See ./CONTRIBUTING.md for details.
+          release = version: nix: ''
+            cp -R --no-preserve=mode ${nix.doc}/share/doc/nix/manual $out/manual/nix/${version}
+
+            # add upstream page redirects of the form `<from> <to> <status>`, excluding comments and empty lines
+            # not all releases have that though
+            if [[ -f ${nix.doc}/share/doc/nix/manual/_redirects ]]; then
+              sed '/^#/d;/^$/d;s#^\(.*\) \(.*\) #/manual/nix/${version}\1 /manual/nix/${version}\2 #g' ${nix.doc}/share/doc/nix/manual/_redirects >> $out/_redirects
+            fi
+          '';
+          # Redirects from mutable URLs like /manual/nix/latest/... to /manual/nix/2.21/...
+          mutableRedirect = mutable: immutable: ''
+            echo "/manual/nix/${mutable}/* /manual/nix/${immutable}/:splat 302" >> $out/_redirects
+          '';
+        in
+        ''
+          mkdir -p $out/manual/nix
+          cp -R build/html/* $out/
+          ${lib.concatStringsSep "\n" (lib.mapAttrsToList release releases.nixReleases)}
+          ${lib.concatStringsSep "\n" (lib.mapAttrsToList mutableRedirect releases.mutableNixManualRedirects)}
         '';
-        versions = with pkgs.lib; lists.unique (attrsets.attrValues releases);
-      in
-      with pkgs.lib.attrsets;
-      with pkgs.lib.strings;
-      ''
-        mkdir -p $out
-        cp -R build/html/* $out/
-        # NOTE: the comma in the shell expansion makes it also work for singleton lists
-        mkdir -p $out/manual/nix/{${concatStringsSep "," versions},}
-        ${concatStringsSep "\n" (map copy versions)}
-        ${concatStringsSep "\n" (map redirects versions)}
-        ${concatStringsSep "\n" (mapAttrsToList shortlink releases)}
-      '';
-  };
+    };
 
   devmode =
     let
@@ -104,6 +105,8 @@ let
         python ${pkgs.writeText "live.py" script}
       '';
     };
+  update-nix-releases = pkgs.callPackage ./nix/update-nix-releases.nix { };
+  update-nixpkgs-releases = pkgs.callPackage ./nix/update-nixpkgs-releases.nix { };
 in
 {
   # build with `nix-build -A build`
@@ -113,6 +116,8 @@ in
     inputsFrom = [ nix-dev ];
     packages = [
       devmode
+      update-nix-releases
+      update-nixpkgs-releases
       pkgs.niv
       pkgs.python310.pkgs.black
       pkgs.vale

nix/inputs.nix

@@ -0,0 +1,36 @@
+# An abstraction over the different source pins in this directory
+{
+
+  # The main niv pins, these are without any standard names:
+  # {
+  #   <name> = <source>;
+  # }
+  main = import ./sources.nix {
+    sourcesFile = ./sources.json;
+  };
+
+  # Sources for Nix releases, the attribute name is the release version.
+  # These are done specially because updating these is non-trivial.
+  # See ./update-nix-releases.nix
+  # {
+  #   <version> = <source>;
+  # }
+  nix =
+    builtins.mapAttrs (name: value:
+      # This matches the nix-prefetch-url --unpack --name source call in ./update-nix-releases.nix
+      fetchTarball {
+        name = "source";
+        url = value.url;
+        sha256 = value.sha256;
+      }
+    ) (builtins.fromJSON (builtins.readFile ./nix-versions.json));
+
+  # Sources for Nixpkgs releases, the attribute name is the release name.
+  # These can be updated with the standard niv tooling, but are tracked separately to avoid having to filter them out during processing.
+  # See ./update-nixpkgs-releases.nix
+  nixpkgs =
+    import ./sources.nix {
+      sourcesFile = ./nixpkgs-versions.json;
+    };
+
+}

nix/nix-versions.json

@@ -0,0 +1,92 @@
+{
+  "2.21": {
+    "url": "https://github.com/nixos/nix/tarball/92e38fe30ccd7ca7e872b5da3b0fc138fed3b438",
+    "version": "2.21.2",
+    "sha256": "1wdaq3iv4dkajnvp4mn6ykq1f3pb24kgls3vggch3h08aadd2nj6"
+  },
+  "2.20": {
+    "url": "https://github.com/nixos/nix/tarball/7bc4af7301df34ea4e157338ac3792c1a9ae35b7",
+    "version": "2.20.6",
+    "sha256": "17mpp5s3045zmilhvysa29g44clcfi5wn41sslkhxm7a50k7qa85"
+  },
+  "2.19": {
+    "url": "https://github.com/nixos/nix/tarball/82f44d8633c849497d64980b5705d09ddbb01164",
+    "version": "2.19.5",
+    "sha256": "03vwahd1v8vq580cksr2grspj7iibdh5adyxdff8gq32scxahqzz"
+  },
+  "2.18": {
+    "url": "https://github.com/nixos/nix/tarball/76364a847dd39e77ad5eebfcfa86df25f518ab1f",
+    "version": "2.18.3",
+    "sha256": "1d7b3khvkrmq3m34vw75l52a83rc7f2c4al7c5nyggwclkmnh11w"
+  },
+  "2.17": {
+    "url": "https://github.com/nixos/nix/tarball/27c8b92f31fc4fb1d5be28d5196dc5ac0da58aee",
+    "version": "2.17.3",
+    "sha256": "1v1nw39j67y34hjnh9126975j1l1ha4ysfr70mnvpvsl3zb6mjvv"
+  },
+  "2.16": {
+    "url": "https://github.com/nixos/nix/tarball/d9c204fafcbb5a05aef5c36fc83f4c3d6cc4749d",
+    "version": "2.16.4",
+    "sha256": "0m3x4qgp2m61d1dd5mfqwxw5r0dgy7pzdc7zg0z9x9l7slghx3ma"
+  },
+  "2.15": {
+    "url": "https://github.com/nixos/nix/tarball/b59275b4417afd459dcf08a097fbd659f63161c2",
+    "version": "2.15.4",
+    "sha256": "0wx0z9b8abhdxcdm10mahx9dqqmwm63yrpmb5ax8ykzl7ij58wgv"
+  },
+  "2.14": {
+    "url": "https://github.com/nixos/nix/tarball/9e19aca31bc9237d699b86bce1d31aa099f09121",
+    "version": "2.14.1",
+    "sha256": "1wnaah9bpycl89bjv85b967vxvgrppfl1a8f1h7vvavhb2fcgxkb"
+  },
+  "2.13": {
+    "url": "https://github.com/nixos/nix/tarball/17e5d0325d0d7bb108d35f678622e3c237ca9fe9",
+    "version": "2.13.7",
+    "sha256": "1brlmfm2ab37hzsi5p6m13gpfhqzqkr1nvwkg2ks8ij1mcz7dh67"
+  },
+  "2.12": {
+    "url": "https://github.com/nixos/nix/tarball/3c1f09ca492f5ec2252b511c9b627eddd2e98f88",
+    "version": "2.12.2",
+    "sha256": "14bjd85i3qm8jzzyafv1jcsdq5ylrp96563141dimc7p2085jczs"
+  },
+  "2.11": {
+    "url": "https://github.com/nixos/nix/tarball/1aac1884c5d5976cc74a0a4e193e0b0ca3f25dbf",
+    "version": "2.11.2",
+    "sha256": "0ivcvz7g838sbh05vgzirz0agbr1dbr3gw1ywmxzmm3bh75hw2gs"
+  },
+  "2.10": {
+    "url": "https://github.com/nixos/nix/tarball/73b3ba878a86e9a4d5e46fb16ce735b7dea4c929",
+    "version": "2.10.3",
+    "sha256": "0s7i6mj3x9l5l3b73fp4jq0cl1sjah7lq11gkymzb2jrqhxc681z"
+  },
+  "2.9": {
+    "url": "https://github.com/nixos/nix/tarball/338418790adac11a4e73218e40fa109c4cfbc86d",
+    "version": "2.9.3",
+    "sha256": "0kl36c365dd9mzjcr5py52lgm78hj81cvw7lws9f16zkj64hcy9v"
+  },
+  "2.8": {
+    "url": "https://github.com/nixos/nix/tarball/4555784afb613247563896273dad7887c1a2a0a3",
+    "version": "2.8.2",
+    "sha256": "0zg7y4c1fjz241fspzmba3acjw1qbigvkngp7q2xb5mdm5sl6swp"
+  },
+  "2.7": {
+    "url": "https://github.com/nixos/nix/tarball/5ed3a9db6add8b28bdb484c00f45033f50ccb853",
+    "version": "2.7.0",
+    "sha256": "03y59yxc32ghkfrfkrrjkhpnksyp5q29mlhq1ssxia29wjmv5mz3"
+  },
+  "2.6": {
+    "url": "https://github.com/nixos/nix/tarball/e044ccb67ce38d11059de32515306f5f1bd2f04f",
+    "version": "2.6.1",
+    "sha256": "0pfx87zgyhhhf4mnrknzq0pgry8i2d5pxvsbf52sqrxxzznr1n0k"
+  },
+  "2.5": {
+    "url": "https://github.com/nixos/nix/tarball/16b3e41ed0e4b6c2ceffcbe8956acdd8b0cde71d",
+    "version": "2.5.1",
+    "sha256": "1px45lab0yb9gx6bysr296w3bc6cv1cm3x3lik1hhi3k5fpfc3pn"
+  },
+  "2.4": {
+    "url": "https://github.com/nixos/nix/tarball/fbe3b09248feb251f7baf24dc5cd6d675ec0cb2c",
+    "version": "2.4.1",
+    "sha256": "0kzn3ggmir8iv1nk4yds6676x8y4q72xv06ks5p2nrvazhdarl8y"
+  }
+}