Move pkgs/env.txt lockfile to conda-meta/initial-state.explicit.txt to preserve it (#1059)

Co-authored-by: Marco Esters <[email protected]>

Author: jaimergp

.gitignore

@@ -159,3 +159,6 @@ build/
 
 # setuptools-scm
 constructor/_version.py
+
+# Temporary workspaces
+tmp/

constructor/header.sh

@@ -598,8 +598,7 @@ CONDA_EXTRA_SAFETY_CHECKS=no \
 CONDA_CHANNELS="{{ channels }}" \
 CONDA_PKGS_DIRS="$PREFIX/pkgs" \
 CONDA_QUIET="$BATCH" \
-"$CONDA_EXEC" install --offline --file "$PREFIX/pkgs/env.txt" -yp "$PREFIX" $shortcuts {{ no_rcs_arg }} || exit 1
-rm -f "$PREFIX/pkgs/env.txt"
+"$CONDA_EXEC" install --offline --file "$PREFIX/conda-meta/initial-state.explicit.txt" -yp "$PREFIX" $shortcuts {{ no_rcs_arg }} || exit 1
 
 {%- if has_conda %}
 mkdir -p "$PREFIX/envs"
@@ -638,8 +637,7 @@ for env_pkgs in "${PREFIX}"/pkgs/envs/*/; do
     CONDA_CHANNELS="$env_channels" \
     CONDA_PKGS_DIRS="$PREFIX/pkgs" \
     CONDA_QUIET="$BATCH" \
-    "$CONDA_EXEC" install --offline --file "${env_pkgs}env.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts {{ no_rcs_arg }} || exit 1
-    rm -f "${env_pkgs}env.txt"
+    "$CONDA_EXEC" install --offline --file "$PREFIX/envs/$env_name/conda-meta/initial-state.explicit.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts {{ no_rcs_arg }} || exit 1
 done
 {%- endif %}
 

constructor/nsis/main.nsi.tmpl

@@ -1375,13 +1375,12 @@ Section "Install"
     ${Print} "Setting up the {{ env.name }} environment..."
     SetDetailsPrint listonly
 
-    # List of packages to install
-    SetOutPath "{{ env.env_txt_dir }}"
-    File "{{ env.env_txt_abspath }}"
 
     # A conda-meta\history file is required for a valid conda prefix
     SetOutPath "{{ env.conda_meta }}"
     File "{{ env.history_abspath }}"
+    # List of packages to install, as a lockfile
+    File "{{ env.lockfile_txt_abspath }}"
 
     # Set channels
     System::Call 'kernel32::SetEnvironmentVariable(t,t)i("CONDA_CHANNELS", "{{ channels }}").r0'
@@ -1391,21 +1390,17 @@ Section "Install"
     # Run conda install
     ${If} $Ana_CreateShortcuts_State = ${BST_CHECKED}
         ${Print} "Installing packages for {{ env.name }}, creating shortcuts if necessary..."
-        push '"$INSTDIR\_conda.exe" install --offline -yp "{{ env.prefix }}" --file "{{ env.env_txt }}" {{ env.shortcuts }} {{ env.no_rcs_arg }}'
+        push '"$INSTDIR\_conda.exe" install --offline -yp "{{ env.prefix }}" --file "{{ env.lockfile_txt }}" {{ env.shortcuts }} {{ env.no_rcs_arg }}'
     ${Else}
         ${Print} "Installing packages for {{ env.name }}..."
-        push '"$INSTDIR\_conda.exe" install --offline -yp "{{ env.prefix }}" --file "{{ env.env_txt }}" --no-shortcuts {{ env.no_rcs_arg }}'
+        push '"$INSTDIR\_conda.exe" install --offline -yp "{{ env.prefix }}" --file "{{ env.lockfile_txt }}" --no-shortcuts {{ env.no_rcs_arg }}'
     ${EndIf}
     push 'Failed to link extracted packages to {{ env.prefix }}!'
     push 'WithLog'
     SetDetailsPrint listonly
     call AbortRetryNSExecWait
     SetDetailsPrint both
 
-    # Cleanup {{ env.name }} env.txt
-    SetOutPath "$INSTDIR"
-    Delete "{{ env.env_txt }}"
-
     # Restore shipped conda-meta\history for remapped
     # channels and retain only the first transaction
     SetOutPath "{{ env.conda_meta }}"

constructor/osx/run_installation.sh

@@ -45,6 +45,11 @@ fi
 
 # Perform the conda install
 notify "Installing packages. This might take a few minutes."
+
+# 'install' below will modify the history file in a way we don't want;
+# keep a copy to restore later
+cp "$PREFIX/conda-meta/history" "$PREFIX/conda-meta/history.bak"
+
 # shellcheck disable=SC2086
 if ! \
 CONDA_REGISTER_ENVS="{{ register_envs }}" \
@@ -53,14 +58,13 @@ CONDA_SAFETY_CHECKS=disabled \
 CONDA_EXTRA_SAFETY_CHECKS=no \
 CONDA_CHANNELS={{ channels }} \
 CONDA_PKGS_DIRS="$PREFIX/pkgs" \
-"$CONDA_EXEC" install --offline --file "$PREFIX/pkgs/env.txt" -yp "$PREFIX" $shortcuts {{ no_rcs_arg }}; then
+"$CONDA_EXEC" install --offline --file "$PREFIX/conda-meta/initial-state.explicit.txt" -yp "$PREFIX" $shortcuts {{ no_rcs_arg }}; then
     echo "ERROR: could not complete the conda install"
     exit 1
 fi
 
-# Move the prepackaged history file into place
-mv "$PREFIX/pkgs/conda-meta/history" "$PREFIX/conda-meta/history"
-rm -f "$PREFIX/env.txt"
+# Restore history file as provided by installer
+mv "$PREFIX/conda-meta/history.bak" "$PREFIX/conda-meta/history"
 
 # Same, but for the extra environments
 
@@ -73,8 +77,9 @@ for env_pkgs in "${PREFIX}"/pkgs/envs/*/; do
     fi
 
     notify "Installing ${env_name} packages..."
-    mkdir -p "$PREFIX/envs/$env_name/conda-meta"
-    touch "$PREFIX/envs/$env_name/conda-meta/history"
+    # 'install' below will modify the history file in a way we don't want;
+    # keep a copy to restore later
+    cp "$PREFIX/envs/$env_name/conda-meta/history" "$PREFIX/envs/$env_name/conda-meta/history.bak"
 
     if [[ -f "${env_pkgs}channels.txt" ]]; then
         env_channels="$(cat "${env_pkgs}channels.txt")"
@@ -99,10 +104,10 @@ for env_pkgs in "${PREFIX}"/pkgs/envs/*/; do
     CONDA_EXTRA_SAFETY_CHECKS=no \
     CONDA_CHANNELS="$env_channels" \
     CONDA_PKGS_DIRS="$PREFIX/pkgs" \
-    "$CONDA_EXEC" install --offline --file "${env_pkgs}env.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts {{ no_rcs_arg }} || exit 1
-    # Move the prepackaged history file into place
-    mv "${env_pkgs}/conda-meta/history" "$PREFIX/envs/$env_name/conda-meta/history"
-    rm -f "${env_pkgs}env.txt"
+    "$CONDA_EXEC" install --offline --file "$PREFIX/envs/$env_name/conda-meta/initial-state.explicit.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts {{ no_rcs_arg }} || exit 1
+
+    # Restore history file as provided by installer
+    mv "$PREFIX/envs/$env_name/conda-meta/history.bak" "$PREFIX/envs/$env_name/conda-meta/history"
 done
 
 # Cleanup!

constructor/osxpkg.py

@@ -563,7 +563,7 @@ def create(info, verbose=False):
     fresh_dir(SCRIPTS_DIR)
     pkgs_dir = join(prefix, "pkgs")
     os.makedirs(pkgs_dir)
-    preconda.write_files(info, pkgs_dir)
+    preconda.write_files(info, prefix)
     preconda.copy_extra_files(info.get("extra_files", []), prefix)
     # These are the user-provided scripts, maybe patched to have a shebang
     # They will be called by a wrapping script added later, if present

constructor/preconda.py

@@ -48,7 +48,12 @@
 except ImportError:
     import ruamel_json as json
 
-files = ".constructor-build.info", "urls", "urls.txt", "env.txt"
+files = (
+    "pkgs/.constructor-build.info",
+    "pkgs/urls",
+    "pkgs/urls.txt",
+    "conda-meta/initial-state.explicit.txt",
+)
 
 
 def write_index_cache(info, dst_dir, used_packages):
@@ -135,8 +140,28 @@ def system_info():
     return out
 
 
-def write_files(info, dst_dir):
-    with open(join(dst_dir, ".constructor-build.info"), "w") as fo:
+def write_files(info: dict, workspace: str):
+    """
+    Prepare files on disk to be shipped as part of the pre-conda payload, mostly
+    configuration and metadata files:
+
+    - `conda-meta/initial-state.explicit.txt`: Lockfile to provision the base environment.
+    - `conda-meta/history`: Prepared history file with the right requested specs in input file.
+    - `pkgs/urls` and `pkgs/urls.txt`: Direct URLs of packages used, with and without MD5 hashes.
+    - `pkgs/cache/*.json`: Trimmed repodata to mock offline channels in use.
+    - `pkgs/channels.txt`: Channels in use.
+    - `pkgs/shortcuts.txt`: Which packages should have their shortcuts created, if any.
+
+    If extra envs are requested, this will also write:
+
+    - Their corresponding `envs/<env-name>/conda-meta/` files.
+    - Their corresponding `pkgs/channels.txt` and `pkgs/shortcuts.txt` under
+      `pkgs/envs/<env-name>`.
+    """
+    os.makedirs(join(workspace, "conda-meta"), exist_ok=True)
+    pkgs_dir = join(workspace, "pkgs")
+    os.makedirs(pkgs_dir, exist_ok=True)
+    with open(join(pkgs_dir, ".constructor-build.info"), "w") as fo:
         json.dump(system_info(), fo)
 
     all_urls = info["_urls"].copy()
@@ -146,15 +171,15 @@ def write_files(info, dst_dir):
     final_urls_md5s = tuple((get_final_url(info, url), md5) for url, md5 in info["_urls"])
     all_final_urls_md5s = tuple((get_final_url(info, url), md5) for url, md5 in all_urls)
 
-    with open(join(dst_dir, "urls"), "w") as fo:
+    with open(join(pkgs_dir, "urls"), "w") as fo:
         for url, md5 in all_final_urls_md5s:
             maybe_different_url = ensure_transmuted_ext(info, url)
             if maybe_different_url != url:  # transmuted, no md5
                 fo.write(f"{maybe_different_url}\n")
             else:
                 fo.write(f"{url}#{md5}\n")
 
-    with open(join(dst_dir, "urls.txt"), "w") as fo:
+    with open(join(pkgs_dir, "urls.txt"), "w") as fo:
         for url, _ in all_final_urls_md5s:
             fo.write("%s\n" % url)
 
@@ -163,33 +188,36 @@ def write_files(info, dst_dir):
         all_dists += env_info["_dists"]
     all_dists = list({dist: None for dist in all_dists})  # de-duplicate
 
-    write_index_cache(info, dst_dir, all_dists)
+    write_index_cache(info, pkgs_dir, all_dists)
 
     # base environment conda-meta
-    write_conda_meta(info, dst_dir, final_urls_md5s)
+    write_conda_meta(info, join(workspace, "conda-meta"), final_urls_md5s)
 
-    write_repodata_record(info, dst_dir)
+    write_repodata_record(info, pkgs_dir)
 
     # base environment file used with conda install --file
     # (list of specs/dists to install)
-    write_env_txt(info, dst_dir, final_urls_md5s)
+    write_initial_state_explicit_txt(info, join(workspace, "conda-meta"), final_urls_md5s)
 
     for fn in files:
-        os.chmod(join(dst_dir, fn), 0o664)
+        os.chmod(join(workspace, fn), 0o664)
 
     for env_name, env_info in info.get("_extra_envs_info", {}).items():
         env_config = info["extra_envs"][env_name]
-        env_dst_dir = os.path.join(dst_dir, "envs", env_name)
+        env_pkgs = os.path.join(workspace, "pkgs", "envs", env_name)
+        env_conda_meta = os.path.join(workspace, "envs", env_name, "conda-meta")
+        os.makedirs(env_pkgs, exist_ok=True)
+        os.makedirs(env_conda_meta, exist_ok=True)
         # environment conda-meta
         env_urls_md5 = tuple((get_final_url(info, url), md5) for url, md5 in env_info["_urls"])
         user_requested_specs = env_config.get("user_requested_specs", env_config.get("specs", ()))
-        write_conda_meta(info, env_dst_dir, env_urls_md5, user_requested_specs)
+        write_conda_meta(info, env_conda_meta, env_urls_md5, user_requested_specs)
         # environment installation list
-        write_env_txt(info, env_dst_dir, env_urls_md5)
+        write_initial_state_explicit_txt(info, env_conda_meta, env_urls_md5)
         # channels
-        write_channels_txt(info, env_dst_dir, env_config)
+        write_channels_txt(info, env_pkgs, env_config)
         # shortcuts
-        write_shortcuts_txt(info, env_dst_dir, env_config)
+        write_shortcuts_txt(info, env_pkgs, env_config)
 
 
 def write_conda_meta(info, dst_dir, final_urls_md5s, user_requested_specs=None):
@@ -212,9 +240,7 @@ def write_conda_meta(info, dst_dir, final_urls_md5s, user_requested_specs=None):
         builder.append("# update specs: %s" % update_specs)
     builder.append("\n")
 
-    if not isdir(join(dst_dir, "conda-meta")):
-        os.makedirs(join(dst_dir, "conda-meta"))
-    with open(join(dst_dir, "conda-meta", "history"), "w") as fh:
+    with open(join(dst_dir, "history"), "w") as fh:
         fh.write("\n".join(builder))
 
 
@@ -245,7 +271,7 @@ def write_repodata_record(info, dst_dir):
             json.dump(rr_json, rf, indent=2, sort_keys=True)
 
 
-def write_env_txt(info, dst_dir, urls):
+def write_initial_state_explicit_txt(info, dst_dir, urls):
     """
     urls is an iterable of tuples with url and md5 values
     """
@@ -257,7 +283,7 @@ def write_env_txt(info, dst_dir, urls):
         @EXPLICIT
         """
     ).lstrip()
-    with open(join(dst_dir, "env.txt"), "w") as envf:
+    with open(join(dst_dir, "initial-state.explicit.txt"), "w") as envf:
         envf.write(header)
         for url, md5 in urls:
             maybe_different_url = ensure_transmuted_ext(info, url)

constructor/shar.py

@@ -131,15 +131,15 @@ def create(info, verbose=False):
     postconda_tarball = join(tmp_dir, "postconda.tar.bz2")
     pre_t = tarfile.open(preconda_tarball, "w:bz2")
     post_t = tarfile.open(postconda_tarball, "w:bz2")
-    for dist in preconda_files:
-        fn = filename_dist(dist)
-        pre_t.add(join(tmp_dir, fn), "pkgs/" + fn)
+    for rel_path in preconda_files:
+        pre_t.add(join(tmp_dir, rel_path), rel_path)
 
     for env_name in info.get("_extra_envs_info", ()):
-        pre_t.add(join(tmp_dir, "envs", env_name, "env.txt"), f"pkgs/envs/{env_name}/env.txt")
-        pre_t.add(
-            join(tmp_dir, "envs", env_name, "shortcuts.txt"), f"pkgs/envs/{env_name}/shortcuts.txt"
-        )
+        for rel_path in (
+            f"pkgs/envs/{env_name}/shortcuts.txt",
+            f"envs/{env_name}/conda-meta/initial-state.explicit.txt",
+        ):
+            pre_t.add(join(tmp_dir, rel_path), rel_path)
 
     for key in "pre_install", "post_install":
         if key in info:
@@ -165,7 +165,7 @@ def create(info, verbose=False):
         elif filename_dist(dist).endswith(".tar.bz2"):
             _dist = filename_dist(dist)[:-8]
         record_file = join(_dist, "info", "repodata_record.json")
-        record_file_src = join(tmp_dir, record_file)
+        record_file_src = join(tmp_dir, "pkgs", record_file)
         record_file_dest = join("pkgs", record_file)
         pre_t.add(record_file_src, record_file_dest)
     pre_t.addfile(tarinfo=tarfile.TarInfo("conda-meta/history"))
@@ -185,7 +185,7 @@ def create(info, verbose=False):
     pre_t.close()
     post_t.close()
 
-    tarball = join(tmp_dir, "tmp.tar")
+    tarball = join(tmp_dir, "pkgs", "tmp.tar")
     t = tarfile.open(tarball, "w")
     t.add(preconda_tarball, basename(preconda_tarball))
     t.add(postconda_tarball, basename(postconda_tarball))

constructor/winexe.py

@@ -81,11 +81,10 @@ def setup_envs_commands(info, dir_path):
         {
             "name": "base",
             "prefix": r"$INSTDIR",
-            "env_txt": r"$INSTDIR\pkgs\env.txt",  # env.txt as seen by the running installer
-            "env_txt_dir": r"$INSTDIR\pkgs",  # env.txt location in the installer filesystem
-            "env_txt_abspath": join(
-                dir_path, "env.txt"
-            ),  # env.txt path while building the installer
+            # initial-state.explicit.txt as seen by the running installer
+            "lockfile_txt": r"$INSTDIR\conda-meta\initial-state.explicit.txt",
+            # initial-state.explicit.txt path while building the installer
+            "lockfile_txt_abspath": join(dir_path, "conda-meta", "initial-state.explicit.txt"),
             "conda_meta": r"$INSTDIR\conda-meta",
             "history_abspath": join(dir_path, "conda-meta", "history"),
             "final_channels": get_final_channels(info),
@@ -108,9 +107,12 @@ def setup_envs_commands(info, dir_path):
             {
                 "name": env_name,
                 "prefix": join("$INSTDIR", "envs", env_name),
-                "env_txt": join("$INSTDIR", "pkgs", "envs", env_name, "env.txt"),
-                "env_txt_dir": join("$INSTDIR", "pkgs", "envs", env_name),
-                "env_txt_abspath": join(dir_path, "envs", env_name, "env.txt"),
+                "lockfile_txt": join(
+                    "$INSTDIR", "envs", env_name, "conda-meta", "initial-state.explicit.txt"
+                ),
+                "lockfile_txt_abspath": join(
+                    dir_path, "envs", env_name, "conda-meta", "initial-state.explicit.txt"
+                ),
                 "conda_meta": join("$INSTDIR", "envs", env_name, "conda-meta"),
                 "history_abspath": join(dir_path, "envs", env_name, "conda-meta", "history"),
                 "final_channels": get_final_channels(channel_info),
@@ -169,20 +171,20 @@ def make_nsi(
         "outfile": info["_outpath"],
         "vipv": make_VIProductVersion(info["version"]),
         "constructor_version": info["CONSTRUCTOR_VERSION"],
+        # @-prefixed paths point to {dir_path}
         "iconfile": "@icon.ico",
         "headerimage": "@header.bmp",
         "welcomeimage": "@welcome.bmp",
         "licensefile": abspath(info.get("license_file", join(NSIS_DIR, "placeholder_license.txt"))),
         "conda_history": "@" + join("conda-meta", "history"),
         "conda_exe": "@_conda.exe",
-        "env_txt": "@env.txt",
-        "urls_file": "@urls",
-        "urls_txt_file": "@urls.txt",
+        "urls_file": "@" + join("pkgs", "urls"),
+        "urls_txt_file": "@" + join("pkgs", "urls.txt"),
         "pre_install": "@pre_install.bat",
         "post_install": "@post_install.bat",
         "pre_uninstall": "@pre_uninstall.bat",
-        "index_cache": "@cache",
-        "repodata_record": "@repodata_record.json",
+        "index_cache": "@" + join("pkgs", "cache"),
+        "repodata_record": "@" + join("pkgs", "repodata_record.json"),
     }
 
     conclusion_text = info.get("conclusion_text", "")

news/1059-lockfiles

@@ -0,0 +1,19 @@
+### Enhancements
+
+* Ship `conda-meta/initial-state.explicit.txt` as a copy of the lockfile that provisions the initial state of each environment. (#1052 via #1059)
+
+### Bug fixes
+
+* <news item>
+
+### Deprecations
+
+* <news item>
+
+### Docs
+
+* <news item>
+
+### Other
+
+* <news item>