From a6c4843806f3d6abc0716530b60e9ea6d18439d9 Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Mon, 26 May 2025 10:20:20 +0200 Subject: [PATCH 1/4] add pip_compile function and integrate uv support for requirements generation --- example/requirements.txt | 5 ++- src/appenv.py | 85 +++++++++++++++------------------------- 2 files changed, 35 insertions(+), 55 deletions(-) diff --git a/example/requirements.txt b/example/requirements.txt index 50242ed..efa4719 100644 --- a/example/requirements.txt +++ b/example/requirements.txt @@ -1,3 +1,4 @@ -# appenv-requirements-hash: 23726ae5a1e34c3fd5735728c69f2c436b6801d9ac7df6e3d0d5eb7d03fc2a0d +# This file was autogenerated by uv via the following command: +# uv pip compile --output-file /Users/elikoga/appenv/example/./requirements.txt requirements.in ducker==2.0.1 -setuptools==65.5.0 + # via -r requirements.in diff --git a/src/appenv.py b/src/appenv.py index 4533a36..9b6e1d9 100755 --- a/src/appenv.py +++ b/src/appenv.py @@ -62,6 +62,14 @@ def pip(path, c, **kwargs): return python(path, ["-m", "pip"] + c, **kwargs) +def pip_compile(use_uv, path, c, **kwargs): + if use_uv: + return cmd([os.path.join(path, "bin/uv"), "pip", "compile"] + c, + **kwargs) + else: + return cmd([os.path.join(path, "bin/pip-compile")] + c, **kwargs) + + def get(host, path, f): conn = http.client.HTTPSConnection(host) conn.request("GET", path) @@ -141,6 +149,19 @@ def ensure_venv(target): python(target, ["-m", "ensurepip", "--default-pip"]) pip(target, ["install", "--upgrade", "pip"]) + # try to install uv + print("Ensuring uv ...") + uses_uv_pip_compile = False + try: + pip(target, ["install", "uv"]) + uses_uv_pip_compile = True + except ValueError: + print("uv not available, falling back to pip-compile") + # pip-compile is not available in the venv, so we need to install it + # in the system python + cmd(["pip", "install", "pip-tools"]) + return uses_uv_pip_compile + def parse_preferences(): preferences = None @@ -520,59 +541,17 @@ def update_lockfile(self, args=None, remaining=None): tmpdir = os.path.join(self.appenv_dir, "updatelock") if os.path.exists(tmpdir): cmd(["rm", "-rf", tmpdir]) - ensure_venv(tmpdir) - print("Installing packages ...") - pip(tmpdir, ["install", "-r", "requirements.in"]) - - extra_specs = [] - result = pip( - tmpdir, ["freeze", "--all", "--exclude", "pip"], - merge_stderr=False).decode('ascii') - # They changed this behaviour in https://github.com/pypa/pip/pull/12032 - pinned_versions = {} - for line in result.splitlines(): - if line.strip().startswith('-e '): - # We'd like to pick up the original -e statement here. - continue - parsed_requirement = parse_requirement_string(line) - pinned_versions[parsed_requirement.name] = parsed_requirement - requested_versions = {} - with open('requirements.in') as f: - for line in f.readlines(): - if line.strip().startswith('-e '): - extra_specs.append(line.strip()) - continue - if line.strip().startswith('--'): - extra_specs.append(line.strip()) - continue - - # filter comments, in particular # appenv-python-preferences - if line.strip().startswith('#'): - continue - parsed_requirement = parse_requirement_string(line) - requested_versions[ - parsed_requirement.name] = parsed_requirement - - final_versions = {} - for spec in requested_versions.values(): - # Pick versions with URLs to ensure we don't get the screwed up - # results from pip freeze. - if spec.url: - final_versions[spec.name] = spec - for spec in pinned_versions.values(): - # Ignore versions we already picked - if spec.name in final_versions: - continue - final_versions[spec.name] = spec - lines = [str(spec) for spec in final_versions.values()] - lines.extend(extra_specs) - lines.sort() - with open(os.path.join(self.base, "requirements.txt"), "w") as f: - f.write('# appenv-requirements-hash: {}\n'.format( - self._hash_requirements())) - f.write('\n'.join(lines)) - f.write('\n') - cmd(["rm", "-rf", tmpdir]) + has_uv = ensure_venv(tmpdir) + # print("Installing packages ...") + # use uv pip compile or pip-compile to generate the requirements.txt + + pip_compile( + has_uv, + tmpdir, [ + "--output-file", + os.path.join(self.base, "requirements.txt"), "requirements.in" + ], + merge_stderr=False) def main(): From 6383cb58456c0570c682cdf28bba2b801ac9ed3f Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Mon, 2 Jun 2025 00:48:59 +0200 Subject: [PATCH 2/4] add --generate-hashes option to requirements generation --- src/appenv.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/appenv.py b/src/appenv.py index 9b6e1d9..48b943c 100755 --- a/src/appenv.py +++ b/src/appenv.py @@ -549,8 +549,9 @@ def update_lockfile(self, args=None, remaining=None): has_uv, tmpdir, [ "--output-file", - os.path.join(self.base, "requirements.txt"), "requirements.in" - ], + os.path.join(self.base, "requirements.txt"), + "requirements.in" + "--generate-hashes",], merge_stderr=False) From 84ffd8af7609d546fabdd6128ba0cdf489e427a0 Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Mon, 2 Jun 2025 00:53:20 +0200 Subject: [PATCH 3/4] add --generate-hashes option to requirements generation in AppEnv class --- src/appenv.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/appenv.py b/src/appenv.py index 48b943c..13768f5 100755 --- a/src/appenv.py +++ b/src/appenv.py @@ -549,9 +549,8 @@ def update_lockfile(self, args=None, remaining=None): has_uv, tmpdir, [ "--output-file", - os.path.join(self.base, "requirements.txt"), - "requirements.in" - "--generate-hashes",], + os.path.join(self.base, "requirements.txt"), "requirements.in", + "--generate-hashes"], merge_stderr=False) From 24d9453147b323091765258298e30cc07e19875f Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Sun, 29 Jun 2025 15:43:50 +0200 Subject: [PATCH 4/4] add requirements hash generation and update requirements file output --- example/.appenv/current | 1 + example/requirements.txt | 7 ++++--- src/appenv.py | 16 ++++++++++++---- 3 files changed, 17 insertions(+), 7 deletions(-) create mode 120000 example/.appenv/current diff --git a/example/.appenv/current b/example/.appenv/current new file mode 120000 index 0000000..cfb020b --- /dev/null +++ b/example/.appenv/current @@ -0,0 +1 @@ +1060a47a \ No newline at end of file diff --git a/example/requirements.txt b/example/requirements.txt index efa4719..5cfc7f9 100644 --- a/example/requirements.txt +++ b/example/requirements.txt @@ -1,4 +1,5 @@ -# This file was autogenerated by uv via the following command: -# uv pip compile --output-file /Users/elikoga/appenv/example/./requirements.txt requirements.in -ducker==2.0.1 +# appenv-requirements-hash: 23726ae5a1e34c3fd5735728c69f2c436b6801d9ac7df6e3d0d5eb7d03fc2a0d +ducker==2.0.1 \ + --hash=sha256:159b84ee1103e495bbf89a69922f64f98e73f92ee76acea53a207cd45fc80ac9 \ + --hash=sha256:50131934a2cd8aa62af8ff5e114463c7639c1fd8bb1a0c70102a351a64afe40a # via -r requirements.in diff --git a/src/appenv.py b/src/appenv.py index 13768f5..7ddf380 100755 --- a/src/appenv.py +++ b/src/appenv.py @@ -545,14 +545,22 @@ def update_lockfile(self, args=None, remaining=None): # print("Installing packages ...") # use uv pip compile or pip-compile to generate the requirements.txt - pip_compile( + requirements_out = pip_compile( has_uv, tmpdir, [ - "--output-file", - os.path.join(self.base, "requirements.txt"), "requirements.in", - "--generate-hashes"], + "requirements.in", "--generate-hashes", "--no-header", + "--upgrade"], merge_stderr=False) + requirements_out = requirements_out.decode("utf-8", "replace") + # prepend appenv-requirements-hash + requirements_hash = self._hash_requirements() + requirements_out = ( + "# appenv-requirements-hash: {}\n".format(requirements_hash) + + requirements_out) + with open("requirements.txt", "w") as f: + f.write(requirements_out) + def main(): base = os.path.dirname(__file__)