From c5b0406d010118c96831abdd36ca9d0a13048206 Mon Sep 17 00:00:00 2001 From: Laura Promberger Date: Thu, 20 Nov 2025 18:25:21 +0100 Subject: [PATCH] Python script to promote a version from prerelease to release (#1973) Allows to promote .tar.gz and wheels from release candidate to release (e.g. 7.10.0rc1 --> 7.10.0). Can work on a single file, partial matches or entire directory. Creates new files for the promotion. Has an option to delete the old prerelease version on successful promotion. As we have a few custom files that also contain the version, added a modified third-party `change_wheel_version.py` to be able to also change the version in those non-standard files inside the wheel. Works on Windows and Linux. Includes a promotion test. Issue #1891 --------- Co-authored-by: Scott Todd --- .../packaging/promote_from_rc_to_final.py | 361 ++++++++++++++ build_tools/packaging/requirements.txt | 4 + .../tests/promote_from_rc_to_final_test.py | 468 ++++++++++++++++++ build_tools/setup_venv.py | 24 +- .../third_party/change_wheel_version/LICENSE | 21 + .../change_wheel_version.py | 254 ++++++++++ 6 files changed, 1123 insertions(+), 9 deletions(-) create mode 100644 build_tools/packaging/promote_from_rc_to_final.py create mode 100644 build_tools/packaging/requirements.txt create mode 100644 build_tools/packaging/tests/promote_from_rc_to_final_test.py create mode 100644 build_tools/third_party/change_wheel_version/LICENSE create mode 100644 build_tools/third_party/change_wheel_version/change_wheel_version.py diff --git a/build_tools/packaging/promote_from_rc_to_final.py b/build_tools/packaging/promote_from_rc_to_final.py new file mode 100644 index 0000000000..282bbb14bf --- /dev/null +++ b/build_tools/packaging/promote_from_rc_to_final.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python +"""Promotes release candidate packages to final releases. + +This script removes release candidate (rc) suffixes from package versions, such +as promoting 7.10.0rc1 to 7.10.0. It handles both Python wheels (.whl) and +source distributions (.tar.gz). This only updates the version strings in the packages but +does not change any other package content. + +PREREQUISITES: + - pip install -r ./build_tools/packaging/requirements.txt + +SIDE EFFECTS: + - Creates NEW promoted package files side-by-side with the original files + - By default, DOES NOT delete original RC files (safe to run with no --delete flag) + - With --delete-old-on-success flag, removes original RC files after promotion + - If no arguments are given, it will promote all RC packages in the current directory. + +TYPICAL USAGE: + # Promote all RC packages in a directory (keeps original RC files): + python ./build_tools/packaging/promote_from_rc_to_final.py --input-dir=./release_candidates/rc1/ + + # Promote and delete original RC files on success: + python ./build_tools/packaging/promote_from_rc_to_final.py --input-dir=./release_candidates/rc1/ --delete-old-on-success + + # Promote only specific files matching a pattern: + python ./build_tools/packaging/promote_from_rc_to_final.py --input-dir=./release_candidates/ --match-files='*rc2*' + + # Promote some nightly for testing + python ./build_tools/packaging/promote_from_rc_to_final.py --input-dir=./release_candidates/ --release-type="a" + +TESTING: + python ./build_tools/packaging/tests/promote_from_rc_to_final_test.py +""" + +import argparse +import shutil +from packaging.version import Version +import pathlib +from pkginfo import Wheel +import sys +import tempfile +import tarfile +import fileinput +import os +import importlib.util + +# Need dynamic load as change_wheel_version needs to be imported via parent directory +this_file = pathlib.Path(__file__).resolve() +build_tools_dir = this_file.parent.parent +# ../third_party/change_wheel_version/change_wheel_version.py +change_wheel_version_path = ( + build_tools_dir / "third_party" / "change_wheel_version" / "change_wheel_version.py" +) + +spec = importlib.util.spec_from_file_location( + "third_party_change_wheel_version", change_wheel_version_path +) +change_wheel_version = importlib.util.module_from_spec(spec) +assert spec.loader is not None +spec.loader.exec_module(change_wheel_version) + +assert hasattr( + change_wheel_version, "change_wheel_version" +), "change_wheel_version module does not expose change_wheel_version function" + + +def parse_arguments(argv): + parser = argparse.ArgumentParser( + description="""Promotes packages from release candidate to final release (e.g. 7.10.0rc1 --> 7.10.0). + +Promotion works for for wheels and .tar.gz. +Wheels version is determined by python library to interact with the wheel. +For tar.gz., the version is extract from <.tar.gz>/PKG-INFO file. +""", + usage="python ./build_tools/packaging/promote_from_rc_to_final.py --input-dir=./release_candidates/rc1/ --delete-old-on-success", + ) + parser.add_argument( + "--input-dir", + help="Path to the directory that contains .whl and .tar.gz files to promote", + type=pathlib.Path, + required=True, + ) + parser.add_argument( + "--match-files", + help="Limits selection in '--input-dir' to files matchings this argument. Use wild cards if needed, e.g. '*rc2*' (default '*' to promote all files in '--input-dir')", + default="*", + ) + parser.add_argument( + "--delete-old-on-success", + help="Deletes old file after successful promotion", + action="store_true", + ) + parser.add_argument( + "--prerelease-type", + help="Prerelease type to use instead of 'rc' (e.g. 'dev' or 'a')", + nargs="+", + default=["rc"], + choices=["rc", "dev", "a"], + ) + + return parser.parse_args(argv) + + +def wheel_change_extra_files(new_dir_path: pathlib.Path, old_version, new_version): + # extract "rocm_sdk_core" from /tmp/tmp3swrl25j/wheel/rocm_sdk_core-7.10.0 + package_name_no_version = new_dir_path.name.split(str(new_version))[0][:-1] + + # correct capitalization and hyphenation + # of interest for amdgpu arch: wheels are all lower case + # (e.g. rocm_sdk_libraries_gfx94x_dcgpu-7.10.0rc1-py3-none-linux_x86_64.whl) + # but inside we have to match to rocm_sdk_libraries_gfx94X-dcgpu/ with a capital "X-dcgpu" instead of "x_dcgpu" + if "gfx" in package_name_no_version: + files = list(new_dir_path.glob("*gfx*")) + for file in files: + if len(file.name) == len(package_name_no_version): + package_name_no_version = file.name + + old_rocm_version = ( + str(old_version) + if not "rocm" in str(old_version) + else str(old_version).split("+rocm")[-1] + ) + new_rocm_version = ( + str(new_version) + if not "rocm" in str(new_version) + else str(new_version).split("+rocm")[-1] + ) + + print(" Changing ROCm-specific files that contain the version") + + if not "torch" in new_dir_path.name: # rocm packages + files_to_change = [ + new_dir_path / package_name_no_version / "_dist_info.py", + ] + # only torch and NOT triton, torchaudio, torchvision + elif "torch" == package_name_no_version: + files_to_change = [ + new_dir_path / package_name_no_version / "_rocm_init.py", + new_dir_path / package_name_no_version / "version.py", + ] + + # special handling + # we only want to change required-dist matching "rocm" + metadata_path = ( + new_dir_path + / f"{package_name_no_version}-{old_version}.dist-info" + / "METADATA" + ) + print(f" {metadata_path}") + with fileinput.input( + files=(metadata_path), + encoding="utf-8", + inplace=True, + ) as f: + for line in f: + if "Requires-Dist" in line: + if "rocm" in line: + print(line.replace(old_rocm_version, new_rocm_version), end="") + continue + print(line, end="") + # torchaudio, torchvision + elif not "triton" in package_name_no_version: + files_to_change = [ + new_dir_path / package_name_no_version / "version.py", + ] + # triton + else: + # no additional (rocm-specific) files needed to be changed that contain the version + return + + for f in files_to_change: + print(f" {f}") + with fileinput.input(files=(files_to_change), encoding="utf-8", inplace=True) as f: + for line in f: + print(line.replace(old_rocm_version, new_rocm_version), end="") + + print(" ...done") + + +def promote_wheel(filename: pathlib.Path, prerelease_type: str) -> bool: + print(f"Promoting whl from rc to final: {filename}") + + original_wheel = Wheel(filename) + original_version = Version(original_wheel.version) + new_base_version = str(original_version.base_version) + + print(f" Detected version: {original_version}") + + if original_version.local: # torch packages + if not prerelease_type in original_version.local: + print( + f" [ERROR] Only prerelease versions of type '{prerelease_type}' can be promoted! Skipping!" + ) + return False + new_local_version = str(original_version.local).split(prerelease_type, 1)[0] + new_base_version = str(original_version.public) + else: # rocm packages + if not prerelease_type in str(original_version): + print( + f" [ERROR] Only prerelease versions of type '{prerelease_type}' can be promoted! Skipping!" + ) + return False + new_local_version = None + + print(f" New base version: {new_base_version}") + print(f" New local version: {new_local_version}") + + print(" Starting to execute version change") + new_wheel_path = change_wheel_version.change_wheel_version( + filename, + new_base_version, + new_local_version, + callback_func=wheel_change_extra_files, + ) + print(" Version change done") + + new_wheel = Wheel(new_wheel_path) + new_version = Version(new_wheel.version) + print(f"New wheel has {new_version} and path is {new_wheel_path}") + return True + + +def promote_targz_sdist(filename: pathlib.Path, prerelease_type: str) -> bool: + print(f"Found tar.gz: {filename}") + + base_dir = filename.parent + package_name = filename.name.removesuffix(".tar.gz") # removes .tar.gz + + with tempfile.TemporaryDirectory(prefix=package_name + "-") as tmp_dir: + print(f" Extracting tar file to {tmp_dir}", end="") + + tmp_path = pathlib.Path(tmp_dir) + + targz = tarfile.open(filename) + targz.extractall(tmp_path) + targz.close() + print(" ...done") + + with open(tmp_path / f"{package_name}" / "PKG-INFO", "r") as info: + for line in info.readlines(): + if line.startswith("Version"): + version = Version(line.removeprefix("Version:").strip()) + + assert version, f"No version found in {filename}/PKG-INFO." + + print(f" Detected version: {version}") + + if not prerelease_type in str(version): + print( + f" [ERROR] Only prerelease versions of type '{prerelease_type}' can be promoted! Skipping!" + ) + return False + + base_version = version.base_version + + print( + f" Editing files to change version from {version} to {base_version}", + end="", + ) + + files_to_change = [ + tmp_path / f"{package_name}" / "src" / "rocm.egg-info" / "requires.txt", + tmp_path / f"{package_name}" / "src" / "rocm.egg-info" / "PKG-INFO", + tmp_path / f"{package_name}" / "src" / "rocm_sdk" / "_dist_info.py", + tmp_path / f"{package_name}" / "PKG-INFO", + ] + + with fileinput.input( + files=(files_to_change), encoding="utf-8", inplace=True + ) as f: + for line in f: + print(line.replace(str(version), str(base_version)), end="") + + print(" ...done") + + print(" Creating new archive for it", end="") + # Rename temporary directory to package name with promoted version + package_name_no_version = package_name.removesuffix(str(version)) + new_archive_name = package_name_no_version + str(base_version) + os.rename(tmp_path / f"{package_name}", tmp_path / f"{new_archive_name}") + + print(f" {new_archive_name}", end="") + + with tarfile.open(f"{base_dir}/{new_archive_name}.tar.gz", "w|gz") as tar: + tar.add(tmp_path / f"{new_archive_name}", arcname=new_archive_name) + + print(f" ...done") + print( + f"Repacked {package_name} as release {base_dir}/{new_archive_name}.tar.gz" + ) + return True + + +def promote_targz_tarball( + filename: pathlib.Path, delete: bool, prerelease_type: str +) -> bool: + old_name = filename.name.removesuffix(".tar.gz") + old_version = Version(old_name.split("-")[-1]) + + print(f"Promoting tarball from rc to final: {filename.name}") + print(f" Detected version: {old_version}") + + if not prerelease_type in str(old_version): + print( + f" [ERROR] Only prerelease versions of type '{prerelease_type}' can be promoted! Skipping!" + ) + return False + + new_version = Version(old_version.base_version) + new_name = old_name.replace(str(old_version), str(new_version)) + ".tar.gz" + + print(f" New version: {new_version}") + + if delete: + os.rename(filename, filename.parent / new_name) + print(f" Rename {filename.name} to {new_name}", end="") + else: + print(f" Copy {filename.name} to {new_name}", end="") + shutil.copy2(filename, filename.parent / new_name) + print(" ...done") + + print(f"Repacked {filename.name} as release {filename.parent}/{new_name}") + return True + + +def main( + input_dir: pathlib.Path, + match_files: str = "*", + delete: bool = False, + prerelease_type: str = "rc", +) -> None: + print(f"Looking for .whl and .tar.gz in {input_dir}/{match_files}") + + files = input_dir.glob(match_files) + + for file in files: + print("") + if file.is_dir(): + print(f"Skipping directory: {file}") + continue + if file.suffix == ".whl": + if promote_wheel(file, prerelease_type) and delete: + print(f"Removing old wheel: {file}") + os.remove(file) + elif file.suffixes[-1] == ".gz" and file.suffixes[-2] == ".tar": + if file.name.startswith("therock-dist"): + promote_targz_tarball(file, delete, prerelease_type) + else: + if promote_targz_sdist(file, prerelease_type) and delete: + print(f"Removing old sdist .tar.gz: {file}") + os.remove(file) + else: + print(f"File found that cannot be promoted: {file}") + + +if __name__ == "__main__": + print("Parsing arguments", end="") + p = parse_arguments(sys.argv[1:]) + print(" ...done") + + main(p.input_dir, p.match_files, p.delete_old_on_success, p.prerelease_type[0]) diff --git a/build_tools/packaging/requirements.txt b/build_tools/packaging/requirements.txt new file mode 100644 index 0000000000..74a5da8c05 --- /dev/null +++ b/build_tools/packaging/requirements.txt @@ -0,0 +1,4 @@ +installer==0.7.0 +packaging==25.0 +pkginfo==1.12.1.2 +wheel==0.45.1 diff --git a/build_tools/packaging/tests/promote_from_rc_to_final_test.py b/build_tools/packaging/tests/promote_from_rc_to_final_test.py new file mode 100644 index 0000000000..e9bd0f060c --- /dev/null +++ b/build_tools/packaging/tests/promote_from_rc_to_final_test.py @@ -0,0 +1,468 @@ +#!/usr/bin/env python +"""Test suite for the promote_from_rc_to_final package promotion script. + +This test suite validates the functionality of promote_from_rc_to_final.py, which +promotes release candidate (RC) packages to final releases by removing RC suffixes +from version strings (e.g., 7.9.0rc1 → 7.9.0). + +The test suite downloads RC packages from https://rocm.prereleases.amd.com/ and runs +comprehensive validation tests to ensure that: + 1. Complete promotion of all ROCm SDK and PyTorch packages works correctly + 2. Partial promotions (only ROCm or only PyTorch) fail as expected + 3. Promoted packages have correct filenames without RC suffixes + 4. All promoted wheels have consistent version strings + 5. Promoted packages can be successfully installed in a virtual environment + +NOTES: + - Tests run in isolated temporary directories (including venv) to avoid polluting the workspace + - Original RC packages are downloaded fresh or loaded from cache for each test run + (set with --cache-dir, directory is created if it doesn't exist) + - Tests use real package files to ensure end-to-end validation + - Platform is auto-detected based on the system but can be overridden with the --platform argument. + +TEST SCENARIOS: + - checkPromoteEverything: Tests promotion of all packages. This test should SUCCEED. + + - checkPromoteOnlyRocm: Tests promotion of only ROCm SDK packages while leaving + PyTorch packages as RC. This test should FAIL. + + - checkPromoteOnlyTorch: Tests promotion of only PyTorch-related packages while + leaving ROCm SDK packages as RC. This test should FAIL. + +PACKAGE TYPES TESTED: + - ROCm SDK packages: rocm, rocm_sdk_core, rocm_sdk_devel, rocm_sdk_libraries_* + - PyTorch packages: torch, torchaudio, torchvision, pytorch_triton_rocm + - Distribution tarballs: therock-dist-{platform}-gfx{arch}-{version}.tar.gz + +PREREQUISITES: + - pip install -r ./build_tools/packaging/requirements.txt + +USAGE: + # Test on current platform (auto-detected): + python ./build_tools/packaging/tests/promote_from_rc_to_final_test.py + + # Use cached packages to speed up repeated test runs: + python ./build_tools/packaging/tests/promote_from_rc_to_final_test.py --cache-dir=/tmp/package_cache +""" + +import argparse +import shutil +import sys +import os +from pathlib import Path +import tempfile +from packaging.version import Version +from pkginfo import Wheel +import subprocess +import urllib +import platform as platform_module + +sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) +import promote_from_rc_to_final + +sys.path.insert(0, os.fspath(Path(__file__).parent.parent.parent)) +import setup_venv + + +def checkPromotedFileNames(dir_path: Path, platform: str) -> tuple[bool, str]: + if platform == "linux": + expected_promoted_pkgs = [ + "rocm-7.9.0.tar.gz", + "rocm_sdk_core-7.9.0-py3-none-linux_x86_64.whl", + "rocm_sdk_devel-7.9.0-py3-none-linux_x86_64.whl", + "rocm_sdk_libraries_gfx94x_dcgpu-7.9.0-py3-none-linux_x86_64.whl", + "pytorch_triton_rocm-3.3.1+rocm7.9.0-cp312-cp312-linux_x86_64.whl", + "torch-2.7.1+rocm7.9.0-cp312-cp312-linux_x86_64.whl", + "torchaudio-2.7.1a0+rocm7.9.0-cp312-cp312-linux_x86_64.whl", + "torchvision-0.22.1+rocm7.9.0-cp312-cp312-linux_x86_64.whl", + "therock-dist-linux-gfx1151-7.9.0.tar.gz", + ] + else: + expected_promoted_pkgs = [ + "rocm-7.9.0.tar.gz", + "rocm_sdk_core-7.9.0-py3-none-win_amd64.whl", + "rocm_sdk_devel-7.9.0-py3-none-win_amd64.whl", + "rocm_sdk_libraries_gfx1151-7.9.0-py3-none-win_amd64.whl", + "torch-2.9.0+rocm7.9.0-cp312-cp312-win_amd64.whl", + "torchaudio-2.9.0+rocm7.9.0-cp312-cp312-win_amd64.whl", + "torchvision-0.24.0+rocm7.9.0-cp312-cp312-win_amd64.whl", + "therock-dist-windows-gfx1151-7.9.0.tar.gz", + ] + + # get files and strip path from them + files = dir_path.glob("*") + files = [file.name for file in files] + + if len(files) != len(expected_promoted_pkgs): + return ( + False, + f"Files found and expected promoted packages are not the same amount ({len(files)} vs {len(expected_promoted_pkgs)})", + ) + + for file in files: + if not file in expected_promoted_pkgs: + return False, f"{file} not matching any of the expected package names" + + return True, "" + + +def checkAllWheelsSameVersion( + dir_path: Path, expected_version: Version +) -> tuple[bool, str]: + for file in dir_path.glob("*.whl"): + wheel = Wheel(file) + version = Version(wheel.version) + + if ( + str(version) == str(expected_version) and version.local == None + ): # rocm packages + continue + elif str(version.local) == "rocm" + str(expected_version): # torch packages + continue + else: + return ( + False, + f"{file} has version {version}, but expected version is {expected_version}", + ) + + return True, "" + + +def checkInstallation(dir_path: Path) -> tuple[bool, str]: + """ + Note: dir_path must be a TemporaryDirectory, otherwise you must clean up the .venv created here yourself. + """ + try: + setup_venv.create_venv(dir_path / ".venv") + python_exe = setup_venv.find_venv_python(dir_path / ".venv") + if python_exe is None: + return ( + False, + "Problem when installing temporary venv: Python executable not found", + ) + + # only install rocm wheels/sdist and pytorch wheels, not therock-dist tarball or .venv + packages = [ + p + for p in dir_path.glob("*") + if p.name != ".venv" and "therock-dist" not in p.name + ] + + proc = subprocess.run( + [python_exe, "-m", "pip", "install"] + packages, + capture_output=True, + encoding="utf-8", + check=True, + ) + except subprocess.CalledProcessError as e: + return False, e.stderr + return True, "" + + +def checkPromoteEverything( + dir_path: Path, expected_version: Version, platform: str +) -> tuple[bool, str]: + print("") + print( + "=================================================================================" + ) + print("TEST: Testing promotion of all packages") + print( + "=================================================================================" + ) + success = False + with tempfile.TemporaryDirectory( + prefix="PromoteRcToFinalTest-PromoteEverything-" + ) as tmp: + tmp_dir = Path(tmp) + # make a copy in a separate dir to not pollute it + for file in dir_path.glob("*"): + shutil.copy2(file, tmp_dir) + + promote_from_rc_to_final.main(tmp_dir, delete=True) + success = True + + for func_name, res in [ + ("checkPromotedFileNames", checkPromotedFileNames(tmp_dir, platform)), + ( + "checkAllWheelsSameVersion", + checkAllWheelsSameVersion(tmp_dir, expected_version), + ), + ("checkInstallation", checkInstallation(tmp_dir)), + ]: + if not res[0]: + print("") + print( + f"[ERROR] Failure to promote the packages (failure captured by {func_name}):" + ) + print(res[1]) + success = False + break + print("") + print( + "=================================================================================" + ) + print( + "TEST DONE: Testing promotion of all packages. Result:" + + (" SUCCESS" if success else " FAILURE") + ) + print( + "=================================================================================" + ) + return success + + +def checkPromoteOnlyRocm( + dir_path: Path, expected_version: Version, platform: str +) -> bool: # should fail + print("") + print( + "=================================================================================" + ) + print("TEST: Testing promotion of only rocm packages") + print( + "=================================================================================" + ) + success = False + with tempfile.TemporaryDirectory( + prefix="PromoteRcToFinalTest-PromoteOnlyRocm-" + ) as tmp: + tmp_dir = Path(tmp) + # make a copy in a separate dir to not pollute it + for file in dir_path.glob("*"): + shutil.copy2(file, tmp_dir) + + promote_from_rc_to_final.main(tmp_dir, match_files="rocm*", delete=True) + + success = True + + for func_name, res in [ + ("checkPromotedFileNames", checkPromotedFileNames(tmp_dir, platform)), + ( + "checkAllWheelsSameVersion", + checkAllWheelsSameVersion(tmp_dir, expected_version), + ), + ("checkInstallation", checkInstallation(tmp_dir)), + ]: + if res[0]: + success = False + print("") + print( + f"[ERROR] checkPromoteOnlyRocm: Promotion of packages successful, eventhough it shouldnt be" + ) + print("Function that succeeded (and should NOT have): " + func_name) + proc = subprocess.run( + ["ls", tmp_dir], capture_output=True, encoding="utf-8" + ) + print(proc.stdout) + break + print("") + print( + "=================================================================================" + ) + print( + "TEST DONE: Testing promotion of only rocm packages. Result:" + + (" SUCCESS" if success else " FAILURE") + ) + print( + "=================================================================================" + ) + return success + + +def checkPromoteOnlyTorch( + dir_path: Path, expected_version: Version, platform: str +) -> bool: # should fail + print("") + print( + "=================================================================================" + ) + print("TEST: Testing promotion of only PyTorch packages") + print( + "=================================================================================" + ) + success = False + with tempfile.TemporaryDirectory( + prefix="PromoteRcToFinalTest-PromoteOnlyTorch-" + ) as tmp: + tmp_dir = Path(tmp) + # make a copy in a separate dir to not pollute it + for file in dir_path.glob("*"): + shutil.copy2(file, tmp_dir) + + promote_from_rc_to_final.main(tmp_dir, match_files="*torch*", delete=True) + + success = True + + for func_name, res in [ + ("checkPromotedFileNames", checkPromotedFileNames(tmp_dir, platform)), + ( + "checkAllWheelsSameVersion", + checkAllWheelsSameVersion(tmp_dir, expected_version), + ), + ("checkInstallation", checkInstallation(tmp_dir)), + ]: + if res[0]: + success = False + print("") + print( + f"[ERROR] checkPromoteOnlyTorch: Promotion of packages successful, eventhough it shouldnt be" + ) + print("Function that succeeded (and should NOT have): " + func_name) + proc = subprocess.run( + ["ls", tmp_dir], capture_output=True, encoding="utf-8" + ) + print(proc.stdout) + break + print("") + print( + "=================================================================================" + ) + print( + "TEST DONE: Testing promotion of only PyTorch packages. Result:" + + (" SUCCESS" if success else " FAILURE") + ) + print( + "=================================================================================" + ) + return success + + +def fetchPackage(URL: str, package_name: str, tmp_dir: Path, cache_dir: Path) -> None: + # Check first if the package is cached + if cache_dir is not None: + if (cache_dir / package_name).exists(): + print(f" Found in cache: {package_name}") + shutil.copy2(cache_dir / package_name, tmp_dir / package_name) + return + # Otherwise download the package + print(f" Downloading {package_name}") + # Use safe encoding, otherwise CURL gets unhappy with the "+" in the URL + url_safe_encoding = URL + urllib.parse.quote(package_name) + print(url_safe_encoding) + subprocess.run( + ["curl", "--output", tmp_dir / package_name, url_safe_encoding], + check=True, + ) + # Let's cache the package + if cache_dir is not None: + print(f" Caching {package_name}") + shutil.copy2(tmp_dir / package_name, cache_dir / package_name) + + +def getLinuxPackagesLinks() -> tuple[list[tuple[str, str]], Version, Version]: + # download some version + URL = "https://rocm.prereleases.amd.com/whl/gfx94X-dcgpu/" + version = Version("7.9.0rc1") + expected_version = Version("7.9.0") + packages = [ + "rocm-7.9.0rc1.tar.gz", + "rocm_sdk_core-7.9.0rc1-py3-none-linux_x86_64.whl", + "rocm_sdk_devel-7.9.0rc1-py3-none-linux_x86_64.whl", + "rocm_sdk_libraries_gfx94x_dcgpu-7.9.0rc1-py3-none-linux_x86_64.whl", + "pytorch_triton_rocm-3.3.1+rocm7.9.0rc1-cp312-cp312-linux_x86_64.whl", + "torch-2.7.1+rocm7.9.0rc1-cp312-cp312-linux_x86_64.whl", + "torchaudio-2.7.1a0+rocm7.9.0rc1-cp312-cp312-linux_x86_64.whl", + "torchvision-0.22.1+rocm7.9.0rc1-cp312-cp312-linux_x86_64.whl", + ] + + url_and_packages = [(URL, package) for package in packages] + url_and_packages.append( + ( + "https://rocm.prereleases.amd.com/tarball/", + "therock-dist-linux-gfx1151-7.9.0rc1.tar.gz", + ) + ) + + return url_and_packages, version, expected_version + + +def getWindowsPackagesLinks() -> tuple[list[tuple[str, str]], Version, Version]: + # download some version + URL = "https://rocm.prereleases.amd.com/whl/gfx1151/" + version = Version("7.9.0rc1") + expected_version = Version("7.9.0") + packages = [ + "rocm-7.9.0rc1.tar.gz", + "rocm_sdk_core-7.9.0rc1-py3-none-win_amd64.whl", + "rocm_sdk_devel-7.9.0rc1-py3-none-win_amd64.whl", + "rocm_sdk_libraries_gfx1151-7.9.0rc1-py3-none-win_amd64.whl", + "torch-2.9.0+rocm7.9.0rc1-cp312-cp312-win_amd64.whl", + "torchaudio-2.9.0+rocm7.9.0rc1-cp312-cp312-win_amd64.whl", + "torchvision-0.24.0+rocm7.9.0rc1-cp312-cp312-win_amd64.whl", + ] + + url_and_packages = [(URL, package) for package in packages] + url_and_packages.append( + ( + "https://rocm.prereleases.amd.com/tarball/", + "therock-dist-windows-gfx1151-7.9.0rc1.tar.gz", + ) + ) + + return url_and_packages, version, expected_version + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="""Tests promotion of packages from release candidate to final release (e.g. 7.10.0rc1 --> 7.10.0). +""" + ) + parser.add_argument( + "--platform", + help="OS platform: either 'linux' (default) or 'win'", + default="linux" if platform_module.system() != "Windows" else "windows", + ) + parser.add_argument( + "--cache-dir", + help="Path to the directory that contains the cache of the packages", + type=Path, + default=None, + ) + p = parser.parse_args(sys.argv[1:]) + platform = p.platform + cache_dir = p.cache_dir + + if cache_dir is not None: + if not cache_dir.exists(): + print(f" Creating cache directory: {cache_dir}") + cache_dir.mkdir(parents=True, exist_ok=True) + + # make tmpdir + with tempfile.TemporaryDirectory(prefix=f"PromoteRcToFinalTest-{platform}-") as tmp: + tmp_dir = Path(tmp) + if platform == "linux": + url_and_packages, version, expected_version = getLinuxPackagesLinks() + elif platform == "windows": # win + url_and_packages, version, expected_version = getWindowsPackagesLinks() + else: + raise ValueError(f"Unknown platform: {platform}") + + print( + f"Testing promotion of {version} to {expected_version} on platform {platform}" + ) + print(f"Fetching packages", end="") + print(f" Using cache directory: {cache_dir}") + for URL, package in url_and_packages: + fetchPackage(URL, package, tmp_dir, cache_dir) + print(" ...done") + + res_everything = checkPromoteEverything(tmp_dir, expected_version, platform) + res_rocm = checkPromoteOnlyRocm(tmp_dir, expected_version, platform) + res_torch = checkPromoteOnlyTorch(tmp_dir, expected_version, platform) + + print("") + print("") + print( + "=================================================================================" + ) + print("SUMMARY") + print( + "=================================================================================" + ) + print("checkPromoteEverything: " + ("SUCCESS" if res_everything else "FAILURE")) + print("checkPromoteOnlyRocm: " + ("SUCCESS" if res_rocm else "FAILURE")) + print("checkPromoteOnlyTorch: " + ("SUCCESS" if res_torch else "FAILURE")) + print( + "=================================================================================" + ) diff --git a/build_tools/setup_venv.py b/build_tools/setup_venv.py index 11c2914497..d25a9bf254 100755 --- a/build_tools/setup_venv.py +++ b/build_tools/setup_venv.py @@ -32,7 +32,6 @@ """ import argparse -import os from pathlib import Path import platform import shlex @@ -69,6 +68,10 @@ def exec(args: list[str | Path], cwd: Path = Path.cwd()): subprocess.check_call(args, cwd=str(cwd), stdin=subprocess.DEVNULL) +def get_system_py_command(use_uv: bool) -> list[str]: + return ["uv"] if use_uv else [sys.executable, "-m"] + + def find_venv_python(venv_path: Path) -> Path | None: paths = [venv_path / "bin" / "python", venv_path / "Scripts" / "python.exe"] for p in paths: @@ -77,14 +80,15 @@ def find_venv_python(venv_path: Path) -> Path | None: return None -def create_venv(venv_dir: Path, py_cmd: list[str]): - cwd = Path.cwd() +def create_venv(venv_dir: Path, py_cmd: list[str] | None = None): + if not py_cmd: + py_cmd = get_system_py_command(use_uv=False) log(f"Creating venv at '{venv_dir}'") # Log some other variations of the path too. try: - venv_dir_relative = venv_dir.relative_to(cwd) + venv_dir_relative = venv_dir.relative_to(Path.cwd()) except ValueError: venv_dir_relative = venv_dir venv_dir_resolved = venv_dir.resolve() @@ -93,12 +97,12 @@ def create_venv(venv_dir: Path, py_cmd: list[str]): log("") # Create with 'python -m venv' as needed. - python_exe = find_venv_python(venv_dir) + python_exe = find_venv_python(venv_dir_resolved) if python_exe: log(f" Found existing python executable at '{python_exe}', skipping creation") log(" Run again with --clean to clear the existing directory instead") else: - exec(py_cmd + ["venv", str(venv_dir)]) + exec(py_cmd + ["venv", str(venv_dir_resolved)]) def upgrade_pip(python_exe: Path): @@ -106,7 +110,10 @@ def upgrade_pip(python_exe: Path): exec([str(python_exe), "-m", "pip", "install", "--upgrade", "pip"]) -def install_packages(args: argparse.Namespace, py_cmd: list[str]): +def install_packages(args: argparse.Namespace, py_cmd: list[str] | None): + if not py_cmd: + py_cmd = get_system_py_command(use_uv=False) + log("") if args.index_name: @@ -192,8 +199,7 @@ def log_activate_instructions(venv_dir: Path): def run(args: argparse.Namespace): venv_dir = args.venv_dir - # selects uv if available - py_cmd = ["uv"] if args.use_uv else [sys.executable, "-m"] + py_cmd = get_system_py_command(use_uv=args.use_uv) if args.clean and venv_dir.exists(): log(f"Clearing existing venv_dir '{venv_dir}'") diff --git a/build_tools/third_party/change_wheel_version/LICENSE b/build_tools/third_party/change_wheel_version/LICENSE new file mode 100644 index 0000000000..f98af2fc43 --- /dev/null +++ b/build_tools/third_party/change_wheel_version/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 hauntsaninja + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/build_tools/third_party/change_wheel_version/change_wheel_version.py b/build_tools/third_party/change_wheel_version/change_wheel_version.py new file mode 100644 index 0000000000..0ff30ba7dd --- /dev/null +++ b/build_tools/third_party/change_wheel_version/change_wheel_version.py @@ -0,0 +1,254 @@ +################################################################################# +# original version from https://github.com/hauntsaninja/change_wheel_version +# (commit e28436ebdb6fe5a81cd29922f7276d00675d1bd3, Sep 13, 2024) +# +# modified to allow a hook to update the version in other files part of the wheel +# modified parts are marked with " # AMD-added " +################################################################################# + +import argparse +import email.parser +import email.policy +import os +import shutil +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path +from typing import Any, Optional, no_type_check + +import installer.utils +import packaging.version +from packaging.tags import parse_tag + +# AMD-added +from collections.abc import Callable + + +def version_replace( + v: packaging.version.Version, **kwargs: Any +) -> packaging.version.Version: + # yikes :-) + self = packaging.version.Version.__new__(packaging.version.Version) + self._version = v._version._replace(**kwargs) + return packaging.version.Version(str(self)) + + +class ExecutablePreservingZipfile(zipfile.ZipFile): + @no_type_check + def _extract_member(self, member, targetpath, pwd): + if not isinstance(member, zipfile.ZipInfo): + member = self.getinfo(member) + + targetpath = super()._extract_member(member, targetpath, pwd) + + mode = member.external_attr >> 16 + if mode != 0: + os.chmod(targetpath, mode) + return targetpath + + +def wheel_unpack(wheel: Path, dest_dir: Path, name_ver: str) -> None: + # This is the moral equivalent of: + # subprocess.check_output( + # [sys.executable, "-m", "wheel", "unpack", "-d", str(dest_dir), str(wheel)] + # ) + # Except we need to preserve permissions + # https://github.com/pypa/wheel/issues/505 + with ExecutablePreservingZipfile(wheel) as wf: + wf.extractall(dest_dir / name_ver) + + +def change_platform_tag( + wheel_path: Path, tag: str, parser: email.parser.BytesParser +) -> str: + """Changes the WHEEL file to specify `tag`. Returns a canonicalized copy of that tag.""" + platform_tags = list(parse_tag(tag)) + if len(platform_tags) != 1: + raise ValueError( + f"Parsed '{tag}' as {len(platform_tags)}; there must be exactly one." + ) + platform_tag = platform_tags[0] + is_pure = platform_tag.abi == "none" + if is_pure != (platform_tag.platform == "any"): + raise ValueError(f"ABI and platform are inconsistent in '{platform_tag}'.") + if is_pure != platform_tag.interpreter.startswith("py"): + raise ValueError( + f"Interpreter and platform are inconsistent in '{platform_tag}'." + ) + with open(wheel_path, "rb") as f: + msg = parser.parse(f) + msg.replace_header("Tag", str(platform_tag)) + msg.replace_header("Root-Is-Purelib", str(is_pure).lower()) + with open(wheel_path, "wb") as f: + f.write(msg.as_bytes()) + return str(platform_tag) + + +def change_wheel_version( + wheel: Path, + version: Optional[str], + local_version: Optional[str], + allow_same_version: bool = False, + platform_tag: Optional[str] = None, + callback_func: Optional[Callable] = None, # AMD-added +) -> Path: + old_parts = installer.utils.parse_wheel_filename(wheel.name) + old_version = packaging.version.Version(old_parts.version) + distribution = old_parts.distribution + + if version is None: + if local_version is not None: + # just replace the local version + new_version = version_replace( + old_version, local=packaging.version._parse_local_version(local_version) + ) + else: + new_version = old_version + else: + # replace the base version and (possibly) the local version + new_version = packaging.version.Version(version) + assert not new_version.local + if local_version: + new_version = version_replace( + new_version, local=packaging.version._parse_local_version(local_version) + ) + + if version == old_version: + if allow_same_version: + return wheel + raise ValueError(f"Version {version} is the same as the old version") + + with tempfile.TemporaryDirectory() as _tmpdir: + tmpdir = Path(_tmpdir) + dest_dir = tmpdir / "wheel" + + print(" Unpacking wheel...", end="") + wheel_unpack(wheel, dest_dir, f"{distribution}-{old_version}") + print(" done") + + old_slug = f"{distribution}-{old_version}" + new_slug = f"{distribution}-{new_version}" + assert (dest_dir / old_slug).exists() + assert (dest_dir / old_slug / f"{old_slug}.dist-info").exists() + + # copy everything over + print(f" Move directory from {old_slug} to {new_slug}...", end="") + shutil.move(dest_dir / old_slug, dest_dir / new_slug) + print(" done") + + # AMD-added + print(" Changing wheel-specific files that contain the version") + if callback_func != None: + callback_func(dest_dir / new_slug, old_version, new_version) + + # rename dist-info + shutil.move( + dest_dir / new_slug / f"{old_slug}.dist-info", + dest_dir / new_slug / f"{new_slug}.dist-info", + ) + print(f" Rename directory from {old_slug}.dist-info to {new_slug}.dist-info") + # rename data + if (dest_dir / new_slug / f"{old_slug}.data").exists(): + shutil.move( + dest_dir / new_slug / f"{old_slug}.data", + dest_dir / new_slug / f"{new_slug}.data", + ) + print(f" Rename directory from {old_slug}.data to {new_slug}.data") + + metadata_path = dest_dir / new_slug / f"{new_slug}.dist-info" / "METADATA" + wheel_path = dest_dir / new_slug / f"{new_slug}.dist-info" / "WHEEL" + + # This is actually a non-conformant email policy as per + # https://packaging.python.org/en/latest/specifications/core-metadata/ + # However, it works around this bug in setuptools in cases where the version is really long + # https://github.com/pypa/setuptools/issues/3808 + max_line_length = 200 + policy = email.policy.compat32.clone(max_line_length=200) + parser = email.parser.BytesParser(policy=policy) + version_str = str(new_version) + if len(version_str) >= max_line_length: + raise ValueError(f"Version {version_str} is too long") + + with open(metadata_path, "rb") as f: + msg = parser.parse(f) + msg.replace_header("Version", version_str) + with open(metadata_path, "wb") as f: + f.write(msg.as_bytes()) + + print(f" {metadata_path}") + + print(" ... done") + + + if platform_tag: + print(" Changing platform tag...", end="") + # We don't need to sort, because we assume len(parse_tag(platform_tag)) == 1 + new_tag = change_platform_tag(wheel_path, platform_tag, parser) + print(" done") + else: + print(" Generating platform tag...", end="") + # Generate the tags that will be associated with the wheel after it is repacked. + # `wheel pack` sorts the tags, so we need to do the same if we're not changing it. + new_tag = "-".join( + ".".join(sorted(t.split("."))) for t in old_parts.tag.split("-") + ) + print(" done") + + # print is sometimes buffered, so we need to flush it, as repacking can take a while + print(" Repacking wheel and rewriting RECORD file...", end="", flush=True) + # wheel pack rewrites the RECORD file + subprocess.check_output( + [ + sys.executable, + "-m", + "wheel", + "pack", + "-d", + str(wheel.parent), + str(dest_dir / new_slug), + ] + ) + print(" done") + + new_parts = old_parts._replace(version=str(new_version), tag=new_tag) + new_wheel_name = "-".join(p for p in new_parts if p) + ".whl" + new_wheel = wheel.with_name(new_wheel_name) + + if not new_wheel.exists(): + raise RuntimeError( + f"Failed to create new wheel {new_wheel}\n" + f"Directory contents: {list(wheel.parent.iterdir())}" + ) + return new_wheel + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("wheel", type=Path) + parser.add_argument("--local-version") + parser.add_argument("--version") + parser.add_argument("--delete-old-wheel", action="store_true") + parser.add_argument("--allow-same-version", action="store_true") + parser.add_argument( + "--platform-tag", + help="Force the platform tag to this value. For example, 'py3-none-any'. See " + "https://packaging.python.org/en/latest/specifications/platform-compatibility-tags.", + ) + args = parser.parse_args() + + new_wheel = change_wheel_version( + wheel=args.wheel, + version=args.version, + local_version=args.local_version, + allow_same_version=args.allow_same_version, + platform_tag=args.platform_tag, + ) + print(new_wheel) + if args.delete_old_wheel: + args.wheel.unlink() + + +if __name__ == "__main__": + main()