Skip to content

Adds compress-level option to sdist and wheel configs #1995

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 28 additions & 3 deletions backend/src/hatchling/builders/sdist.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@


class SdistArchive:
def __init__(self, name: str, *, reproducible: bool) -> None:
def __init__(self, name: str, *, reproducible: bool, compress_level: int) -> None:
"""
https://peps.python.org/pep-0517/#source-distributions
"""
Expand All @@ -38,7 +38,8 @@ def __init__(self, name: str, *, reproducible: bool) -> None:

raw_fd, self.path = tempfile.mkstemp(suffix='.tar.gz')
self.fd = os.fdopen(raw_fd, 'w+b')
self.gz = gzip.GzipFile(fileobj=self.fd, mode='wb', mtime=self.timestamp)

self.gz = gzip.GzipFile(fileobj=self.fd, mode='wb', mtime=self.timestamp, compresslevel=compress_level)
self.tf = tarfile.TarFile(fileobj=self.gz, mode='w', format=tarfile.PAX_FORMAT)
self.gettarinfo = lambda *args, **kwargs: self.normalize_tar_metadata(self.tf.gettarinfo(*args, **kwargs))

Expand Down Expand Up @@ -90,10 +91,32 @@ class SdistBuilderConfig(BuilderConfig):
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)

self.__compress_level: int | None = None
self.__core_metadata_constructor: Callable[..., str] | None = None
self.__strict_naming: bool | None = None
self.__support_legacy: bool | None = None

@property
def compress_level(self) -> int:
if self.__compress_level is None:
try:
compress_level = int(self.target_config.get('compress-level', 9))
except ValueError as e:
message = f'Field `tool.hatch.build.{self.plugin_name}.compress-level` must be an integer'
raise TypeError(message) from e

if not (0 <= compress_level <= 9): # noqa: PLR2004
message = (
'Value field '
f'`tool.hatch.build.targets.{self.plugin_name}.compress-level` '
'must be an integer from 0 to 9'
)
raise ValueError(message)

self.__compress_level = compress_level

return self.__compress_level

@property
def core_metadata_constructor(self) -> Callable[..., str]:
if self.__core_metadata_constructor is None:
Expand Down Expand Up @@ -166,7 +189,9 @@ def clean( # noqa: PLR6301
def build_standard(self, directory: str, **build_data: Any) -> str:
found_packages = set()

with SdistArchive(self.artifact_project_id, reproducible=self.config.reproducible) as archive:
with SdistArchive(
self.artifact_project_id, reproducible=self.config.reproducible, compress_level=self.config.compress_level
) as archive:
for included_file in self.recurse_included_files():
if self.config.support_legacy:
possible_package, file_name = os.path.split(included_file.relative_path)
Expand Down
32 changes: 27 additions & 5 deletions backend/src/hatchling/builders/wheel.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ def __exit__(


class WheelArchive:
def __init__(self, project_id: str, *, reproducible: bool) -> None:
def __init__(self, project_id: str, *, reproducible: bool, compress_level: int) -> None:
"""
https://peps.python.org/pep-0427/#abstract
"""
Expand All @@ -81,7 +81,7 @@ def __init__(self, project_id: str, *, reproducible: bool) -> None:

raw_fd, self.path = tempfile.mkstemp(suffix='.whl')
self.fd = os.fdopen(raw_fd, 'w+b')
self.zf = zipfile.ZipFile(self.fd, 'w', compression=zipfile.ZIP_DEFLATED)
self.zf = zipfile.ZipFile(self.fd, 'w', compression=zipfile.ZIP_DEFLATED, compresslevel=compress_level)

@staticmethod
def get_reproducible_time_tuple() -> TIME_TUPLE:
Expand Down Expand Up @@ -187,13 +187,35 @@ class WheelBuilderConfig(BuilderConfig):
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)

self.__compress_level: int | None = None
self.__core_metadata_constructor: Callable[..., str] | None = None
self.__shared_data: dict[str, str] | None = None
self.__shared_scripts: dict[str, str] | None = None
self.__extra_metadata: dict[str, str] | None = None
self.__strict_naming: bool | None = None
self.__macos_max_compat: bool | None = None

@property
def compress_level(self) -> int:
if self.__compress_level is None:
try:
compress_level = int(self.target_config.get('compress-level', 9))
except ValueError as e:
message = f'Field `tool.hatch.build.{self.plugin_name}.compress-level` must be an integer'
raise TypeError(message) from e

if not (0 <= compress_level <= 9): # noqa: PLR2004
message = (
'Value field '
f'`tool.hatch.build.targets.{self.plugin_name}.compress-level` '
'must be an integer from 0 to 9'
)
raise ValueError(message)

self.__compress_level = compress_level

return self.__compress_level

@cached_property
def default_file_selection_options(self) -> FileSelectionOptions:
include = self.target_config.get('include', self.build_config.get('include', []))
Expand Down Expand Up @@ -472,7 +494,7 @@ def build_standard(self, directory: str, **build_data: Any) -> str:
build_data['tag'] = self.get_default_tag()

with WheelArchive(
self.artifact_project_id, reproducible=self.config.reproducible
self.artifact_project_id, reproducible=self.config.reproducible, compress_level=self.config.compress_level
) as archive, RecordFile() as records:
for included_file in self.recurse_included_files():
record = archive.add_file(included_file)
Expand Down Expand Up @@ -501,7 +523,7 @@ def build_editable_detection(self, directory: str, **build_data: Any) -> str:
build_data['tag'] = self.get_default_tag()

with WheelArchive(
self.artifact_project_id, reproducible=self.config.reproducible
self.artifact_project_id, reproducible=self.config.reproducible, compress_level=self.config.compress_level
) as archive, RecordFile() as records:
exposed_packages = {}
for included_file in self.recurse_selected_project_files():
Expand Down Expand Up @@ -582,7 +604,7 @@ def build_editable_explicit(self, directory: str, **build_data: Any) -> str:
build_data['tag'] = self.get_default_tag()

with WheelArchive(
self.artifact_project_id, reproducible=self.config.reproducible
self.artifact_project_id, reproducible=self.config.reproducible, compress_level=self.config.compress_level
) as archive, RecordFile() as records:
directories = sorted(
os.path.normpath(os.path.join(self.root, relative_directory))
Expand Down
16 changes: 16 additions & 0 deletions docs/config/build.md
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,22 @@ skip-excluded-dirs = true
!!! warning
This may result in not shipping desired files. For example, if you want to include the file `a/b/c.txt` but your [VCS ignores](#vcs) `a/b`, the file `c.txt` will not be seen because its parent directory will not be entered. In such cases you can use the [`force-include`](#forced-inclusion) option.

#### Compression level

You can change the level used for compressing the sdist tarballs and wheels. In some circumstances, lowering it from the default of 9 can massively reduce build times without affecting the output size too much.

Note that for widely distributed packages it probably makes the most sense to use the default, highest compression level to conserve the amount of bytes transferred over the network.

```toml config-example
[tool.hatch.build.targets.sdist]
compress-level = 1

[tool.hatch.build.targets.wheel]
compress-level = 1
```

Accepted values are 0 (no compression) to 9 (highest compression).

## Reproducible builds

By default, [build targets](#build-targets) will build in a reproducible manner provided that they support that behavior. To disable this, set `reproducible` to `false`:
Expand Down