From 1881a56eb40c37331b7a6b333b2b630a1062f85c Mon Sep 17 00:00:00 2001 From: Christian Zagrodnick Date: Mon, 2 Mar 2026 13:46:20 +0100 Subject: [PATCH] =?UTF-8?q?Update=20python=20to=20=E2=89=A53.9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit update tox accordingly and update tool configs to match pre-commit --- CHANGES.d/20260302_134629_cz_update_python.md | 1 + pyproject.toml | 25 ++------- src/batou_ext/cron.py | 8 ++- src/batou_ext/fcio.py | 23 +++++--- src/batou_ext/file.py | 4 +- src/batou_ext/geoip.py | 8 ++- src/batou_ext/git.py | 24 ++++++--- src/batou_ext/jenkins.py | 12 +++-- src/batou_ext/journalbeat.py | 4 +- src/batou_ext/mail.py | 12 +++-- src/batou_ext/mirror.py | 8 ++- src/batou_ext/nix.py | 52 ++++++++++++++----- src/batou_ext/oci.py | 8 ++- src/batou_ext/php.py | 12 +++-- src/batou_ext/postfixadmin/__init__.py | 12 +++-- src/batou_ext/postfixadmin/dovecot.py | 8 ++- src/batou_ext/postfixadmin/postfix.py | 4 +- src/batou_ext/postgres.py | 4 +- src/batou_ext/python.py | 20 +++++-- src/batou_ext/resources/watchdog-wrapper.py | 8 ++- src/batou_ext/roundcube/__init__.py | 8 ++- src/batou_ext/run.py | 4 +- src/batou_ext/s3.py | 8 ++- src/batou_ext/s3_bootstrap.py | 32 +++++++++--- src/batou_ext/ssh.py | 4 +- src/batou_ext/ssl.py | 23 +++++--- src/batou_ext/tests/test_configure.py | 4 +- src/batou_ext/tests/test_jenkins.py | 4 +- src/batou_ext/tests/test_oci.py | 8 ++- src/batou_ext/tests/test_postgres.py | 4 +- src/batou_ext/versions.py | 4 +- tox.ini | 8 +-- 32 files changed, 261 insertions(+), 107 deletions(-) create mode 100644 CHANGES.d/20260302_134629_cz_update_python.md diff --git a/CHANGES.d/20260302_134629_cz_update_python.md b/CHANGES.d/20260302_134629_cz_update_python.md new file mode 100644 index 00000000..400a8b19 --- /dev/null +++ b/CHANGES.d/20260302_134629_cz_update_python.md @@ -0,0 +1 @@ +- Update required Python to ≥3.9 diff --git a/pyproject.toml b/pyproject.toml index c66229ec..7bc6e79f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,9 +18,9 @@ classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", ] -requires-python = ">=3.6" +requires-python = ">=3.9" dependencies = [ - "batou >= 2.3b4", + "batou >= 2.5", "pyaml", "setuptools", "six", @@ -56,16 +56,6 @@ zip-safe = false [tool.setuptools.packages.find] where = ["src"] -[tool.black] -line-length = 80 -target-version = ['py36', 'py37', 'py38'] - -[tool.isort] -profile = "black" -line_length = 80 -multi_line_output = 3 -include_trailing_comma = true - [tool.pytest.ini_options] addopts = "--pyargs batou_ext" @@ -79,16 +69,11 @@ new_fragment_template = "- New changelog entry." insert_marker = "- Nothing changed yet." categories = [] -[flake8] -ignore = [ - "E231", # Missing whitespace after ',', ';', or ':' -- enforced by yapf config - "W503", # Line break occurred before a binary operator -- enforced by yapf config - "W504", # Line break occurred after a binary operator -- enforced by yapf config -] -max-line-length = 80 - [zest.releaser] version-levels = 3 history-file = "CHANGES.md" release = "yes" history_format = "md" + +[tool.ruff] +line-length = 80 diff --git a/src/batou_ext/cron.py b/src/batou_ext/cron.py index a68b6d2c..0981955a 100644 --- a/src/batou_ext/cron.py +++ b/src/batou_ext/cron.py @@ -87,7 +87,9 @@ def configure(self): self += batou.lib.file.File( self.expand("{{component.tag}}.sh"), - content=(files(__spec__.parent) / "resources/cron-wrapper.sh").read_bytes(), + content=( + files(__spec__.parent) / "resources/cron-wrapper.sh" + ).read_bytes(), mode=0o755, ) self.wrapped_command = self._.path @@ -145,7 +147,9 @@ class SystemdTimer(batou.component.Component): additional_service_config = None run_as = batou.component.Attribute( str, - default=batou.component.ConfigString("{{component.environment.service_user}}"), + default=batou.component.ConfigString( + "{{component.environment.service_user}}" + ), ) def configure(self): diff --git a/src/batou_ext/fcio.py b/src/batou_ext/fcio.py index 797e313c..e0b38fce 100644 --- a/src/batou_ext/fcio.py +++ b/src/batou_ext/fcio.py @@ -81,7 +81,9 @@ def _compute_calls(self): def _call(self): api = xmlrpc.client.ServerProxy( - "https://{s.project}:{s.api_key}@api.flyingcircus.io/v1".format(s=self) + "https://{s.project}:{s.api_key}@api.flyingcircus.io/v1".format( + s=self + ) ) api.apply(self.calls) @@ -129,7 +131,8 @@ def _check_aliases(self): error = True else: result = ", ".join( - sockaddr[0] for (family, type, proto, canonname, sockaddr) in addrs + sockaddr[0] + for (family, type, proto, canonname, sockaddr) in addrs ) results.append("{}: {}".format(fqdn, result)) return error, results @@ -148,7 +151,9 @@ def create_xmlrpc_client(environment: batou.environment.Environment): ) raise api_url = environment.overrides["provision"].get("api_url", API_URL) - api = xmlrpc.client.ServerProxy(api_url.format(project=rg_name, api_key=api_key)) + api = xmlrpc.client.ServerProxy( + api_url.format(project=rg_name, api_key=api_key) + ) return api @@ -208,7 +213,9 @@ def config(name): classes = ["role::" + r for r in roles if r] if d.get("environment", config("vm_environment")) is None: - raise ValueError("'environment' for {} must be set.".format(name)) + raise ValueError( + "'environment' for {} must be set.".format(name) + ) call = dict( __type__="virtualmachine", @@ -254,7 +261,9 @@ def alias(interface): print(vm) for key, (old, new) in sorted(changes.items()): if old or new: - print(" {key:20}: {old} → {new}".format(**locals())) + print( + " {key:20}: {old} → {new}".format(**locals()) + ) else: print(" {key}".format(**locals())) else: @@ -383,7 +392,9 @@ def update(self): ) -def change_maintenance_state(xmlrpc, rg_name, desired_state, predict_only=False): +def change_maintenance_state( + xmlrpc, rg_name, desired_state, predict_only=False +): rg = next( (rg for rg in xmlrpc.query("resourcegroup") if rg["name"] == rg_name), None, diff --git a/src/batou_ext/file.py b/src/batou_ext/file.py index f6f6328b..489354d2 100644 --- a/src/batou_ext/file.py +++ b/src/batou_ext/file.py @@ -51,7 +51,9 @@ class SymlinkAndCleanup(batou.component.Component): systemd_extra_args = None def configure(self): - self._current_link = f"{self.prefix}-current" if self.prefix else "current" + self._current_link = ( + f"{self.prefix}-current" if self.prefix else "current" + ) self._last_link = f"{self.prefix}-last" if self.prefix else "last" p = Path(self.current) diff --git a/src/batou_ext/geoip.py b/src/batou_ext/geoip.py index 81cfe920..2b6eb1ee 100644 --- a/src/batou_ext/geoip.py +++ b/src/batou_ext/geoip.py @@ -33,7 +33,9 @@ def configure(self): self.provide("geoip_database", self) self += batou.lib.file.File( "geoip-update.sh", - source=os.path.join(os.path.dirname(__file__), "resources/geoip-update.sh"), + source=os.path.join( + os.path.dirname(__file__), "resources/geoip-update.sh" + ), mode=0o744, ) self.script = self._.path @@ -47,7 +49,9 @@ def configure(self): checkCritical=30000, ) - self.database_file = self.expand("{{component.workdir}}/GeoLite2-City.mmdb") + self.database_file = self.expand( + "{{component.workdir}}/GeoLite2-City.mmdb" + ) def verify(self): self.assert_no_changes() diff --git a/src/batou_ext/git.py b/src/batou_ext/git.py index 234fee61..533370c6 100644 --- a/src/batou_ext/git.py +++ b/src/batou_ext/git.py @@ -65,9 +65,13 @@ def configure(self): if self.scan_host: if not self.git_host: - self.git_host = urllib.parse.urlparse(self.git_clone_url).hostname + self.git_host = urllib.parse.urlparse( + self.git_clone_url + ).hostname if not self.git_port: - self.git_port = urllib.parse.urlparse(self.git_clone_url).port or 22 + self.git_port = ( + urllib.parse.urlparse(self.git_clone_url).port or 22 + ) # Add remote host to known hosts self += batou_ext.ssh.ScanHost(self.git_host, port=self.git_port) @@ -86,10 +90,14 @@ def configure(self): # add custom files if self.sync_parent_folder: self.prepared_path = self.map( - "{}/prepared-{}".format(self.sync_parent_folder, self.git_revision) + "{}/prepared-{}".format( + self.sync_parent_folder, self.git_revision + ) ) else: - self.prepared_path = self.map("prepared-{}".format(self.git_revision)) + self.prepared_path = self.map( + "prepared-{}".format(self.git_revision) + ) self += batou.lib.file.Directory(self.prepared_path, leading=True) self += batou.lib.file.Directory( self.prepared_path, @@ -123,11 +131,15 @@ def update(self): self.cmd("git config user.email '{{component.author_email}}'") self.cmd("git config user.name '{{component.author_name}}'") self.cmd("git add {{component.filename}}") - self.cmd("git commit -m '{{component.message}}' {{component.filename}}") + self.cmd( + "git commit -m '{{component.message}}' {{component.filename}}" + ) def has_changes(self): with self.chdir(self.workingdir): - stdout, stderr = self.cmd("git status --porcelain {{component.filename}}") + stdout, stderr = self.cmd( + "git status --porcelain {{component.filename}}" + ) return bool(stdout.strip()) diff --git a/src/batou_ext/jenkins.py b/src/batou_ext/jenkins.py index eb1fa6f1..ef536226 100644 --- a/src/batou_ext/jenkins.py +++ b/src/batou_ext/jenkins.py @@ -8,7 +8,9 @@ def git_ls_remote(url, ref): - cmd = subprocess.Popen(["git", "ls-remote", url, ref], stdout=subprocess.PIPE) + cmd = subprocess.Popen( + ["git", "ls-remote", url, ref], stdout=subprocess.PIPE + ) stdout, _ = cmd.communicate() if cmd.returncode != 0: raise ValueError( @@ -84,7 +86,9 @@ def update(self, service, version): def update_git(self, service, version, extra_args): resolved = git_resolve(self.config.get(service, "url"), version) if not resolved: - raise ValueError("%s: Could not resolve version %s." % (service, version)) + raise ValueError( + "%s: Could not resolve version %s." % (service, version) + ) log("%s: resolved version %s to: %s", service, version, resolved) self.config.set(service, "revision", resolved) self.config.set(service, "version", version) @@ -141,7 +145,9 @@ def main(): "versions_file", help="Name of versions.ini. If exists it will be overwritten.", ) - p.add_argument("version_mapping_json", help="JSON: mapping of service: version") + p.add_argument( + "version_mapping_json", help="JSON: mapping of service: version" + ) p.set_defaults(func=set_versions) args = parser.parse_args() diff --git a/src/batou_ext/journalbeat.py b/src/batou_ext/journalbeat.py index c0665076..63149ebb 100644 --- a/src/batou_ext/journalbeat.py +++ b/src/batou_ext/journalbeat.py @@ -17,5 +17,7 @@ class JournalBeatTransport(batou.component.Component): def configure(self): self += batou.lib.file.File( self.nix_file_path, - content=(files(__spec__.parent) / "resources/journalbeat.nix").read_bytes(), + content=( + files(__spec__.parent) / "resources/journalbeat.nix" + ).read_bytes(), ) diff --git a/src/batou_ext/mail.py b/src/batou_ext/mail.py index d091cb51..2f0c6bc5 100644 --- a/src/batou_ext/mail.py +++ b/src/batou_ext/mail.py @@ -127,7 +127,9 @@ class Mailhog(batou.component.Component): mailport = batou.component.Attribute(int, 1025) uiport = batou.component.Attribute(int, 8025) apiport = batou.component.Attribute(int, 8025) - purge_old_mailhog_configs = batou.component.Attribute("literal", default=True) + purge_old_mailhog_configs = batou.component.Attribute( + "literal", default=True + ) http_auth_enable = batou.component.Attribute("literal", default=False) http_basic_auth = None @@ -196,7 +198,9 @@ def configure(self): if not self.public_smtp_name: self.public_smtp_name = self.host.fqdn - self.address = batou.utils.Address(self.public_smtp_name, self.smtp_port) + self.address = batou.utils.Address( + self.public_smtp_name, self.smtp_port + ) if self.http_basic_auth is None: self.http_auth = self.require_one("http_basic_auth") @@ -205,5 +209,7 @@ def configure(self): self += batou.lib.file.File( "/etc/local/nixos/mailpit.nix", - content=(files(__spec__.parent) / "resources/mailpit.nix").read_bytes(), + content=( + files(__spec__.parent) / "resources/mailpit.nix" + ).read_bytes(), ) diff --git a/src/batou_ext/mirror.py b/src/batou_ext/mirror.py index bbd4b56f..1c6bfa64 100644 --- a/src/batou_ext/mirror.py +++ b/src/batou_ext/mirror.py @@ -99,14 +99,18 @@ def nginx_enable_config(self): self += self.cert if self.authstring: - self.htpasswdfile = batou.lib.file.File("htpasswd", content=self.authstring) + self.htpasswdfile = batou.lib.file.File( + "htpasswd", content=self.authstring + ) self += self.htpasswdfile assert self.nginx_config_path self += batou.lib.file.File( "{}/{}.conf".format(self.nginx_config_path, self.public_name), - source=os.path.join(os.path.dirname(__file__), "resources/mirror.conf"), + source=os.path.join( + os.path.dirname(__file__), "resources/mirror.conf" + ), ) self += self.cert.activate_letsencrypt() diff --git a/src/batou_ext/nix.py b/src/batou_ext/nix.py index 58848184..910f469f 100644 --- a/src/batou_ext/nix.py +++ b/src/batou_ext/nix.py @@ -153,7 +153,9 @@ def configure(self): self.nix_env_name = self.expand( "{{component.profile_name}}-1.{{component.checksum.hexdigest()}}" ) - self += batou.lib.file.File(self.profile_name + ".nix", content=self.profile) + self += batou.lib.file.File( + self.profile_name + ".nix", content=self.profile + ) self += Package(self.nix_env_name, file=self._.path) self.user_profile_path = os.path.expanduser( "~/.nix-profile/etc/profile.d/{}.sh".format(self.profile_name) @@ -276,7 +278,9 @@ def configure(self): ExecStart=os.path.join(self.root.workdir, self.executable), Restart="always", Environment=[ - self.expand("LOCALE_ARCHIVE={{component.env['LOCALE_ARCHIVE']}}"), + self.expand( + "LOCALE_ARCHIVE={{component.env['LOCALE_ARCHIVE']}}" + ), self.expand("PATH={{component.env['PATH']}}"), self.expand("TZDIR={{component.env['TZDIR']}}"), ], @@ -304,7 +308,9 @@ def configure(self): self.checksum = self.parent.checksum self += batou.lib.file.File( service_file, - content=(files(__spec__.parent) / "resources/systemd.service").read_bytes(), + content=( + files(__spec__.parent) / "resources/systemd.service" + ).read_bytes(), ) self += Rebuild() @@ -411,13 +417,17 @@ class SensuChecks(batou.component.Component): purge_old_batou_json = batou.component.Attribute("literal", default=True) def configure(self): - self.services = self.require(batou.lib.nagios.Service.key, host=self.host) + self.services = self.require( + batou.lib.nagios.Service.key, host=self.host + ) checks = {} for service in self.services: assert getattr(service, "name", None) checks[service.name] = check = dict( standalone=True, - command=service.expand("{{component.command}} {{component.args}}"), + command=service.expand( + "{{component.command}} {{component.args}}" + ), ) if service.interval: @@ -452,7 +462,9 @@ def configure(self): "Need to set service_user inside environment file." ) user = self.environment.service_user - user_logrotate_conf = os.path.join("/etc/local/logrotate", user, "batou.conf") + user_logrotate_conf = os.path.join( + "/etc/local/logrotate", user, "batou.conf" + ) self += batou.lib.file.File( user_logrotate_conf, content=self.parent.logrotate_conf.content ) @@ -479,21 +491,29 @@ class PythonWithNixPackages(batou.component.Component): def configure(self): if not self.pythonPackages: - self.pythonPackages = "pkgs.{}Packages".format(self.python.replace(".", "")) + self.pythonPackages = "pkgs.{}Packages".format( + self.python.replace(".", "") + ) self += batou.lib.file.File( "{}.nix".format(self.python), - content=(files(__spec__.parent) / "resources/python.nix").read_bytes(), + content=( + files(__spec__.parent) / "resources/python.nix" + ).read_bytes(), ) self += batou.lib.file.File( "setupEnv-{}".format(self.python), mode=0o755, - content=(files(__spec__.parent) / "resources/setupEnv.sh").read_bytes(), + content=( + files(__spec__.parent) / "resources/setupEnv.sh" + ).read_bytes(), ) self.env_file = self._ self += batou.lib.file.File( "{}.c".format(self.python), - content=(files(__spec__.parent) / "resources/loader.c").read_bytes(), + content=( + files(__spec__.parent) / "resources/loader.c" + ).read_bytes(), ) self.provide(self.python, os.path.join(self.workdir, self.python)) @@ -531,7 +551,9 @@ def mapping_to_nix(obj): def str_to_nix(value): # https://nixos.org/manual/nix/stable/language/values.html#type-string - value = value.replace("\\", "\\\\").replace("${", "\\${").replace('"', '\\"') + value = ( + value.replace("\\", "\\\\").replace("${", "\\${").replace('"', '\\"') + ) return f'"{value}"' @@ -703,9 +725,13 @@ def verify(self, predicting=False): capture_output=True, ) except FileNotFoundError: - self.log("Cannot syntax-check Nix file, nix-instantiate not found.") + self.log( + "Cannot syntax-check Nix file, nix-instantiate not found." + ) except subprocess.CalledProcessError as e: - raise NixSyntaxCheckFailed(e.stderr.decode("utf8"), path=self.path) + raise NixSyntaxCheckFailed( + e.stderr.decode("utf8"), path=self.path + ) if update_needed: raise UpdateNeeded() diff --git a/src/batou_ext/oci.py b/src/batou_ext/oci.py index 87a73453..c8644581 100644 --- a/src/batou_ext/oci.py +++ b/src/batou_ext/oci.py @@ -180,7 +180,9 @@ def configure(self): else "docker" ) - if (self.registry_user or self.registry_password) and not self.registry_address: + if ( + self.registry_user or self.registry_password + ) and not self.registry_address: self.log( "WARN: you might want to specify the registry explicitly" " unless you really intend to log into the default" @@ -414,6 +416,8 @@ def _validate_remote_image(self, image_ident): # `docker manifest inspect` silently raises an error when unauthorized, # returns exit code 0 if stderr == "unauthorized": - raise RuntimeError("Wrong credentials for remote container registry") + raise RuntimeError( + "Wrong credentials for remote container registry" + ) valid = True return valid diff --git a/src/batou_ext/php.py b/src/batou_ext/php.py index 7a7f4060..ff847e2d 100644 --- a/src/batou_ext/php.py +++ b/src/batou_ext/php.py @@ -36,7 +36,9 @@ class Ini(batou.component.Component): logs = None def configure(self): - self._extensions_dir = os.path.expanduser("~/.nix-profile/lib/php/extensions/") + self._extensions_dir = os.path.expanduser( + "~/.nix-profile/lib/php/extensions/" + ) if self.logs is None: self.logs = self.map("logs") @@ -50,7 +52,9 @@ def configure(self): # Providing a php.ini self += batou.lib.file.File( "php.ini", - content=(files(__spec__.parent) / "resources/php/php.ini").read_bytes(), + content=( + files(__spec__.parent) / "resources/php/php.ini" + ).read_bytes(), ) self.php_ini = self._ @@ -123,7 +127,9 @@ def configure(self): self += batou.lib.file.File( self.name, mode=0o755, - content=(files(__spec__.parent) / "resources/php/php-fpm.sh").read_bytes(), + content=( + files(__spec__.parent) / "resources/php/php-fpm.sh" + ).read_bytes(), ) self._checksum.update(self._.content) diff --git a/src/batou_ext/postfixadmin/__init__.py b/src/batou_ext/postfixadmin/__init__.py index 010f7e74..8058d642 100644 --- a/src/batou_ext/postfixadmin/__init__.py +++ b/src/batou_ext/postfixadmin/__init__.py @@ -21,7 +21,9 @@ class PFA(Component): admin_password = None salt = "ab8f1b639d31875b59fa047481c581fd" - config = os.path.join(os.path.dirname(__file__), "postfixadmin", "config.local.php") + config = os.path.join( + os.path.dirname(__file__), "postfixadmin", "config.local.php" + ) def configure(self): self.db = self.require_one("pfa::database") @@ -43,7 +45,9 @@ def configure(self): self += SyncDirectory( self.basedir, - source=self.map("postfixadmin.orig/postfixadmin-{}".format(self.release)), + source=self.map( + "postfixadmin.orig/postfixadmin-{}".format(self.release) + ), ) self += File(self.basedir + "/config.local.php", source=self.config) @@ -55,6 +59,8 @@ def configure(self): def admin_password_encrypted(self): # password generation ported from postfixadmin/setup.php encrypt = hashlib.sha1() - encrypt.update("{}:{}".format(self.salt, self.admin_password).encode("utf-8")) + encrypt.update( + "{}:{}".format(self.salt, self.admin_password).encode("utf-8") + ) return "{}:{}".format(self.salt, encrypt.hexdigest()) diff --git a/src/batou_ext/postfixadmin/dovecot.py b/src/batou_ext/postfixadmin/dovecot.py index ef7779f3..a4019dda 100644 --- a/src/batou_ext/postfixadmin/dovecot.py +++ b/src/batou_ext/postfixadmin/dovecot.py @@ -5,8 +5,12 @@ class PFADovecot(Component): - local_conf = os.path.join(os.path.dirname(__file__), "dovecot", "local.conf") - database_conf = os.path.join(os.path.dirname(__file__), "dovecot", "database.conf") + local_conf = os.path.join( + os.path.dirname(__file__), "dovecot", "local.conf" + ) + database_conf = os.path.join( + os.path.dirname(__file__), "dovecot", "database.conf" + ) def configure(self): self.db = self.require_one("pfa::database") diff --git a/src/batou_ext/postfixadmin/postfix.py b/src/batou_ext/postfixadmin/postfix.py index c0d29c96..7eb7da48 100644 --- a/src/batou_ext/postfixadmin/postfix.py +++ b/src/batou_ext/postfixadmin/postfix.py @@ -23,7 +23,9 @@ def configure(self): self.provide("postfix", self.address) - self += File("/etc/postfix/myhostname", content=self.address.connect.host) + self += File( + "/etc/postfix/myhostname", content=self.address.connect.host + ) self += File( "/etc/postfix/main.d/40_local.cf", source=self.resource("local.cf") ) diff --git a/src/batou_ext/postgres.py b/src/batou_ext/postgres.py index 9e2a252c..c807e372 100644 --- a/src/batou_ext/postgres.py +++ b/src/batou_ext/postgres.py @@ -165,7 +165,9 @@ def configure(self): if self.extension_name is None: raise ValueError("Need to set extension name") if self.db is None: - raise ValueError("Need to specify a database to create extension in") + raise ValueError( + "Need to specify a database to create extension in" + ) def verify(self): cmd_out, cmt_err = self.pgcmd( diff --git a/src/batou_ext/python.py b/src/batou_ext/python.py index f7f0ccec..f915258f 100644 --- a/src/batou_ext/python.py +++ b/src/batou_ext/python.py @@ -39,7 +39,9 @@ def configure(self): def verify(self): with self.chdir(self.target): - self.assert_file_is_current(self.executable, ["Pipfile", "Pipfile.lock"]) + self.assert_file_is_current( + self.executable, ["Pipfile", "Pipfile.lock"] + ) # Is this Python (still) functional 'enough' # from a setuptools/distribute perspective? self.assert_cmd( @@ -251,11 +253,15 @@ def __patchelf(self, args, paths): # works for all ELFs below in the dependency tree in contrast to DT_RUNPATH. # It's impossible to use both at the same time because DT_RPATH will always # be ignored then. For more context, see `ld.so(8)`. - patchelf = f"nix run --impure --expr {patchelf_expr} -- patchelf --force-rpath" + patchelf = ( + f"nix run --impure --expr {patchelf_expr} -- patchelf --force-rpath" + ) cmd = f"xargs -P {self.patchelf_jobs} {patchelf} {args_}" proc = self.cmd(cmd, communicate=False) - stdout, stderr = proc.communicate(input=bytes("\n".join(paths) + "\n", "utf-8")) + stdout, stderr = proc.communicate( + input=bytes("\n".join(paths) + "\n", "utf-8") + ) if proc.returncode != 0: raise CmdExecutionError(cmd, proc.returncode, stdout, stderr) @@ -322,7 +328,9 @@ class BuildEnv(Component): def configure(self): self.env_dir = os.path.join(self.workdir, ".raw-python-env") - self.executable = os.path.join(self.env_dir, f"bin/python{self.version}") + self.executable = os.path.join( + self.env_dir, f"bin/python{self.version}" + ) def environment_variables(self): """Return dict of required environment variables to use the env.""" @@ -357,7 +365,9 @@ def verify(self): out, err = self.cmd(f"nix derivation show -f '{self.nix_file}'") derivation = json.loads(out) - expected_store_path = list(derivation.values())[0]["outputs"]["out"]["path"] + expected_store_path = list(derivation.values())[0]["outputs"]["out"][ + "path" + ] try: current_store_path = os.path.realpath(self.env_dir) except OSError: diff --git a/src/batou_ext/resources/watchdog-wrapper.py b/src/batou_ext/resources/watchdog-wrapper.py index 8deb8503..95bb3a34 100644 --- a/src/batou_ext/resources/watchdog-wrapper.py +++ b/src/batou_ext/resources/watchdog-wrapper.py @@ -54,7 +54,9 @@ def sleep(self, timeout: int): sleep(seconds_to_sleep) - def __report_failing_healthcheck(self, sleep_seconds: int, will_time_out: bool): + def __report_failing_healthcheck( + self, sleep_seconds: int, will_time_out: bool + ): log_message = f"Healthcheck failure (Reason: {self.last_healthcheck}), sleeping {sleep_seconds}." if will_time_out: warning(f"{log_message} Watchdog will likely kill process") @@ -90,7 +92,9 @@ def is_service_available(url: str, timeout: int) -> HealthCheckResult: return HealthCheckResult(False, ex) -def await_service(url: str, healthcheck_timeout: int, startup_loop_interval: int): +def await_service( + url: str, healthcheck_timeout: int, startup_loop_interval: int +): # No need to handle timeouts here: due to `Type=notify`, # this unit won't be up until this loop has terminated. # The timeout for that can be controlled in the unit directly diff --git a/src/batou_ext/roundcube/__init__.py b/src/batou_ext/roundcube/__init__.py index 594a2f4d..b360a4a6 100644 --- a/src/batou_ext/roundcube/__init__.py +++ b/src/batou_ext/roundcube/__init__.py @@ -55,7 +55,9 @@ def configure(self): self += Extract(download.target, target="roundcube.orig") self += SyncDirectory( self.basedir, - source=self.map("roundcube.orig/roundcubemail-{}".format(self.release)), + source=self.map( + "roundcube.orig/roundcubemail-{}".format(self.release) + ), ) self.db_dsnw = "{}://{}:{}@{}/{}".format( @@ -66,7 +68,9 @@ def configure(self): self.db.database, ) - self += File(self.basedir + "/config/config.inc.php", source=self.config) + self += File( + self.basedir + "/config/config.inc.php", source=self.config + ) self.fpm = FPM("roundcube") self += self.fpm diff --git a/src/batou_ext/run.py b/src/batou_ext/run.py index 29a9f2db..58b46eee 100644 --- a/src/batou_ext/run.py +++ b/src/batou_ext/run.py @@ -35,7 +35,9 @@ class Run(batou.component.Component): def configure(self): if not self.file and self.content: - self += batou.lib.file.File(self.command, content=self.content, mode=0o700) + self += batou.lib.file.File( + self.command, content=self.content, mode=0o700 + ) self.command_file = self._ elif self.file: self.command_file = self.file diff --git a/src/batou_ext/s3.py b/src/batou_ext/s3.py index a8162701..e6a7046d 100644 --- a/src/batou_ext/s3.py +++ b/src/batou_ext/s3.py @@ -132,7 +132,9 @@ def verify(self): if not os.path.exists(self.target): raise batou.UpdateNeeded() if self.checksum: - if self.checksum != batou.utils.hash(self.target, self.checksum_function): + if self.checksum != batou.utils.hash( + self.target, self.checksum_function + ): raise batou.UpdateNeeded() else: if not os.path.exists(self.etag_file): @@ -151,7 +153,9 @@ def verify(self): def update(self): self.obj.download_file(self.target) if self.checksum: - target_checksum = batou.utils.hash(self.target, self.checksum_function) + target_checksum = batou.utils.hash( + self.target, self.checksum_function + ) assert self.checksum == target_checksum, """\ Checksum mismatch! expected: %s diff --git a/src/batou_ext/s3_bootstrap.py b/src/batou_ext/s3_bootstrap.py index d192ceed..56fa5808 100644 --- a/src/batou_ext/s3_bootstrap.py +++ b/src/batou_ext/s3_bootstrap.py @@ -147,7 +147,9 @@ def verify(self, account_created): if not account_created and not rg_account_exists( self.rg_name, self.host, self.sudo_password ): - raise ConfigurationError(f"Account for rg {self.rg_name} does not exist!") + raise ConfigurationError( + f"Account for rg {self.rg_name} does not exist!" + ) def apply(self, _) -> Keypair: data = json.loads( @@ -164,7 +166,9 @@ def apply(self, _) -> Keypair: ], ) ) - kp = Keypair(data["keys"][-1]["access_key"], data["keys"][-1]["secret_key"]) + kp = Keypair( + data["keys"][-1]["access_key"], data["keys"][-1]["secret_key"] + ) print(f"AWS_ACCESS_KEY_ID={kp.key_id}") print(f"AWS_SECRET_ACCESS_KEY={kp.secret_key}") @@ -190,7 +194,9 @@ def verify(self, key: Union[Keypair, None]): self.bucket = self.resource.Bucket(self.name) if self.bucket.creation_date: - raise StateAlreadyExists(f"Bucket with name {self.name} already exists!") + raise StateAlreadyExists( + f"Bucket with name {self.name} already exists!" + ) def apply(self, keys): self.bucket.create() @@ -212,7 +218,9 @@ def add(self, prefix: str, days: int): self.rules[prefix] = days def __str__(self) -> str: - return ", ".join(f"'{prefix}' expires in {days} days" for prefix, days in self) + return ", ".join( + f"'{prefix}' expires in {days} days" for prefix, days in self + ) class CreateLifecyclePolicyConfiguration(Step): @@ -266,7 +274,9 @@ def run(): if ( keypair_needed - or inquirer.confirm(message="Do you need a keypair for access?").execute() + or inquirer.confirm( + message="Do you need a keypair for access?" + ).execute() ): if sudo_password is None: sudo_password = ask_sudo_password() @@ -275,11 +285,17 @@ def run(): message="Which host to use to call radosgw-admin (must be an FQDN)?" ).execute() plan += CreateKeypair(rg_name, sudo_password, ceph_osd_host) - elif not all(x in environ for x in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"]): - error("AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY missing from environment") + elif not all( + x in environ for x in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] + ): + error( + "AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY missing from environment" + ) exit(1) - bucket_name = inquirer.text(message="What's the name of the bucket?").execute() + bucket_name = inquirer.text( + message="What's the name of the bucket?" + ).execute() plan += CreateBucket(bucket_name) rules = Rules() diff --git a/src/batou_ext/ssh.py b/src/batou_ext/ssh.py index 3660efc4..885b1b4b 100644 --- a/src/batou_ext/ssh.py +++ b/src/batou_ext/ssh.py @@ -73,7 +73,9 @@ def configure(self): self += Purge(f"~/.ssh/{file_name_ed25519}") if self.id_ed25519_pub: - self += File(f"~/.ssh/{file_name_ed25519}.pub", content=self.id_ed25519_pub) + self += File( + f"~/.ssh/{file_name_ed25519}.pub", content=self.id_ed25519_pub + ) # ScanHost for host in self.scan_hosts: diff --git a/src/batou_ext/ssl.py b/src/batou_ext/ssl.py index 4c1aa664..8a782ffe 100644 --- a/src/batou_ext/ssl.py +++ b/src/batou_ext/ssl.py @@ -68,7 +68,9 @@ class Certificate(batou.component.Component): "/082da2527cb4aaa3a4740ba03e550205b076f822/dehydrated" ) dehydrated_checksum = "md5:95a90950d3b9c01174e4f4f98cf3bd53" - dehydrated_publickey_algo = batou.component.Attribute(str, default="secp384r1") + dehydrated_publickey_algo = batou.component.Attribute( + str, default="secp384r1" + ) extracommand = None @@ -108,10 +110,14 @@ class Certificate(batou.component.Component): def configure(self): if not isinstance(self.alternative_names, (tuple, list)): - raise ValueError('"alternative_names" needs to be a tuple of string.') + raise ValueError( + '"alternative_names" needs to be a tuple of string.' + ) if not self.refresh_timing: h = int( - hashlib.md5(six.ensure_binary(self.domain, "UTF-8")).hexdigest(), + hashlib.md5( + six.ensure_binary(self.domain, "UTF-8") + ).hexdigest(), 16, ) self.refresh_timing = "{} {} * * *".format(h % 60, h % 24) @@ -170,7 +176,9 @@ def configure(self): self += batou.lib.file.Mode("dehydrated", mode=0o755) if not self.wellknown and self.docroot: - self.wellknown = "{}/.well-known/acme-challenge".format(self.docroot) + self.wellknown = "{}/.well-known/acme-challenge".format( + self.docroot + ) self += batou.lib.file.File( self.wellknown, ensure="directory", leading=True ) @@ -201,7 +209,9 @@ def configure(self): self += batou.lib.file.File( self.expand("cert-{{component.domain}}.sh"), - content=(files(__spec__.parent) / "resources/cert.sh").read_bytes(), + content=( + files(__spec__.parent) / "resources/cert.sh" + ).read_bytes(), mode=0o700, ) self.cert_sh = self._ @@ -330,7 +340,8 @@ def configure(self): self += batou.lib.file.File( "cert_check_{}.sh".format(self.name), content=( - files(__spec__.parent) / "resources/ssl/local_certificate_check.sh" + files(__spec__.parent) + / "resources/ssl/local_certificate_check.sh" ).read_bytes(), mode=0o755, ) diff --git a/src/batou_ext/tests/test_configure.py b/src/batou_ext/tests/test_configure.py index 2867c8e1..b801d7ab 100644 --- a/src/batou_ext/tests/test_configure.py +++ b/src/batou_ext/tests/test_configure.py @@ -49,7 +49,9 @@ def pytest_generate_tests(metafunc): idlist = [dotted_name(x) for x in classes] argvalues = [(x,) for x in classes] - metafunc.parametrize(("component",), argvalues, ids=idlist, scope="function") + metafunc.parametrize( + ("component",), argvalues, ids=idlist, scope="function" + ) def dotted_name(cls): diff --git a/src/batou_ext/tests/test_jenkins.py b/src/batou_ext/tests/test_jenkins.py index 1f76036e..bba78dcd 100644 --- a/src/batou_ext/tests/test_jenkins.py +++ b/src/batou_ext/tests/test_jenkins.py @@ -23,7 +23,9 @@ def test_set_versions_git_mode(tmpdir): with mock.patch("batou_ext.jenkins.git_resolve") as git_resolve: git_resolve.return_value = "abcdef" - batou_ext.jenkins.set_versions(str(ini), '{"prog1": "a-tag", "prog2": ""}') + batou_ext.jenkins.set_versions( + str(ini), '{"prog1": "a-tag", "prog2": ""}' + ) assert dedent( """\ diff --git a/src/batou_ext/tests/test_oci.py b/src/batou_ext/tests/test_oci.py index 355f1906..0f362d34 100644 --- a/src/batou_ext/tests/test_oci.py +++ b/src/batou_ext/tests/test_oci.py @@ -95,7 +95,9 @@ def test_remote_image_validation_is_cached(root, activate, mocker): activate.verify() # Validate is called with the local digest - activate._validate_remote_image.assert_called_with("alpine:latest@local-digest") + activate._validate_remote_image.assert_called_with( + "alpine:latest@local-digest" + ) activate._validate_remote_image.reset_mock() # A *different* container will re-use the cache! @@ -110,7 +112,9 @@ def test_remote_image_validation_is_cached(root, activate, mocker): a2._validate_remote_image.return_value = False a2._get_local_digest.return_value = "local-digest" - with pytest.raises(batou.UpdateNeeded, match="Cached remote version update."): + with pytest.raises( + batou.UpdateNeeded, match="Cached remote version update." + ): a2.verify() activate._validate_remote_image.assert_not_called() diff --git a/src/batou_ext/tests/test_postgres.py b/src/batou_ext/tests/test_postgres.py index 925130db..799e6761 100644 --- a/src/batou_ext/tests/test_postgres.py +++ b/src/batou_ext/tests/test_postgres.py @@ -109,4 +109,6 @@ def test_grant_check_permissions_missing(pgcmd_mock, grant): ] result = grant._check_permissions() assert "table permissions" in result[0] - assert "INSERT" in result[0] or "UPDATE" in result[0] or "DELETE" in result[0] + assert ( + "INSERT" in result[0] or "UPDATE" in result[0] or "DELETE" in result[0] + ) diff --git a/src/batou_ext/versions.py b/src/batou_ext/versions.py index 33df065a..c8ee0f33 100644 --- a/src/batou_ext/versions.py +++ b/src/batou_ext/versions.py @@ -84,7 +84,9 @@ def select_versions_interactive(basedir: str) -> dict: return interative_versions -def get_current_versions(basedir: str, environment: batou.environment.Environment): +def get_current_versions( + basedir: str, environment: batou.environment.Environment +): versions = configparser.ConfigParser() versions_ini = find_versions_ini(environment) versions.read(os.path.join(basedir, versions_ini)) diff --git a/tox.ini b/tox.ini index 908c72e2..541350a3 100644 --- a/tox.ini +++ b/tox.ini @@ -5,10 +5,12 @@ [tox] envlist = - py36 - py37 - py38 py39 + py310 + py311 + py312 + py313 + py314 [testenv] usedevelop = true