From a89583d3fc9b26fb3b1ad72f8be51360629b9a19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89rico=20Andrei?= Date: Mon, 28 Apr 2025 20:00:46 -0300 Subject: [PATCH 1/2] Refactor Migration Tool --- Products/CMFPlone/MigrationTool.py | 472 +++++++++++++++++------------ Products/CMFPlone/configure.zcml | 6 + 2 files changed, 288 insertions(+), 190 deletions(-) diff --git a/Products/CMFPlone/MigrationTool.py b/Products/CMFPlone/MigrationTool.py index 349ca888d5..75716ed160 100644 --- a/Products/CMFPlone/MigrationTool.py +++ b/Products/CMFPlone/MigrationTool.py @@ -2,10 +2,13 @@ from AccessControl.class_init import InitializeClass from AccessControl.requestmethod import postonly from App.config import getConfiguration +from collections.abc import Generator +from contextlib import contextmanager from importlib.metadata import PackageNotFoundError from importlib.metadata import version as dist_version from io import StringIO from OFS.SimpleItem import SimpleItem +from OFS.Traversable import Traversable from plone.base.interfaces import IMigrationTool from Products.CMFCore.permissions import ManagePortal from Products.CMFCore.utils import getToolByName @@ -13,9 +16,13 @@ from Products.CMFCore.utils import UniqueObject from Products.CMFPlone.factory import _DEFAULT_PROFILE from Products.CMFPlone.PloneBaseTool import PloneBaseTool +from Products.GenericSetup.tool import SetupTool from ZODB.POSException import ConflictError +from zope.component import getUtility +from zope.component.hooks import getSite from zope.interface import implementer - +from zope.interface import Interface +from typing import TypedDict import logging import sys import transaction @@ -25,6 +32,43 @@ _upgradePaths = {} +def _pil_version() -> str: + """Return version of the image package being used.""" + version = "unknown" + try: + version = dist_version("PIL") + except PackageNotFoundError: + try: + version = dist_version("PILwoTK") + except PackageNotFoundError: + try: + vars["PIL"] = "%s (Pillow)" % dist_version("Pillow") + except PackageNotFoundError: + try: + import _imaging + + _imaging # pyflakes + except ImportError: + pass + return version + + +@contextmanager +def get_logger(stream: StringIO) -> Generator[logging.Logger]: + """Setup logging during migration.""" + handler = logging.StreamHandler(stream) + handler.setLevel(logging.DEBUG) + logger.addHandler(handler) + gslogger = logging.getLogger("GenericSetup") + gslogger.addHandler(handler) + try: + yield logger + finally: + # Remove new handler + logger.removeHandler(handler) + gslogger.removeHandler(handler) + + class Addon: """A profile or product. @@ -46,10 +90,10 @@ def __init__(self, profile_id=None, check_module=None): self.profile_id = profile_id self.check_module = check_module - def __repr__(self): + def __repr__(self) -> str: return f"<{self.__class__.__name__} profile {self.profile_id}>" - def safe(self): + def safe(self) -> bool: # Is this addon safe to upgrade? # Is it safe to pass its profile id to @@ -69,52 +113,96 @@ def safe(self): __import__(self.check_module) except ImportError: logger.info( - "Cannot import module %s. Ignoring %s", self.check_module, self + f"Cannot import module {self.check_module}. Ignoring {self}" ) return False return True class AddonList(list): - def upgrade_all(self, context): + + def upgrade_all(self, context: Traversable) -> None: setup = getToolByName(context, "portal_setup") for addon in self: if addon.safe(): setup.upgradeProfile(addon.profile_id, quiet=True) -# List of upgradeable packages. Obvious items to add here, are all -# core packages that actually have upgrade steps. -# Good start is portal_setup.listProfilesWithUpgrades() -# Please use 'check_module' for packages that are not direct dependencies -# of Products.CMFPlone, but of the Plone package. -ADDON_LIST = AddonList( - [ - Addon(profile_id="Products.CMFEditions:CMFEditions"), - Addon( - profile_id="Products.CMFPlacefulWorkflow:CMFPlacefulWorkflow", - check_module="Products.CMFPlacefulWorkflow", - ), - Addon(profile_id="Products.PlonePAS:PlonePAS"), - Addon(profile_id="plone.app.caching:default", check_module="plone.app.caching"), - Addon(profile_id="plone.app.contenttypes:default"), - Addon(profile_id="plone.app.dexterity:default"), - Addon(profile_id="plone.app.discussion:default"), - Addon(profile_id="plone.app.event:default"), - Addon(profile_id="plone.app.iterate:default", check_module="plone.app.iterate"), - Addon( - profile_id="plone.app.multilingual:default", - check_module="plone.app.multilingual", - ), - Addon(profile_id="plone.app.querystring:default"), - Addon(profile_id="plone.app.theming:default"), - Addon(profile_id="plone.app.users:default"), - Addon(profile_id="plone.restapi:default", check_module="plone.restapi"), - Addon(profile_id="plone.session:default"), - Addon(profile_id="plone.staticresources:default"), - Addon(profile_id="plone.volto:default", check_module="plone.volto"), - Addon(profile_id="plonetheme.barceloneta:default"), - ] +class IAddonList(Interface): + """Utility providing a list of add ons managed by the migration tool.""" + + addon_list: AddonList + + +@implementer(IAddonList) +class LocalAddonList: + # List of upgradeable packages. Obvious items to add here, are all + # core packages that actually have upgrade steps. + # Good start is portal_setup.listProfilesWithUpgrades() + # Please use 'check_module' for packages that are not direct dependencies + # of Products.CMFPlone, but of the Plone package + addon_list: AddonList = AddonList( + [ + Addon(profile_id="Products.CMFEditions:CMFEditions"), + Addon( + profile_id="Products.CMFPlacefulWorkflow:CMFPlacefulWorkflow", + check_module="Products.CMFPlacefulWorkflow", + ), + Addon(profile_id="Products.PlonePAS:PlonePAS"), + Addon( + profile_id="plone.app.caching:default", check_module="plone.app.caching" + ), + Addon(profile_id="plone.app.contenttypes:default"), + Addon(profile_id="plone.app.dexterity:default"), + Addon(profile_id="plone.app.discussion:default"), + Addon(profile_id="plone.app.event:default"), + Addon( + profile_id="plone.app.iterate:default", check_module="plone.app.iterate" + ), + Addon( + profile_id="plone.app.multilingual:default", + check_module="plone.app.multilingual", + ), + Addon(profile_id="plone.app.querystring:default"), + Addon(profile_id="plone.app.theming:default"), + Addon(profile_id="plone.app.users:default"), + Addon(profile_id="plone.restapi:default", check_module="plone.restapi"), + Addon(profile_id="plone.session:default"), + Addon(profile_id="plone.staticresources:default"), + Addon(profile_id="plone.volto:default", check_module="plone.volto"), + Addon(profile_id="plonetheme.barceloneta:default"), + ] + ) + + +_DEFAULT_PACKAGE_NAME = "Products.CMFPlone" +_DEFAULT_FRIENDLY_NAME = "Plone" + + +class CoreVersionInformation(TypedDict): + name: str + package_name: str + package_version: str + instance_version: str + fs_version: str + + +VersionInformation = TypedDict( + "VersionInformation", + { + "Python": str, + "Zope": str, + "Platform": str, + "CMFPlone": str, + "Plone": str, + "Plone Instance": str, + "Plone File System": str, + "CMF": str, + "Debug mode": str, + "PIL": str, + "core": CoreVersionInformation, + "packages": dict[str, str], + }, ) @@ -122,52 +210,47 @@ def upgrade_all(self, context): class MigrationTool(PloneBaseTool, UniqueObject, SimpleItem): """Handles migrations between Plone releases""" - id = "portal_migration" - meta_type = "Plone Migration Tool" - toolicon = "skins/plone_images/site_icon.png" + id: str = "portal_migration" + meta_type: str = "Plone Migration Tool" + toolicon: str = "skins/plone_images/site_icon.png" - _profile = _DEFAULT_PROFILE - _package_name = "Products.CMFPlone" + profile: str = _DEFAULT_PROFILE + package_name: str = _DEFAULT_PACKAGE_NAME + friendly_name: str = _DEFAULT_FRIENDLY_NAME - manage_options = ( + manage_options: tuple[dict[str, str], ...] = ( {"label": "Upgrade", "action": "../@@plone-upgrade"}, ) + SimpleItem.manage_options - _needRecatalog = 0 - _needUpdateRole = 0 + _needRecatalog: int = 0 + _needUpdateRole: int = 0 security = ClassSecurityInfo() - security.declareProtected(ManagePortal, "getInstanceVersion") - - security.declareProtected(ManagePortal, "getBaseProfile") - - def getBaseProfile(self): - """Get the base profile used for migrations""" - return getattr(self, "_profile", _DEFAULT_PROFILE) - - security.declareProtected(ManagePortal, "setBaseProfile") + security.declareProtected(ManagePortal, "initializeTool") - def setBaseProfile(self, profile): - """Set the base profile used for migrations""" - self._profile = profile + @property + def setup(self) -> SetupTool: + site = getSite() + return getToolByName(site, "portal_setup") - security.declareProtected(ManagePortal, "getPackageName") + def initializeTool(self, profile: str, package_name: str, friendly_name: str = ""): + """Initialize the migration tool.""" + self.profile = profile + self.package_name = package_name + self.friendly_name = friendly_name if friendly_name else package_name - def getPackageName(self): - """Get the package name used for migrations""" - return getattr(self, "_package_name", "Products.CMFPlone") + @property + def addon_list(self) -> AddonList: + utility = getUtility(IAddonList, self.package_name) + return utility.addon_list - security.declareProtected(ManagePortal, "setPackageName") - - def setPackageName(self, package_name): - """Set the package name used for migrations""" - self._package_name = package_name + security.declareProtected(ManagePortal, "getInstanceVersion") - def getInstanceVersion(self): + def getInstanceVersion(self) -> str: # The version this instance of plone is on. - setup = getToolByName(self, "portal_setup") - profile = self.getBaseProfile() + setup = self.setup + profile = self.profile version = setup.getLastVersionForProfile(profile) if isinstance(version, tuple): version = ".".join(version) @@ -187,83 +270,89 @@ def getInstanceVersion(self): _version = _version.replace("-", ".") version = _version else: - version = setup.getVersionForProfile(_DEFAULT_PROFILE) + version = setup.getVersionForProfile(profile) self.setInstanceVersion(version) return version security.declareProtected(ManagePortal, "setInstanceVersion") - def setInstanceVersion(self, version): + def setInstanceVersion(self, version: str) -> None: # The version this instance of plone is on. - setup = getToolByName(self, "portal_setup") - profile = self.getBaseProfile() - setup.setLastVersionForProfile(profile, version) + setup = self.setup + setup.setLastVersionForProfile(self.profile, version) self._version = False security.declareProtected(ManagePortal, "getFileSystemVersion") - def getFileSystemVersion(self): + def getFileSystemVersion(self) -> str | None: # The version this instance of plone is on. - setup = getToolByName(self, "portal_setup") - profile = self.getBaseProfile() + setup = self.setup try: - return setup.getVersionForProfile(profile) + return setup.getVersionForProfile(self.profile) except KeyError: pass return None security.declareProtected(ManagePortal, "getSoftwareVersion") - def getSoftwareVersion(self): + def getSoftwareVersion(self) -> str: # The software version. - package_name = self.getPackageName() try: - return dist_version(package_name) + return dist_version(self.package_name) except PackageNotFoundError: # Fall back to CMFPlone for backward compatibility - return dist_version("Products.CMFPlone") + return dist_version(_DEFAULT_PACKAGE_NAME) security.declareProtected(ManagePortal, "needUpgrading") - def needUpgrading(self): + def needUpgrading(self) -> bool: # Need upgrading? return self.getInstanceVersion() != self.getFileSystemVersion() security.declareProtected(ManagePortal, "coreVersions") - def coreVersions(self): + def coreVersions(self) -> VersionInformation: # Useful core information. - vars = {} - vars["Zope"] = dist_version("Zope") - vars["Python"] = sys.version - vars["Platform"] = sys.platform - vars["Plone"] = dist_version("Products.CMFPlone") - vars["Plone Instance"] = self.getInstanceVersion() - vars["Plone File System"] = self.getFileSystemVersion() - vars["CMF"] = dist_version("Products.CMFCore") - vars["Debug mode"] = getConfiguration().debug_mode and "Yes" or "No" - try: - vars["PIL"] = dist_version("PIL") - except PackageNotFoundError: + plone_version = dist_version("Products.CMFPlone") + instance_version = self.getInstanceVersion() + fs_version = self.getFileSystemVersion() + vars = { + "Python": sys.version, + "Zope": dist_version("Zope"), + "Platform": sys.platform, + "CMFPlone": plone_version, + "Plone": plone_version, + "Plone Instance": instance_version, + "Plone File System": fs_version, + "CMF": dist_version("Products.CMFCore"), + "Debug mode": "Yes" if getConfiguration().debug_mode else "No", + "PIL": _pil_version(), + "core": { + "name": self.friendly_name, + "package_name": self.package_name, + "package_version": self.getSoftwareVersion(), + "instance_version": instance_version, + "fs_version": fs_version, + }, + "packages": {}, + } + additional_packages = ( + "plone.classicui", + "plone.distribution", + "plone.exportimport", + "plone.restapi", + "plone.volto", + ) + for package_name in additional_packages: try: - vars["PIL"] = dist_version("PILwoTK") + vars["packages"][package_name] = dist_version(package_name) except PackageNotFoundError: - try: - vars["PIL"] = "%s (Pillow)" % dist_version("Pillow") - except PackageNotFoundError: - try: - import _imaging - - _imaging # pyflakes - vars["PIL"] = "unknown" - except ImportError: - pass - + pass return vars security.declareProtected(ManagePortal, "coreVersionsList") - def coreVersionsList(self): + def coreVersionsList(self) -> list[str | dict | CoreVersionInformation]: # Useful core information. res = self.coreVersions().items() res.sort() @@ -271,36 +360,30 @@ def coreVersionsList(self): security.declareProtected(ManagePortal, "needUpdateRole") - def needUpdateRole(self): + def needUpdateRole(self) -> bool: # Do roles need to be updated? - return self._needUpdateRole + return bool(self._needUpdateRole) security.declareProtected(ManagePortal, "needRecatalog") - def needRecatalog(self): + def needRecatalog(self) -> bool: # Does this thing now need recataloging? - return self._needRecatalog + return bool(self._needRecatalog) security.declareProtected(ManagePortal, "listUpgrades") - def listUpgrades(self): + def listUpgrades(self) -> list: # List available upgrade steps for our default profile. # Do not include upgrade steps for too new versions: # using a newer plone.app.upgrade version should not give problems. - setup = getToolByName(self, "portal_setup") - profile = self.getBaseProfile() + setup = self.setup fs_version = self.getFileSystemVersion() - upgrades = setup.listUpgrades(profile, dest=fs_version) + upgrades = setup.listUpgrades(self.profile, dest=fs_version) return upgrades - security.declareProtected(ManagePortal, "upgrade") - - def upgrade(self, REQUEST=None, dry_run=None, swallow_errors=True): - # Perform the upgrade. - setup = getToolByName(self, "portal_setup") + security.declareProtected(ManagePortal, "list_steps") - # This sets the profile version if it wasn't set yet - version = self.getInstanceVersion() + def list_steps(self) -> list: upgrades = self.listUpgrades() steps = [] for u in upgrades: @@ -308,37 +391,84 @@ def upgrade(self, REQUEST=None, dry_run=None, swallow_errors=True): steps.extend(u) else: steps.append(u) + return steps + def _upgrade_run_steps( + self, steps: list, logger: logging.Logger, swallow_errors: bool + ) -> None: + setup = self.setup + for step in steps: + try: + step_title = step["title"] + step["step"].doStep(setup) + setup.setLastVersionForProfile(self.profile, step["dest"]) + logger.info(f"Ran upgrade step: {step_title}") + except (ConflictError, KeyboardInterrupt): + raise + except Exception: + logger.error("Upgrade aborted. Error:\n", exc_info=True) + + if not swallow_errors: + raise + else: + # abort transaction to safe the zodb + transaction.abort() + break + + def _upgrade_recatalog(self, logger: logging.Logger, swallow_errors: bool) -> None: + if not self.needRecatalog(): + return + logger.info("Recatalog needed. This may take a while...") try: - stream = StringIO() - handler = logging.StreamHandler(stream) - handler.setLevel(logging.DEBUG) - logger.addHandler(handler) - gslogger = logging.getLogger("GenericSetup") - gslogger.addHandler(handler) + catalog = self.portal_catalog + # Reduce threshold for the reindex run + old_threshold = catalog.threshold + pg_threshold = getattr(catalog, "pgthreshold", 0) + catalog.pgthreshold = 300 + catalog.threshold = 2000 + catalog.refreshCatalog(clear=1) + catalog.threshold = old_threshold + catalog.pgthreshold = pg_threshold + self._needRecatalog = 0 + except (ConflictError, KeyboardInterrupt): + raise + except Exception: + logger.error( + "Exception was thrown while cataloging:\n", + exc_info=True, + ) + if not swallow_errors: + raise + + def _upgrade_roles(self, logger: logging.Logger, swallow_errors: bool) -> None: + if self.needUpdateRole(): + logger.info("Role update needed. This may take a while...") + try: + self.portal_workflow.updateRoleMappings() + self._needUpdateRole = 0 + except (ConflictError, KeyboardInterrupt): + raise + except Exception: + logger.error( + "Exception was thrown while updating role mappings", + exc_info=True, + ) + if not swallow_errors: + raise + security.declareProtected(ManagePortal, "upgrade") + + def upgrade(self, REQUEST=None, dry_run=None, swallow_errors=True) -> str: + # This sets the profile version if it wasn't set yet + version = self.getInstanceVersion() + steps = self.list_steps() + stream = StringIO() + with get_logger(stream) as logger: if dry_run: logger.info("Dry run selected.") - logger.info("Starting the migration from version: %s" % version) - - for step in steps: - try: - step["step"].doStep(setup) - setup.setLastVersionForProfile(_DEFAULT_PROFILE, step["dest"]) - logger.info("Ran upgrade step: %s" % step["title"]) - except (ConflictError, KeyboardInterrupt): - raise - except Exception: - logger.error("Upgrade aborted. Error:\n", exc_info=True) - - if not swallow_errors: - raise - else: - # abort transaction to safe the zodb - transaction.abort() - break - + logger.info(f"Starting the migration from version: {version}") + self._upgrade_run_steps(steps, logger, swallow_errors) logger.info("End of upgrade path, main migration has finished.") if self.needUpgrading(): @@ -346,46 +476,12 @@ def upgrade(self, REQUEST=None, dry_run=None, swallow_errors=True): logger.error("Migration has failed") else: logger.info("Starting upgrade of core addons.") - ADDON_LIST.upgrade_all(self) + self.addon_list.upgrade_all(self) logger.info("Done upgrading core addons.") # do this once all the changes have been done - if self.needRecatalog(): - logger.info("Recatalog needed. This may take a while...") - try: - catalog = self.portal_catalog - # Reduce threshold for the reindex run - old_threshold = catalog.threshold - pg_threshold = getattr(catalog, "pgthreshold", 0) - catalog.pgthreshold = 300 - catalog.threshold = 2000 - catalog.refreshCatalog(clear=1) - catalog.threshold = old_threshold - catalog.pgthreshold = pg_threshold - self._needRecatalog = 0 - except (ConflictError, KeyboardInterrupt): - raise - except Exception: - logger.error( - "Exception was thrown while cataloging:" "\n", exc_info=True - ) - if not swallow_errors: - raise - - if self.needUpdateRole(): - logger.info("Role update needed. This may take a while...") - try: - self.portal_workflow.updateRoleMappings() - self._needUpdateRole = 0 - except (ConflictError, KeyboardInterrupt): - raise - except Exception: - logger.error( - "Exception was thrown while updating " "role mappings", - exc_info=True, - ) - if not swallow_errors: - raise + self._upgrade_recatalog(logger, swallow_errors=swallow_errors) + self._upgrade_roles(logger, swallow_errors=swallow_errors) logger.info("Your Plone instance is now up-to-date.") if dry_run: @@ -394,10 +490,6 @@ def upgrade(self, REQUEST=None, dry_run=None, swallow_errors=True): return stream.getvalue() - finally: - logger.removeHandler(handler) - gslogger.removeHandler(handler) - upgrade = postonly(upgrade) diff --git a/Products/CMFPlone/configure.zcml b/Products/CMFPlone/configure.zcml index b0f6a96eb3..232df767bd 100644 --- a/Products/CMFPlone/configure.zcml +++ b/Products/CMFPlone/configure.zcml @@ -97,6 +97,12 @@ + + + From 8d6ce0d3defe75906e8d345d5e65a624461c4b10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89rico=20Andrei?= Date: Tue, 29 Apr 2025 12:08:36 -0300 Subject: [PATCH 2/2] Fix reference to vars --- Products/CMFPlone/MigrationTool.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Products/CMFPlone/MigrationTool.py b/Products/CMFPlone/MigrationTool.py index 75716ed160..3d45d2bd9f 100644 --- a/Products/CMFPlone/MigrationTool.py +++ b/Products/CMFPlone/MigrationTool.py @@ -42,7 +42,8 @@ def _pil_version() -> str: version = dist_version("PILwoTK") except PackageNotFoundError: try: - vars["PIL"] = "%s (Pillow)" % dist_version("Pillow") + pillow_version = dist_version("Pillow") + version = f"{pillow_version} (Pillow)" except PackageNotFoundError: try: import _imaging