diff --git a/mesonbuild/_typing.py b/mesonbuild/_typing.py index 8336c46bd06d..6f3919f0031d 100644 --- a/mesonbuild/_typing.py +++ b/mesonbuild/_typing.py @@ -9,9 +9,11 @@ __all__ = [ 'Protocol', + 'DataclassInstance', 'ImmutableListProtocol' ] +import dataclasses import typing # We can change this to typing when we require python 3.8 @@ -21,6 +23,10 @@ T = typing.TypeVar('T') +# Copied from typeshed. Blarg that they don't expose this +class DataclassInstance(Protocol): + __dataclass_fields__: typing.ClassVar[dict[str, dataclasses.Field[typing.Any]]] + class StringProtocol(Protocol): def __str__(self) -> str: ... diff --git a/mesonbuild/cargo/interpreter.py b/mesonbuild/cargo/interpreter.py index c1b0bdee48f8..13deb82be0ac 100644 --- a/mesonbuild/cargo/interpreter.py +++ b/mesonbuild/cargo/interpreter.py @@ -20,7 +20,7 @@ import typing as T from pathlib import PurePath -from . import builder, version +from . import builder, raw, version from .cfg import eval_cfg from .toml import load_toml from .manifest import Manifest, CargoLock, CargoLockPackage, Workspace, fixup_meson_varname @@ -28,11 +28,11 @@ is_parent_path, lazy_property, MesonException, MachineChoice, PerMachine, unique_list, SubProject, ) +from .validate import validator from .. import coredata, mlog from ..wrap.wrap import PackageDefinition if T.TYPE_CHECKING: - from . import raw from .. import mparser from typing_extensions import Literal @@ -634,8 +634,12 @@ def _load_manifest(self, subdir: str, workspace: T.Optional[Workspace] = None, m self.build_def_files.append(filename) if 'workspace' in raw_manifest: + if not (result := validator(raw.Workspace)(raw_manifest)): + raise MesonException(f'invalid {subdir}/Cargo.toml at {".".join(result.path)}') manifest_ = Workspace.from_raw(raw_manifest, path) elif 'package' in raw_manifest: + if not (result := validator(raw.Manifest)(raw_manifest)): + raise MesonException(f'invalid {subdir}/Cargo.toml at {".".join(result.path)}') manifest_ = Manifest.from_raw(raw_manifest, path, workspace, member_path) else: raise MesonException(f'{subdir}/Cargo.toml does not have [package] or [workspace] section') @@ -850,6 +854,8 @@ def load_cargo_lock(filename: str, subproject_dir: str) -> T.Optional[CargoLock] # provides multiple dependency names. if os.path.exists(filename): toml = load_toml(filename) + if not validator(raw.CargoLock)(toml): + raise MesonException(f'invalid {filename}') raw_cargolock = T.cast('raw.CargoLock', toml) cargolock = CargoLock.from_raw(raw_cargolock) packagefiles_dir = os.path.join(subproject_dir, 'packagefiles') diff --git a/mesonbuild/cargo/manifest.py b/mesonbuild/cargo/manifest.py index 8afd5fe7c2a2..ad2f2544034a 100644 --- a/mesonbuild/cargo/manifest.py +++ b/mesonbuild/cargo/manifest.py @@ -14,20 +14,17 @@ from pathlib import PurePath -from . import version -from ..mesonlib import MesonException, lazy_property, MachineChoice +from . import raw, version +from .raw import EDITION, CRATE_TYPE, LINT_LEVEL +from .validate import dataclass_field_validators from .. import mlog +from ..mesonlib import MesonException, lazy_property, MachineChoice, MesonBugException +from ..wrap.wrap import PackageDefinition if T.TYPE_CHECKING: - from typing_extensions import Protocol, Self + from typing_extensions import Self + from .._typing import DataclassInstance - from . import raw - from .raw import EDITION, CRATE_TYPE, LINT_LEVEL - from ..wrap.wrap import PackageDefinition - - # Copied from typeshed. Blarg that they don't expose this - class DataclassInstance(Protocol): - __dataclass_fields__: T.ClassVar[dict[str, dataclasses.Field[T.Any]]] _DI = T.TypeVar('_DI', bound='DataclassInstance') @@ -151,10 +148,10 @@ def _raw_to_dataclass(raw: T.Mapping[str, object], cls: T.Type[_DI], msg: str, """ new_dict = {} unexpected = set() - fields = {x.name for x in dataclasses.fields(cls)} raw_from_workspace = raw_from_workspace or {} ignored_fields = ignored_fields or [] inherit = raw.get('workspace', False) + typedict = dataclass_field_validators(cls) for orig_k, v in raw.items(): if orig_k == 'workspace': @@ -169,29 +166,43 @@ def _raw_to_dataclass(raw: T.Mapping[str, object], cls: T.Type[_DI], msg: str, # function in the case it wants to merge values. ws_v = raw_from_workspace.get(orig_k) k = fixup_meson_varname(orig_k) - if k not in fields: + if k not in typedict: if orig_k not in ignored_fields: unexpected.add(orig_k) continue if k in kwargs: - new_dict[k] = kwargs[k].convert(v, ws_v) + v = kwargs[k].convert(v, ws_v) else: - new_dict[k] = v if v is not None else ws_v + v = v if v is not None else ws_v + if not typedict[k](v): + # treat it as a Meson bug if the type is not TOML-native + if isinstance(v, (int, str, bool, list, dict)): + raise MesonException(f'unexpected type for key "{k}": "{type(v)}"') + else: + raise Exception(f'unexpected type for key "{k}": "{type(v)}"') + new_dict[k] = v if inherit: # Inherit any keys from the workspace that we don't have yet. for orig_k, ws_v in raw_from_workspace.items(): k = fixup_meson_varname(orig_k) - if k not in fields: + if k not in typedict: if orig_k not in ignored_fields: unexpected.add(orig_k) continue if k in new_dict: continue if k in kwargs: - new_dict[k] = kwargs[k].convert(None, ws_v) + v = kwargs[k].convert(None, ws_v) else: - new_dict[k] = ws_v + v = ws_v + if not typedict[k](v): + # treat it as a Meson bug if the type is not TOML-native + if isinstance(v, (int, str, bool, list, dict)): + raise MesonException(f'unexpected type for key "{k}": "{type(v)}"') + else: + raise MesonBugException(f'unexpected type for key "{k}": "{type(v)}"') + new_dict[k] = v # Finally, set default values. for k, convertor in kwargs.items(): diff --git a/mesonbuild/cargo/raw.py b/mesonbuild/cargo/raw.py index b683f06d82e2..1cfc73e75137 100644 --- a/mesonbuild/cargo/raw.py +++ b/mesonbuild/cargo/raw.py @@ -6,25 +6,23 @@ from __future__ import annotations import typing as T -from typing_extensions import Literal, TypedDict, Required +EDITION = T.Literal['2015', '2018', '2021'] +CRATE_TYPE = T.Literal['bin', 'lib', 'dylib', 'staticlib', 'cdylib', 'rlib', 'proc-macro'] +LINT_LEVEL = T.Literal['allow', 'deny', 'forbid', 'warn'] -EDITION = Literal['2015', '2018', '2021'] -CRATE_TYPE = Literal['bin', 'lib', 'dylib', 'staticlib', 'cdylib', 'rlib', 'proc-macro'] -LINT_LEVEL = Literal['allow', 'deny', 'forbid', 'warn'] - -class FromWorkspace(TypedDict): +class FromWorkspace(T.TypedDict, total=False): """An entry or section that is copied from the workspace.""" workspace: bool -Package = TypedDict( +Package = T.TypedDict( 'Package', { - 'name': Required[str], - 'version': Required[T.Union[FromWorkspace, str]], + 'name': T.Required[str], + 'version': T.Required[T.Union[FromWorkspace, str]], 'authors': T.Union[FromWorkspace, T.List[str]], 'edition': T.Union[FromWorkspace, EDITION], 'rust-version': T.Union[FromWorkspace, str], @@ -55,15 +53,15 @@ class FromWorkspace(TypedDict): ) """A description of the Package Dictionary.""" -class Badge(TypedDict): +class Badge(T.TypedDict, total=False): """An entry in the badge section.""" - status: Literal['actively-developed', 'passively-developed', 'as-is', 'experimental', 'deprecated', 'none'] + status: T.Literal['actively-developed', 'passively-developed', 'as-is', 'experimental', 'deprecated', 'none'] repository: str -Dependency = TypedDict( +Dependency = T.TypedDict( 'Dependency', { 'version': str, @@ -86,7 +84,7 @@ class Badge(TypedDict): """A Dependency entry, either a string or a Dependency Dict.""" -_BaseBuildTarget = TypedDict( +_BaseBuildTarget = T.TypedDict( '_BaseBuildTarget', { 'path': str, @@ -107,7 +105,7 @@ class Badge(TypedDict): class BuildTarget(_BaseBuildTarget, total=False): - name: Required[str] + name: T.Required[str] class LibTarget(_BaseBuildTarget, total=False): @@ -115,17 +113,17 @@ class LibTarget(_BaseBuildTarget, total=False): name: str -class Target(TypedDict): +class Target(T.TypedDict, total=False): """Target entry in the Manifest File.""" dependencies: T.Dict[str, T.Union[FromWorkspace, DependencyV]] -Lint = TypedDict( +Lint = T.TypedDict( 'Lint', { - 'level': Required[LINT_LEVEL], + 'level': T.Required[LINT_LEVEL], 'priority': int, 'check-cfg': T.List[str], }, @@ -142,7 +140,7 @@ class Target(TypedDict): """A Lint entry, either a string or a Lint Dict.""" -class Workspace(TypedDict): +class Workspace(T.TypedDict, total=False): """The representation of a workspace. @@ -153,13 +151,14 @@ class Workspace(TypedDict): the :attribute:`exclude` is always optional """ + resolver: str members: T.List[str] exclude: T.List[str] package: Package dependencies: T.Dict[str, DependencyV] -Manifest = TypedDict( +Manifest = T.TypedDict( 'Manifest', { 'package': Package, @@ -185,7 +184,7 @@ class Workspace(TypedDict): """The Cargo Manifest format.""" -class CargoLockPackage(TypedDict, total=False): +class CargoLockPackage(T.TypedDict, total=False): """A description of a package in the Cargo.lock file format.""" @@ -195,7 +194,7 @@ class CargoLockPackage(TypedDict, total=False): checksum: str -class CargoLock(TypedDict, total=False): +class CargoLock(T.TypedDict, total=False): """A description of the Cargo.lock file format.""" diff --git a/mesonbuild/cargo/validate.py b/mesonbuild/cargo/validate.py new file mode 100644 index 000000000000..f0b0909f6d9e --- /dev/null +++ b/mesonbuild/cargo/validate.py @@ -0,0 +1,288 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations +import typing as T + +from dataclasses import InitVar, is_dataclass +from functools import lru_cache + +if T.TYPE_CHECKING: + from .._typing import DataclassInstance + +PartialValidator = T.Callable[['ValidatorResult', object], 'ValidatorResult'] +Validator = T.Callable[[object], 'ValidatorResult'] + +# All this is_* and extract_* magic is largely based on dacite +# Copyright (c) 2018 Konrad Hałas +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +def extract_origin_collection(collection: T.Type) -> T.Type: + try: + return T.cast('T.Type', collection.__extra__) + except AttributeError: + return T.cast('T.Type', collection.__origin__) + + +def is_generic(type_: T.Type) -> bool: + return hasattr(type_, "__origin__") + + +def is_union(type_: T.Type) -> bool: + if is_generic(type_) and type_.__origin__ == T.Union: + return True + + try: + from types import UnionType + + return isinstance(type_, UnionType) + except ImportError: + return False + + +def is_tuple(type_: T.Type) -> bool: + return is_subclass(type_, tuple) + + +def is_typed_dict(type_: T.Type) -> bool: + return hasattr(type_, "__orig_bases__") and T.TypedDict in type_.__orig_bases__ + + +def is_literal(type_: T.Type) -> bool: + return is_generic(type_) and type_.__origin__ == T.Literal + + +def is_required(type_: T.Type) -> bool: + return is_generic(type_) and type_.__origin__ == T.Required + + +def is_new_type(type_: T.Type) -> bool: + return hasattr(type_, "__supertype__") + + +def extract_new_type(type_: T.Type) -> T.Type: + return T.cast('T.Type', type_.__supertype__) + + +def is_init_var(type_: T.Type) -> bool: + return isinstance(type_, InitVar) or type_ is InitVar + + +def extract_init_var(type_: T.Type) -> T.Union[T.Type, T.Any]: + try: + return type_.type + except AttributeError: + return T.Any + + +def is_generic_collection(type_: T.Type) -> bool: + if not is_generic(type_): + return False + origin = extract_origin_collection(type_) + try: + return bool(origin and issubclass(origin, T.Collection)) + except (TypeError, AttributeError): + return False + + +def extract_generic(type_: T.Type, defaults: T.Tuple = ()) -> T.Tuple: + try: + if getattr(type_, "_special", False): + return defaults + if type_.__args__ == (): + return (type_.__args__,) + return type_.__args__ or defaults + except AttributeError: + return defaults + + +def is_subclass(sub_type: T.Type, base_type: T.Type) -> bool: + if is_generic_collection(sub_type): + sub_type = extract_origin_collection(sub_type) + try: + return issubclass(sub_type, base_type) + except TypeError: + return False + + +def is_type(type_: T.Type) -> bool: + try: + return type_.__origin__ in (type, T.Type) + except AttributeError: + return False + + +class ValidatorResult: + SUCCESS: 'ValidatorResult' + CACHE: T.Dict[T.Type, PartialValidator] = {} + + path: T.List[T.Union[str, int]] + + def __init__(self) -> None: + self.path = [] + + def __bool__(self) -> bool: + return self is ValidatorResult.SUCCESS + + def success(self, obj: object) -> ValidatorResult: + return ValidatorResult.SUCCESS + + def failure(self, obj: object) -> ValidatorResult: + return self + + def union(self, validators: T.List[PartialValidator], obj: object) -> ValidatorResult: + for v in validators: + path_len = len(self.path) + if v(self, obj): + return ValidatorResult.SUCCESS + self.path[:] = self.path[:path_len] + return self + + def mapping(self, origin: T.Type[T.Mapping], kv: PartialValidator, + vv: T.Callable[[object], PartialValidator], + obj: object) -> ValidatorResult: + if not isinstance(obj, origin): + return self + for k, v in obj.items(): + self.path.append(k) + if not kv(self, k) or not vv(k)(self, v): + return self + self.path.pop() + return ValidatorResult.SUCCESS + + def sequence(self, origin: T.Type[T.Sequence], iv: PartialValidator, obj: object) -> ValidatorResult: + if not isinstance(obj, origin): + return self + for k, v in enumerate(obj): + self.path.append(k) + if not iv(self, v): + return self + self.path.pop() + return ValidatorResult.SUCCESS + + def tuple(self, origin: T.Type[T.Tuple], iv: T.List[PartialValidator], obj: object) -> ValidatorResult: + if not isinstance(obj, origin) or len(iv) != len(obj): + return self + it = iter(iv) + for k, v in enumerate(obj): + self.path.append(k) + if not next(it)(self, v): + return self + self.path.pop() + return ValidatorResult.SUCCESS + + @classmethod + def _typeddict(cls, type_: T.Type) -> PartialValidator: + required_keys = {k for k, v in T.get_type_hints(type_, include_extras=True).items() + if is_required(v)} + hints = T.get_type_hints(type_) + validators: T.Dict[object, PartialValidator] = \ + {k: ValidatorResult.get_validator(hint) for k, hint in hints.items()} + default = ValidatorResult.failure if type_.__total__ else ValidatorResult.success + return lambda v, obj: \ + v if not isinstance(obj, dict) else \ + v if any(k not in obj for k in required_keys) else \ + v.mapping(dict, ValidatorResult.success, + lambda k: validators.get(k, default), + obj) + + @classmethod + def get_validator(cls, type_: T.Type) -> PartialValidator: + result = cls.CACHE.get(type_, None) + if not result: + result = cls.get_validator_uncached(type_) + cls.CACHE[type_] = result + return result + + @classmethod + def get_validator_uncached(cls, type_: T.Type) -> PartialValidator: + if type_ == T.Any: + return ValidatorResult.success + + if is_union(type_): + validators = [cls.get_validator(t) for t in extract_generic(type_)] + return lambda v, obj: v.union(validators, obj) + + if is_typed_dict(type_): + return cls._typeddict(type_) + + if is_new_type(type_): + return cls.get_validator(extract_new_type(type_)) + + if is_literal(type_): + generic = extract_generic(type_) + return lambda v, obj: cls.SUCCESS if obj in generic else v + + if is_init_var(type_): + return cls.get_validator(extract_init_var(type_)) + + if is_generic_collection(type_): + origin = extract_origin_collection(type_) + generic = extract_generic(type_) + if not generic: + return lambda v, obj: cls.SUCCESS if isinstance(obj, origin) else v + + if issubclass(origin, T.Mapping): + key_type, val_type = extract_generic(type_, defaults=(T.Any, T.Any)) + kv = cls.get_validator(key_type) + vv = lambda k: cls.get_validator(val_type) + return lambda v, obj: v.mapping(origin, kv, vv, obj) + + elif is_tuple(type_): + if len(generic) == 1 and generic[0] == (): + return lambda v, obj: cls.SUCCESS if isinstance(obj, origin) and not obj else v + if len(generic) == 2 and generic[1] is ...: + iv = cls.get_validator(generic[0]) + return lambda v, obj: v.sequence(origin, iv, obj) + + validators = [cls.get_validator(t) for t in generic] + return lambda v, obj: v.tuple(origin, validators, obj) + + field_type = extract_generic(type_, defaults=(T.Any,))[0] + iv = cls.get_validator(field_type) + return lambda v, obj: v.sequence(origin, iv, obj) + + if is_type(type_): + generic = extract_generic(type_) + if not generic: + return lambda v, obj: cls.SUCCESS if isinstance(obj, type) else v + + return lambda v, obj: cls.SUCCESS if isinstance(obj, type) and issubclass(obj, generic[0]) else v + + if is_dataclass(type_): + return lambda v, obj: cls.SUCCESS if isinstance(obj, type_) else v + + if type_ is complex: + return lambda v, obj: cls.SUCCESS if isinstance(obj, (int, float, complex)) else v + elif type_ is float: + return lambda v, obj: cls.SUCCESS if isinstance(obj, (int, float)) else v + else: + return lambda v, obj: cls.SUCCESS if isinstance(obj, type_) else v + +ValidatorResult.SUCCESS = ValidatorResult() + +def validator(type_: T.Type) -> Validator: + v = ValidatorResult.get_validator(type_) + return lambda value: v(ValidatorResult(), value) + + +@lru_cache(maxsize=None) +def dataclass_field_validators(type_: T.Type[DataclassInstance]) -> T.Dict[str, Validator]: + hints = T.get_type_hints(type_) + return {k: validator(hint) for k, hint in hints.items()} diff --git a/unittests/cargotests.py b/unittests/cargotests.py index 360c0b604a80..18f32c3220cd 100644 --- a/unittests/cargotests.py +++ b/unittests/cargotests.py @@ -13,6 +13,7 @@ from mesonbuild.cargo.interpreter import load_cargo_lock from mesonbuild.cargo.manifest import Dependency, Lint, Manifest, Package, Workspace from mesonbuild.cargo.toml import load_toml +from mesonbuild.cargo.validate import validator from mesonbuild.cargo.version import api, cargo_parse, SemVer from mesonbuild.mesonlib import MesonException @@ -679,3 +680,93 @@ def test_cargo_toml_features(self) -> None: self.assertEqual(manifest.features['v1_42'], ['pango-sys/v1_42']) self.assertEqual(manifest.features['v1_44'], ['v1_42', 'pango-sys/v1_44']) self.assertEqual(manifest.features['default'], []) + + +class A: + pass + + +class TypedDictExample(T.TypedDict, total=False): + name: str + version: str + + +class TypedDictReq(T.TypedDict, total=False): + name: T.Required[str] + version: str + + +class TypedDictTotal(T.TypedDict): + name: T.Required[str] + version: str + + +class ValidatorTest(unittest.TestCase): + + def test_validator(self): + assert validator(bool)(True) + assert not validator(bool)('') + assert not validator(bool)('abc') + + assert validator(float)(1) + + assert validator(T.List)([]) + assert validator(T.List)(['abc']) + assert validator(T.List)(['abc', 'def']) + assert validator(T.List[str])(['abc', 'def']) + assert not validator(T.List[int])(['abc', 'def']) + assert validator(T.List[int])(['abc', 'def']).path == [0] + assert not validator(T.List[int])([123, 'def']) + assert validator(T.List[int])([123, 'def']).path == [1] + assert not validator(T.List[str])([123, 'def']) + assert validator(T.List[str])([123, 'def']).path == [0] + + assert validator(T.Optional[int])(123) + assert validator(T.Optional[int])(None) + assert not validator(int)(None) + + assert validator(T.Union[str, int])(123) + assert validator(T.Union[str, int])('abc') + assert not validator(T.Union[str, int])([]) + assert not validator(T.Union[str, int])(['abc']) + + assert validator(T.Dict[str, int])({'abc': 123}) + assert not validator(T.Dict[str, int])({'abc': 'abc'}) + + assert validator(T.Mapping[str, int])({'abc': 123}) + assert not validator(T.Mapping[str, int])({'abc': 'abc'}) + + assert not validator(T.Tuple[int])(123) + assert validator(T.Tuple[int])((123, )) + assert not validator(T.Tuple[int])((123, 456)) + assert validator(T.Tuple[int, ...])((123, 456)) + assert not validator(T.Tuple[int, ...])((123, 'abc')) + assert validator(T.List[int])([123, 'def']).path == [1] + assert validator(T.Tuple[int, str])((123, 'abc')) + + assert validator(T.Literal['abc', 'def'])('abc') + assert validator(T.Literal['abc', 'def'])('def') + assert not validator(T.Literal['abc', 'def'])('ghi') + assert not validator(T.Literal['abc', 'def'])(123) + + assert validator(T.Type)(A) + assert not validator(T.Type)(A()) + assert validator(T.Type[A])(A) + assert not validator(T.Type[A])(A()) + + assert validator(A)(A()) + assert not validator(A)(A) + + def test_typeddict_validator(self): + assert validator(TypedDictExample)({}) + assert validator(TypedDictExample)({'name': 'abc'}) + assert validator(TypedDictExample)({'name': 'abc', 'extra': 123}) + assert not validator(TypedDictExample)({'name': 123}) + + assert not validator(TypedDictReq)({}) + assert validator(TypedDictReq)({'name': 'abc'}) + assert validator(TypedDictReq)({'name': 'abc', 'extra': 123}) + + assert not validator(TypedDictTotal)({}) + assert validator(TypedDictTotal)({'name': 'abc'}) + assert not validator(TypedDictTotal)({'name': 'abc', 'extra': 123})