Get all tox envs passing

Author: Gallaecio

.github/workflows/ci.yml

@@ -40,7 +40,8 @@ jobs:
         include:
         - python-version: "3.9"
           toxenv: "mypy"
-        - toxenv: "twinecheck"
+        - toxenv: "pylint"
+        - toxenv: "twine"
         - toxenv: "docs"
     steps:
     - uses: actions/checkout@v4

CHANGES.rst

@@ -5,6 +5,9 @@ Release notes
 0.1.0 (unreleased)
 ==================
 
+-   Switched from being a `flake8 <https://flake8.pycqa.org/en/latest/>`_
+    plugin to being a standalone tool, to support rules on non-Python files.
+
 -   Dropped Python 3.8 support, added Python 3.9+ support.
 
 -   Requires Scrapy 2.0.1+.

scrapy_lint/__init__.py

@@ -8,7 +8,7 @@
 from .linter import Linter
 
 if TYPE_CHECKING:
-    from collections.abc import Generator
+    from collections.abc import Generator, Sequence
 
     from .issues import Issue
 
@@ -25,17 +25,18 @@ def get_parser() -> ArgumentParser:
     return parser
 
 
-def lint() -> Generator[Issue]:
+def lint(args: Sequence[str]) -> Generator[Issue]:
     parser = get_parser()
-    args = parser.parse_args()
-    linter = Linter.from_args(args)
+    parsed_args = parser.parse_args(args)
+    linter = Linter.from_args(parsed_args)
     yield from linter.lint()
 
 
-def main() -> None:
+def main(args: Sequence[str] | None = None) -> None:
+    args = args if args is not None else sys.argv[1:]
     try:
         found_issues = False
-        for issue in lint():
+        for issue in lint(args):
             found_issues = True
             print(issue)
     except Exception as e:  # pylint: disable=broad-exception-caught

scrapy_lint/context.py

@@ -4,38 +4,19 @@
 from configparser import ConfigParser
 from dataclasses import dataclass
 from functools import cached_property
-from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
 from packaging.version import Version
-from ruamel.yaml import YAML
-from ruamel.yaml.error import YAMLError
 
 from scrapy_lint.requirements import iter_requirement_lines
 
 if TYPE_CHECKING:
-    from ast import AST
     from collections.abc import Sequence
+    from pathlib import Path
 
     from packaging.requirements import Requirement
 
 
-@dataclass
-class Flake8File:
-    tree: AST | None
-    path: Path
-    lines: Sequence[str] | None = None
-
-    @classmethod
-    def from_params(
-        cls,
-        tree: AST | None,
-        file_path: str,
-        lines: Sequence[str] | None = None,
-    ):
-        return cls(tree, Path(file_path).resolve(), lines)
-
-
 @dataclass
 class Project:
     root: Path
@@ -50,44 +31,6 @@ def setting_module_import_paths(self) -> Sequence[str]:
             return ()
         return tuple(config["settings"].values())
 
-    @staticmethod
-    def find_requirements_file_path(
-        root: Path | None,
-        requirements_file_path: str | None,
-    ) -> Path | None:
-        if requirements_file_path:
-            return Path(requirements_file_path).resolve()
-        if not root:
-            return None
-
-        # Check scrapinghub.yml for requirements file
-        scrapinghub_file = root / "scrapinghub.yml"
-        yaml_parser = YAML(typ="safe")
-        if scrapinghub_file.exists():
-            try:
-                with scrapinghub_file.open() as f:
-                    data = yaml_parser.load(f)
-            except YAMLError:
-                pass
-            else:
-                try:
-                    requirements_file_name = data.get("requirements", {}).get(
-                        "file",
-                        "",
-                    )
-                except AttributeError:
-                    pass
-                else:
-                    scrapinghub_requirements_file = root / requirements_file_name
-                    if scrapinghub_requirements_file.exists():
-                        return scrapinghub_requirements_file.resolve()
-
-        # Fall back to requirements.txt
-        requirements_file = root / "requirements.txt"
-        if requirements_file.exists():
-            return requirements_file.resolve()
-        return None
-
     @cached_property
     def _requirements(self) -> dict[str, list[Requirement]]:
         content = self.requirements_text

scrapy_lint/finders/requirements.py

@@ -71,8 +71,12 @@ def __init__(self, context: Context):
 
     def lint(self, file: Path) -> Generator[Issue]:
         packages: set[str] = set()
+        try:
+            requirements_text = file.read_text(encoding="utf-8")
+        except UnicodeDecodeError:
+            return
         for line_number, name, requirement in iter_requirement_lines(
-            file.read_text(encoding="utf-8").splitlines(),
+            requirements_text.splitlines(),
         ):
             packages.add(name)
             if name not in PACKAGES:

scrapy_lint/finders/settings/__init__.py

@@ -257,8 +257,7 @@ def check_name(
 
     def check_update(self, node: keyword | Constant) -> Generator[Issue]:
         name = node.value if isinstance(node, Constant) else node.arg
-        assert isinstance(name, str)
-        if name not in SETTINGS:
+        if not isinstance(name, str) or name not in SETTINGS:
             return
         setting = SETTINGS[name]
         if setting.is_pre_crawler and not self.in_update_pre_crawler_settings:

scrapy_lint/finders/settings/values.py

@@ -271,13 +271,15 @@ def check_feed_fields(value: expr) -> Generator[Issue]:
         if value.value is not None:
             detail = f"{param!r} must be a list or a dict"
             yield Issue(INVALID_SETTING_VALUE, pos, detail)
-    elif isinstance(value, (List, Set, Tuple)):
+        return
+    if isinstance(value, (List, Set, Tuple)):
         for index, elt in enumerate(value.elts):
             if isinstance(elt, Constant) and not isinstance(elt.value, str):
                 detail = f"{param}[{index}] ({elt.value!r}) must be a string"
                 pos_elt = Pos.from_node(elt)
                 yield Issue(INVALID_SETTING_VALUE, pos_elt, detail)
-    elif not is_dict(value):
+        return
+    if not is_dict(value):
         return
     assert isinstance(value, (Call, Dict))
     for elt_key, elt_value in iter_dict(value):

scrapy_lint/issues.py

@@ -12,9 +12,6 @@ class Pos:
     line: int = 1
     column: int = 0
 
-    def __iter__(self):
-        return iter([self.line, self.column])
-
     @classmethod
     def from_node(cls, node, column: int | None = None, /) -> Pos:
         line = getattr(node, "lineno", 1)

scrapy_lint/linter.py

@@ -5,6 +5,9 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Protocol
 
+from ruamel.yaml import YAML
+from ruamel.yaml.error import YAMLError
+
 try:
     import tomllib  # type: ignore[import-not-found]
 except ImportError:  # Python < 3.11
@@ -128,20 +131,53 @@ def resolve_requirements_file(
         options: dict[str, Any],
     ) -> Path | None:
         requirements_file: Path | None
-        path_str = options.get("requirements_file", "requirements.txt")
-        requirements_file = Path(path_str).resolve()
-        if not requirements_file.exists():
-            return None
-        return requirements_file
+        path_str = options.get("requirements_file")
+        if path_str is not None:
+            requirements_file = Path(path_str).resolve()
+            if requirements_file.exists():
+                return requirements_file
+
+        # Check scrapinghub.yml for requirements file
+        scrapinghub_file = Path("scrapinghub.yml")
+        yaml_parser = YAML(typ="safe")
+        if scrapinghub_file.exists():
+            try:
+                with scrapinghub_file.open(encoding="utf-8") as f:
+                    data = yaml_parser.load(f)
+            except (UnicodeDecodeError, YAMLError):
+                pass
+            else:
+                try:
+                    requirements_file_name = data.get("requirements", {}).get(
+                        "file",
+                        "",
+                    )
+                except AttributeError:
+                    pass
+                else:
+                    if requirements_file_name and isinstance(
+                        requirements_file_name,
+                        str,
+                    ):
+                        scrapinghub_requirements_file = Path(requirements_file_name)
+                        if scrapinghub_requirements_file.exists():
+                            return scrapinghub_requirements_file.resolve()
+
+        # Fall back to requirements.txt
+        requirements_file = Path("requirements.txt")
+        if requirements_file.exists():
+            return requirements_file.resolve()
+
+        return None
 
     @classmethod
     def load_options(cls, root: Path) -> dict[str, Any]:
         pyproject_path = root / "pyproject.toml"
         if not pyproject_path.exists():
-            raise ValueError(f"{pyproject_path} does not exist.")
+            return {}
         try:
             pyproject = tomllib.loads(pyproject_path.read_text(encoding="utf-8"))
-        except tomllib.TOMLDecodeError as e:
+        except (tomllib.TOMLDecodeError, UnicodeDecodeError) as e:
             raise ValueError(f"Invalid pyproject.toml: {e}") from None
         return pyproject.get("tool", {}).get("scrapy-lint", {})
 
@@ -169,17 +205,20 @@ def resolve_files(
                     files.add(zyte_config_path)
                 if requirements_path and requirements_path.exists():
                     files.add(requirements_path)
-            for python_file_path in path.glob("*/**.py"):
-                if spec is None or not spec.match_file(python_file_path):
+            for python_file_path in path.glob("**/*.py"):
+                if spec is None or not spec.match_file(
+                    python_file_path.relative_to(root),
+                ):
                     files.add(python_file_path)
         return sorted(files)
 
     def lint(self) -> Generator[Issue]:
         for file in self.files:
+            absolute_file = file.resolve()
             for issue in self.lint_file(file):
                 if self.is_ignored(issue, file):
                     continue
-                issue.file = file.relative_to(self.root)
+                issue.file = absolute_file.relative_to(self.root)
                 yield issue
 
     def is_ignored(self, issue: Issue, file: Path) -> bool:
@@ -192,15 +231,17 @@ def lint_file(self, file: Path) -> Generator[Issue]:
             yield from self.lint_python_file(file)
         elif file.name == "scrapinghub.yml":
             yield from ZyteCloudConfigIssueFinder(self.context).lint(file)
-        elif file == self.requirements_file:
+        elif self.requirements_file is not None and file == self.requirements_file:
             yield from RequirementsIssueFinder(self.context).lint(file)
 
     def lint_python_file(self, file: Path) -> Generator[Issue]:
         try:
             with file.open("r", encoding="utf-8") as f:
                 source = f.read()
         except UnicodeDecodeError as e:
-            raise ValueError(f"Could not read {file}: {e}") from None
+            raise ValueError(
+                f"Could not read {file.relative_to(self.root)}: {e}",
+            ) from None
         tree = ast.parse(source, filename=str(file))
         setting_module_finder = SettingModuleIssueFinder(
             self.context,

tests/__init__.py

@@ -6,9 +6,11 @@
 from contextlib import contextmanager
 from dataclasses import dataclass
 from pathlib import Path
+from tempfile import TemporaryDirectory
 from typing import TYPE_CHECKING, Any, Callable, Union
 
 import pytest
+import tomli_w
 
 if sys.version_info >= (3, 10):
     from typing import TypeAlias
@@ -81,7 +83,7 @@ def chdir(path: str | Path):
 def cases(test_cases: Cases) -> Callable:
     def decorator(func):
         return pytest.mark.parametrize(
-            ("input", "expected", "flake8_options"),
+            ("files", "expected", "options"),
             test_cases,
             ids=range(len(test_cases)),
         )(func)
@@ -98,3 +100,30 @@ def iter_issues(
         yield issues
         return
     yield from issues
+
+
+@contextmanager
+def project(
+    files: File | Sequence[File] | None = None,
+    options: dict | None = None,
+):
+    if isinstance(files, File):
+        files = [files]
+    elif files is None:
+        files = []
+    with TemporaryDirectory() as directory:
+        for file in files:
+            assert file.path
+            file_path = Path(directory) / file.path
+            file_path.parent.mkdir(parents=True, exist_ok=True)
+            if isinstance(file.text, str):
+                file_path.write_text(file.text)
+            else:
+                file_path.write_bytes(file.text)
+        if options:
+            options_path = Path(directory) / "pyproject.toml"
+            toml_dict = {"tool": {"scrapy-lint": options}}
+            with options_path.open("wb") as f:
+                f.write(tomli_w.dumps(toml_dict).encode("utf-8"))
+        with chdir(directory):
+            yield directory