feat[test]: add xfail_strict, clean up setup.cfg (#3889)

this commit adds `xfail_strict` to pytest options, and cleans up some
code relating to test and lint setup:
- move pytest coverage-enabling options to CI *only*
- suppress annoying mypy notes
- add `--dist worksteal` to setup.cfg
- pin pytest-xdist due to a regression (pytest-dev/pytest-xdist#1034)

Author: charles-cooper

.github/workflows/pull-request.yaml

@@ -38,6 +38,7 @@ jobs:
           # venom: venom changes
           scopes: |
             ci
+            build
             docs
             test
             lang

.github/workflows/test.yml

@@ -136,13 +136,14 @@ jobs:
     - name: Run tests
       run: |
         pytest \
-          --dist worksteal \
           -m "not fuzzing" \
           --optimize ${{ matrix.opt-mode }} \
           --evm-version ${{ matrix.evm-version }} \
           ${{ matrix.debug && '--enable-compiler-debug-mode' || '' }} \
           ${{ matrix.memorymock && '--memorymock' || '' }} \
-          --showlocals -r aR \
+          --cov-branch \
+          --cov-report xml:coverage.xml \
+          --cov=vyper \
           tests/
 
     - name: Upload Coverage
@@ -192,7 +193,8 @@ jobs:
     # NOTE: if the tests get poorly distributed, run this and commit the resulting `.test_durations` file to the `vyper-test-durations` repo.
     # `pytest -m "fuzzing" --store-durations -r aR tests/`
     - name: Fetch test-durations
-      run: curl --location "https://raw.githubusercontent.com/vyperlang/vyper-test-durations/master/test_durations" -o .test_durations
+      run: |
+        curl --location "https://raw.githubusercontent.com/vyperlang/vyper-test-durations/master/test_durations" -o .test_durations
 
     - name: Run tests
       run: |
@@ -201,8 +203,10 @@ jobs:
           --splits 120 \
           --group ${{ matrix.group }} \
           --splitting-algorithm least_duration \
-          --dist worksteal \
-          -r aR tests/
+          --cov-branch \
+          --cov-report xml:coverage.xml \
+          --cov=vyper \
+          tests/
 
     - name: Upload Coverage
       uses: codecov/codecov-action@v4

Makefile

@@ -20,7 +20,12 @@ test:
 lint: mypy black flake8 isort
 
 mypy:
-	mypy --install-types --non-interactive --follow-imports=silent --ignore-missing-imports --implicit-optional -p vyper
+	mypy \
+		--disable-error-code "annotation-unchecked" \
+		--follow-imports=silent \
+		--ignore-missing-imports \
+		--implicit-optional \
+		-p vyper
 
 black:
 	black -C -t py311 vyper/ tests/ setup.py --force-exclude=vyper/version.py

quicktest.sh

@@ -4,6 +4,6 @@
 # ./quicktest.sh
 # ./quicktest.sh tests/.../mytest.py
 
-# run pytest but bail out on first error and suppress coverage.
+# run pytest but bail out on first error
 # useful for dev workflow
-pytest --dist worksteal -q --no-cov -s --instafail -x --disable-warnings "$@"
+pytest -q -s --instafail -x --disable-warnings "$@"

setup.cfg

@@ -25,15 +25,9 @@ line_length = 100
 
 [tool:pytest]
 addopts = -n auto
-	--cov-branch
-	--cov-report term
-	--cov-report html
-	--cov-report xml
-	--cov=vyper
+	--dist worksteal
 python_files = test_*.py
 testpaths = tests
+xfail_strict = true
 markers =
 	fuzzing: Run Hypothesis fuzz test suite (deselect with '-m "not fuzzing"')
-
-[tool:mypy]
-ignore_missing_imports = True

setup.py

@@ -11,7 +11,7 @@
         "pytest>=8.0,<9.0",
         "pytest-cov>=4.1,<5.0",
         "pytest-instafail>=0.4,<1.0",
-        "pytest-xdist>=3.5,<4.0",
+        "pytest-xdist>=3.0,<3.4",
         "pytest-split>=0.7.0,<1.0",
         "eth-tester[py-evm]>=0.10.0b4,<0.11",
         "eth_abi>=4.0.0,<5.0.0",