Use pytest everywhere

.github/workflows/matrix-tests.yml

@@ -34,7 +34,7 @@ jobs:
           pip install -e .[dev]
       - run: pip freeze
       - run: pip list
-      - run: cd everyvoice && python run_tests.py dev
+      - run: pytest
 
   matrix-macos:
     timeout-minutes: 30
@@ -65,4 +65,4 @@ jobs:
       - run: python --version
       - run: pip freeze
       - run: pip list
-      - run: cd everyvoice && python run_tests.py dev
+      - run: pytest

.github/workflows/test.yml

@@ -27,6 +27,11 @@ jobs:
       - name: Verify SoX installation
         run: sox --version
       - uses: FedericoCarboni/setup-ffmpeg@37062fbf7149fc5578d6c57e08aed62458b375d6 # @v3.1, with tool cache
+        id: ffmpeg
+        continue-on-error: true
+      - name: Alternate way to install ffmpeg if need be
+        if: steps.ffmpeg.outcome == 'failure'
+        run: sudo apt-get update && sudo apt-get install ffmpeg
       - uses: actions/setup-python@v6
         with:
           python-version: "3.10"
@@ -38,21 +43,14 @@ jobs:
       - run: pip freeze
       - run: pip list
       - name: Run tests
-        id: tests
-        continue-on-error: true
         run: |
-          cd everyvoice && coverage run run_tests.py dev
+          cd everyvoice
+          coverage run -m pytest
           coverage xml
-      - name: Verbose test outputs in case of failure
-        if: steps.tests.outcome == 'failure'
-        run: |
-          cd everyvoice && EVERYVOICE_VERBOSE_TESTS=1 python run_tests.py --verbose dev
-          false
       - run: cd everyvoice && coverage report
       - name: Check for logs_and_checkpoints/ and preprocessed/
         id: no-extra-directories
         run: |
-          cd everyvoice
           find -type d -name logs_and_checkpoints -or -name preprocessed | grep -v 'tests/data' || true
           [[ $(find -type d -name logs_and_checkpoints -or -name preprocessed | grep --count --invert-match 'tests/data') -eq 0 ]]
       - uses: codecov/codecov-action@v5
@@ -123,8 +121,7 @@ jobs:
       - run: pip freeze
       - run: pip list
       - name: Run tests
-        id: tests
-        run: everyvoice test
+        run: pytest
 
   # Run pre-commit and update a PR automatically if possible
   pre-commit:

README.md

@@ -149,9 +149,9 @@ Many thanks to:
 ## Tests
 
 There are many ways to run the unit tests, if you installed EveryVoice from source:
- - Run all the tests with the most concise output: `pytest`
+ - Run all the tests: `pytest`; add `-v` to list tests cases, `-s` to see all logs
  - Run all the dev tests: `everyvoice/run_tests.py dev` or `everyvoice test dev`
  - Run the tests with verbose logs: `everyvoice/run_tests.py --verbose dev`
  - Show the names of the other suites you can run: `everyvoice/run_tests.py -h`
- - Run all the tests in one test file: `python -m unittest everyvoice/tests/test_<somefilename>.py`
- - Run one specific test case: `python -m unittest everyvoice.tests.<filename>.<class_name>.<function_name>`
+ - Run all the tests in one test file: `pytest everyvoice/tests/test_<somefilename>.py`
+ - Run one specific test case: `pytest everyvoice/tests/<filename>.py::<class_name>::<function_name>`

everyvoice/cli.py

@@ -634,9 +634,10 @@ def test(suite: TestSuites = typer.Argument("dev")):  # pragma: no cover
         run_tests(suite.value)
     except ModuleNotFoundError:
         print(
-            "ERROR: hidden command 'everyvoice test' only works when you install EveryVoice from source.",
+            "ERROR: hidden command 'everyvoice test' only works when you install EveryVoice from source, with dev dependencies.",
             file=sys.stderr,
         )
+        sys.exit(1)
 
 
 # Deferred full initialization to optimize the CLI, but still exposed for unit testing.

everyvoice/run_tests.py

@@ -3,13 +3,13 @@
 """Organize tests into Test Suites"""
 
 import argparse
-import importlib
-import re
+import io
 import sys
 from collections.abc import Iterable
-from os.path import dirname
-from unittest import TestLoader, TestSuite, TextTestRunner
+from contextlib import redirect_stdout
+from pathlib import Path
 
+import pytest
 from loguru import logger
 
 # Unit tests
@@ -19,6 +19,7 @@
     "wav2vec2aligner": ("/model/aligner/wav2vec2aligner/aligner/tests",),
 }
 SUITES: dict[str, tuple[str, ...]] = {
+    "all": (),  # relies on discovery for collection
     "config": ("test_configs",),
     "loader": ("test_dataloader",),
     "text": ("test_text", "test_utils", "test_doctests"),
@@ -49,96 +50,75 @@
 SUITES["dev"] = sum((SUITES[suite] for suite in dev_suites), start=())
 
 
-def remove_test_prefix(test_case: str):
-    for prefix in "<", "everyvoice.", "tests.":
-        if test_case.startswith(prefix):
-            test_case = test_case[len(prefix) :]
-    return "<" + test_case
+class PytestCollectorPlugin:
+    def __init__(self):
+        self.collected = []
 
+    def pytest_collection_modifyitems(self, session, config, items):
+        self.collected.extend([item.nodeid for item in items])
 
-def list_tests(suite: TestSuite):
-    for subsuite in suite:
-        # print(str(subsuite))
-        for match in re.finditer(r"tests=\[([^][]+)\]>", str(subsuite)):
-            for test_case in match[1].split(", "):
-                yield remove_test_prefix(test_case)
 
+def list_tests(suite: Iterable[str]):
+    plugin = PytestCollectorPlugin()
+    pytest_args = ["--collect-only", *suite, "-q"]
+    with redirect_stdout(io.StringIO()):
+        pytest.main(pytest_args, plugins=[plugin])
+    return plugin.collected
 
-def all_test_suites() -> TestSuite:
-    loader = TestLoader()
-    # NOTE: Looking specifically under `/tests` removes empty TestSuites.
-    test_suite = loader.discover(
-        dirname(__file__) + "/tests",
-        top_level_dir=dirname(dirname(__file__)),
-    )
-    for submodule_testsuite in SUBMODULE_SUITES.values():
-        suite = loader.discover(
-            dirname(__file__) + submodule_testsuite[0],
-            top_level_dir=dirname(dirname(__file__)),  # MANDATORY
-        )
-        test_suite.addTests(suite)
-
-    return test_suite
 
-
-def describe_suite(suite: TestSuite):
-    full_suite = all_test_suites()
-    full_list = list(list_tests(full_suite))
-    requested_list = list(list_tests(suite))
+def describe_suite(suite_name, suite_filenames: Iterable[str]):
+    full_list = list_tests([])
+    requested_list = list_tests(suite_filenames)
     requested_set = set(requested_list)
-    print("Test suite includes:", *sorted(requested_list), sep="\n")
+    print(f"Test suite '{suite_name}' includes:", *sorted(requested_list), sep="\n")
     print(
-        "\nTest suite excludes:",
+        f"\nTest suite '{suite_name}' excludes:",
         *sorted(test for test in full_list if test not in requested_set),
         sep="\n",
     )
+    print(
+        "\nTotal test cases",
+        f"found: {len(full_list)};",
+        f"included: {len(requested_list)};",
+        f"excluded: {len(full_list) - len(requested_list)}.",
+    )
 
 
-def run_tests(suite: str, describe: bool = False, verbosity=3):
-    """Decide which Test Suite to run"""
+def run_tests(suite: str, describe=False, verbose=False, no_capture=False):
+    """Run the specified test suite."""
     logger.info(f"Loading test suite '{suite}'. This may take a while...")
-    if suite == "all":
-        test_suite = all_test_suites()
-    else:
-        loader = TestLoader()
-        tests: Iterable[str]
-        if suite in SUITES:
-            tests = SUITES[suite]
-        else:
-            logger.error(
-                f"Please specify a test suite to run: one of '{['all'] + SUITE_NAMES}'."
-            )
-            return False
-        tests = [
-            "everyvoice.tests." + test if not test.startswith("/") else test
-            for test in tests
-        ]
-        test_suite = TestSuite()
-        for test in tests:
-            logger.info(f"Loading {test=}")
-            if test.startswith("/"):
-                sub_suite = loader.discover(
-                    dirname(__file__) + test,
-                    top_level_dir=dirname(dirname(__file__)),  # MANDATORY
-                )
-                test_suite.addTests(sub_suite)
-            else:
-                importlib.import_module(test)
-                test_suite.addTest(loader.loadTestsFromName(test))
-
+    if suite not in SUITES:
+        logger.error(f"Please specify a test suite to run: one of '{SUITE_NAMES}'.")
+        return False
+
+    test_suite = SUITES[suite]
+    root_dir = Path(__file__).parent
+    test_suite_filenames: list[str] = [
+        str(
+            root_dir / test_file[1:]
+            if test_file.startswith("/")
+            else root_dir / "tests" / f"{test_file}.py"
+        )
+        for test_file in test_suite
+    ]
+    # print(test_suite_filenames)
     if describe:
-        describe_suite(test_suite)
+        describe_suite(suite, test_suite_filenames)
         return True
     else:
-        logger.info("Running test suite")
-        return TextTestRunner(verbosity=verbosity).run(test_suite).wasSuccessful()
+        pytest_args = ["--verbose"] if verbose else []
+        if no_capture:
+            pytest_args.append("--capture=no")
+        return 0 == pytest.main([*test_suite_filenames, *pytest_args])
 
 
-def main():
+def main() -> None:
     parser = argparse.ArgumentParser(description="Run EveryVoice test suites.")
-    parser.add_argument("--quiet", "-q", action="store_true", help="reduce output")
     parser.add_argument(
-        "--verbose", "-v", action="store_true", help="let stderr logs go to screen"
+        "--no-capture", "-s", action="store_true", help="let all logs go to screen"
+    )
+    parser.add_argument(
+        "--verbose", "-v", action="store_true", help="show test names as they run"
     )
     parser.add_argument(
         "--describe", action="store_true", help="describe the selected test suite"
@@ -151,12 +131,12 @@ def main():
         choices=SUITE_NAMES,
     )
     args = parser.parse_args()
-    if args.verbose:
+    if args.no_capture:
         import everyvoice.tests.stubs as stubs
 
         stubs.VERBOSE_OVERRIDE = True
 
-    result = run_tests(args.suite, args.describe, 1 if args.quiet else 3)
+    result = run_tests(args.suite, args.describe, args.verbose, args.no_capture)
     if not result:
         sys.exit(1)
 

everyvoice/tests/model_stubs.py

@@ -13,7 +13,6 @@
 )
 from everyvoice.model.vocoder.HiFiGAN_iSTFT_lightning.hfgl.config import HiFiGANConfig
 from everyvoice.model.vocoder.HiFiGAN_iSTFT_lightning.hfgl.utils import HiFiGAN
-from everyvoice.tests.stubs import silence_c_stderr
 
 
 def get_stubbed_vocoder(tmp_dir: Path) -> tuple[HiFiGAN, Path]:
@@ -32,8 +31,7 @@ def get_stubbed_vocoder(tmp_dir: Path) -> tuple[HiFiGAN, Path]:
         contact_name="Test Runner", contact_email="info@everyvoice.ca"
     )
     vocoder = HiFiGAN(HiFiGANConfig(contact=contact_info))
-    with silence_c_stderr():
-        trainer = Trainer(default_root_dir=str(tmp_dir), barebones=True)
+    trainer = Trainer(default_root_dir=str(tmp_dir), barebones=True)
     trainer.strategy.connect(vocoder)
     vocoder_path = tmp_dir / "vocoder"
     trainer.save_checkpoint(vocoder_path)
@@ -59,8 +57,7 @@ def get_stubbed_model(tmp_dir: Path) -> tuple[FastSpeech2, Path]:
             ),
         ),
     )
-    with silence_c_stderr():
-        trainer = Trainer(default_root_dir=str(tmp_dir), barebones=True)
+    trainer = Trainer(default_root_dir=str(tmp_dir), barebones=True)
     trainer.strategy.connect(model)
     model_path = tmp_dir / "model"
     trainer.save_checkpoint(model_path)

everyvoice/tests/preprocessed_audio_fixture.py

@@ -67,6 +67,10 @@ class PreprocessedAudioFixture:
 
     preprocessor = Preprocessor(fp_config)
 
+    # def setUp(self):
+    #    """Each test function should get a fresh preprocessor"""
+    #    self.preprocessor = Preprocessor(self.fp_config)
+
     @classmethod
     def setUpClass(cls):
         """Generate a preprocessed test set that can be used in various test cases."""

everyvoice/tests/regression/test-demo-app-lj-full.py

@@ -7,9 +7,11 @@
  - python test_demo_app_mix.py
 """
 
-from unittest import TestCase, main
+import sys
+from unittest import TestCase
 
 from playwright.sync_api import expect, sync_playwright
+from pytest import main
 
 
 class TestDemoAppErros(TestCase):
@@ -50,4 +52,4 @@ def test_rundemo(self) -> None:
 
 
 if __name__ == "__main__":
-    main()
+    main(sys.argv)

everyvoice/tests/regression/test-demo-app-mix.py

@@ -7,9 +7,11 @@
  - python test_demo_app_mix.py
 """
 
-from unittest import TestCase, main
+import sys
+from unittest import TestCase
 
 from playwright.sync_api import sync_playwright
+from pytest import main
 
 
 class TestDemoAppMix(TestCase):
@@ -85,4 +87,4 @@ def test_rundemo(self) -> None:
 
 
 if __name__ == "__main__":
-    main()
+    main(sys.argv)