CI Speed Test

.appveyor.yml

@@ -1,12 +1,12 @@
 environment:
-  matrix:
+  matrix: []
     # Unit and integration tests.
-    - PYTHON: "C:\\Python27-x64"
-      RUN_INTEGRATION_TESTS: "True"
-    - PYTHON: "C:\\Python35-x64"
-      RUN_INTEGRATION_TESTS: "True"
-    - PYTHON: "C:\\Python36-x64"
-      RUN_INTEGRATION_TESTS: "True"
+    #- PYTHON: "C:\\Python27-x64"
+    #  RUN_INTEGRATION_TESTS: "True"
+    #- PYTHON: "C:\\Python35-x64"
+    #  RUN_INTEGRATION_TESTS: "True"
+    #- PYTHON: "C:\\Python36-x64"
+    #  RUN_INTEGRATION_TESTS: "True"
     # Unit tests only.
     # Nothing for the moment
 

.azure-pipelines/linux.yml

@@ -3,6 +3,6 @@ jobs:
   parameters:
     vmImage: ubuntu-16.04
 
-- template: jobs/package.yml
-  parameters:
-    vmImage: ubuntu-16.04
+#- template: jobs/package.yml
+#  parameters:
+#    vmImage: ubuntu-16.04

.azure-pipelines/macos.yml

@@ -3,6 +3,6 @@ jobs:
   parameters:
     vmImage: xcode9-macos10.13
 
-- template: jobs/package.yml
-  parameters:
-    vmImage: xcode9-macos10.13
+#- template: jobs/package.yml
+#  parameters:
+#    vmImage: xcode9-macos10.13

.azure-pipelines/steps/run-tests-windows.yml

@@ -8,12 +8,50 @@ steps:
     versionSpec: '$(python.version)'
     architecture: '$(python.architecture)'
 
+- powershell: |
+    Invoke-WebRequest -Uri https://go.microsoft.com/fwlink/?linkid=2026036 -OutFile adksetup.exe
+    ./adksetup.exe /features OptionId.WindowsPerformanceToolkit /log adk-setup.log /ceip off /installpath "C:\Program Files (x86)\Windows Kits\10\"
+    # Wait for install to complete.
+    python -c "
+    import os, time
+    start = time.time()
+    last = start
+    while last - start < 60:
+     if os.path.exists('C:/Program Files (x86)/Windows Kits/10/Windows Performance Toolkit/wpr.exe'):
+      break
+     print('Waiting...')
+     time.sleep(2)
+     last = time.time()
+    "
+    Get-Content adk-setup.log
+  displayName: ADK Setup
+
 - bash: pip install --upgrade setuptools tox
   displayName: Install Tox
 
-- script: tox -e py -- -m unit -n 3 --junit-xml=junit/unit-test.xml
+- script: mkdir traces
+  displayName: Make trace output directory
+
+- script:
+    tox -e py -- -m unit --junit-xml=junit/unit-test.xml
+    --use-wpr
+    --wpr-path="C:\Program Files (x86)\Windows Kits\10\Windows Performance Toolkit\wpr.exe"
+    --wpr-profile=GeneralProfile.verbose
+    --wpr-profile=CPU.verbose
+    --wpr-output=traces/wpr-result.etl
   displayName: Tox run unit tests
 
+- script:
+    tracerpt -l traces/wpr-result.etl -export traces/providers.man
+  displayName: Generate trace provider manifest
+
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish trace results'
+  inputs:
+    pathtoPublish: traces
+    artifactName: wpr-result-$(python.version)-$(python.architecture).etl
+  condition: succeededOrFailed()
+
 - ${{ if eq(parameters.runIntegrationTests, 'true') }}:
   - powershell: |
       # Fix Git SSL errors

.azure-pipelines/windows.yml

@@ -3,6 +3,6 @@ jobs:
   parameters:
     vmImage: vs2017-win2016
 
-- template: jobs/package.yml
-  parameters:
-    vmImage: vs2017-win2016
+#- template: jobs/package.yml
+#  parameters:
+#    vmImage: vs2017-win2016

.travis.yml

@@ -12,45 +12,45 @@ stages:
 - secondary
 
 jobs:
-  include:
-    # Basic Checks
-    - stage: primary
-      env: TOXENV=docs
-    - env: TOXENV=lint
-    - env: TOXENV=vendoring
-    # Latest CPython
-    - env: GROUP=1
-      python: 2.7
-    - env: GROUP=2
-      python: 2.7
-    - env: GROUP=1
-    - env: GROUP=2
+  include: []
+    ## Basic Checks
+    #- stage: primary
+    #  env: TOXENV=docs
+    #- env: TOXENV=lint
+    #- env: TOXENV=vendoring
+    ## Latest CPython
+    #- env: GROUP=1
+    #  python: 2.7
+    #- env: GROUP=2
+    #  python: 2.7
+    #- env: GROUP=1
+    #- env: GROUP=2
 
-    # Complete checking for ensuring compatibility
-    # PyPy
-    - stage: secondary
-      env: GROUP=1
-      python: pypy3.5-6.0
-    - env: GROUP=2
-      python: pypy3.5-6.0
-    - env: GROUP=1
-      python: pypy2.7-6.0
-    - env: GROUP=2
-      python: pypy2.7-6.0
-    # Other Supported CPython
-    - env: GROUP=1
-      python: 3.6
-    - env: GROUP=2
-      python: 3.6
-    - env: GROUP=1
-      python: 3.5
-    - env: GROUP=2
-      python: 3.5
+    ## Complete checking for ensuring compatibility
+    ## PyPy
+    #- stage: secondary
+    #  env: GROUP=1
+    #  python: pypy3.5-6.0
+    #- env: GROUP=2
+    #  python: pypy3.5-6.0
+    #- env: GROUP=1
+    #  python: pypy2.7-6.0
+    #- env: GROUP=2
+    #  python: pypy2.7-6.0
+    ## Other Supported CPython
+    #- env: GROUP=1
+    #  python: 3.6
+    #- env: GROUP=2
+    #  python: 3.6
+    #- env: GROUP=1
+    #  python: 3.5
+    #- env: GROUP=2
+    #  python: 3.5
 
-    - env: GROUP=1
-      python: 3.8-dev
-    - env: GROUP=2
-      python: 3.8-dev
+    #- env: GROUP=1
+    #  python: 3.8-dev
+    #- env: GROUP=2
+    #  python: 3.8-dev
 
   fast_finish: true
   # It's okay to fail on the in-development CPython version.

tests/conftest.py

@@ -18,6 +18,12 @@
 from tests.lib.venv import VirtualEnvironment
 
 
+sys.path.append(os.path.dirname(os.path.dirname(__file__)))
+
+
+pytest_plugins = "tests.plugins.wpr"
+
+
 def pytest_addoption(parser):
     parser.addoption(
         "--keep-tmpdir", action="store_true",

tests/plugins/wpr.py

@@ -0,0 +1,105 @@
+import os
+import re
+import subprocess
+import sys
+from itertools import chain, repeat
+
+
+def pytest_addoption(parser):
+    group = parser.getgroup('wpr')
+    group.addoption(
+        '--use-wpr',
+        action='store_true',
+    )
+    group.addoption(
+        '--wpr-path',
+        default=None,
+    )
+    group.addoption(
+        '--wpr-output',
+        default=None,
+    )
+    group.addoption(
+        '--wpr-profile',
+        action='append',
+    )
+
+
+class Plugin(object):
+    def __init__(self, config):
+        self.config = config
+
+    def pytest_runtest_logstart(self, nodeid, location):
+        wpr_path = self.config.getoption('--wpr-path')
+        if not wpr_path:
+            return
+        msg = "{} ({}) - begin".format(nodeid, os.getpid())
+        create_mark(wpr_path, msg)
+
+    def pytest_runtest_logfinish(self, nodeid, location):
+        wpr_path = self.config.getoption('--wpr-path')
+        if not wpr_path:
+            return
+        msg = "{} ({}) - end".format(nodeid, os.getpid())
+        create_mark(wpr_path, msg)
+
+
+def pytest_configure(config):
+    if sys.platform != 'win32':
+        return
+
+    if not config.getoption('--use-wpr'):
+        return
+
+    config.pluginmanager.register(Plugin(config))
+
+    wpr_path = config.getoption('--wpr-path')
+    wpr_profiles = config.getoption('--wpr-profile')
+
+    start_wpr(wpr_path, wpr_profiles)
+
+
+def pytest_unconfigure(config):
+    if sys.platform != 'win32':
+        return
+
+    if not config.getoption('--use-wpr'):
+        return
+
+    wpr_path = config.getoption('--wpr-path')
+    wpr_output = config.getoption('--wpr-output')
+
+    stop_wpr(wpr_path, wpr_output)
+
+
+_name_re = re.compile(r"(?P<file>.+?)::(?P<name>.+?) \(.*\)$")
+
+
+def current_test_name():
+    try:
+        name = os.environ["PYTEST_CURRENT_TEST"]
+    except KeyError:
+        return "<outside test>"
+    m = _name_re.match(name)
+    if not m:
+        raise RuntimeError(
+            "Could not extract test name from {}".format(name)
+        )
+    return m.group("name")
+
+
+def start_wpr(executable, profiles):
+    # type: (str, List[str]) -> None
+    args = [executable]
+    args.extend(chain.from_iterable(zip(repeat('-start'), profiles)))
+    subprocess.check_call(args)
+
+
+def stop_wpr(executable, output_path):
+    # type: (str, str) -> None
+    subprocess.check_call([executable, '-stop', output_path])
+
+
+def create_mark(executable, message):
+    # type: (str, str) -> None
+    subprocess.check_call([executable, '-marker', message])

tools/requirements/tests.txt

@@ -3,6 +3,7 @@ mock
 pretend
 pytest==3.8.2
 pytest-cov
+pytest-custom_exit_code
 # Prevent installing 7.0 which has install_requires "pytest >= 3.10".
 pytest-rerunfailures<7.0
 pytest-timeout

tox.ini

@@ -20,7 +20,7 @@ deps = -r{toxinidir}/tools/requirements/tests.txt
 commands_pre =
 	python -c 'import shutil, sys; shutil.rmtree(sys.argv[1], ignore_errors=True)' {toxinidir}/tests/data/common_wheels
 	{[helpers]pip} wheel -w {toxinidir}/tests/data/common_wheels -r {toxinidir}/tools/requirements/tests-common_wheels.txt
-commands = pytest --timeout 300 []
+commands = pytest --timeout 300 --suppress-no-test-exit-code [] -k test_build_env_allow_only_one_install tests/unit/test_build_env.py
 install_command = {[helpers]pip} install {opts} {packages}
 list_dependencies_command = {[helpers]pip} freeze --all