Update test runner

- Support "skip_tests" and "uitest_auto_screenshots" in launch options.

- Improve simulator_test stability.

Author: albertdai

xctestrunner/shared/ios_constants.py

@@ -30,6 +30,9 @@ def enum(**enums):
 SUPPORTED_SIM_OSS = [OS.IOS]
 
 TEST_STARTED_SIGNAL = 'Test Suite'
+XCTRUNNER_STARTED_SIGNAL = 'Running tests...'
+
+CORESIMULATOR_INTERRUPTED_ERROR = 'CoreSimulatorService connection interrupted'
 
 LAUNCH_OPTIONS_JSON_HELP = (
     """The path of json file, which contains options of launching test.
@@ -55,6 +58,14 @@ def enum(**enums):
     The specific test classes or test methods to run. Each item should be
     string and its format is Test-Class-Name[/Test-Method-Name]. It is supported
     in Xcode 8+.
+  skip_tests: array
+    The specific test classes or test methods to skip. Each item should be
+    string and its format is Test-Class-Name[/Test-Method-Name]. Logic test
+    does not support that.
+  uitest_auto_screenshots: bool
+    Whether captures screenshots automatically in ui test. If yes, will save the
+    screenshots when the test failed. By default, it is false. Prior Xcode 9,
+    this option does not work and the auto screenshot is enable by default.
   """)
 
 SIGNING_OPTIONS_JSON_HELP = (

xctestrunner/simulator_control/simulator_util.py

@@ -120,9 +120,7 @@ def Shutdown(self):
           'Can not shut down the simulator in state CREATING.')
     logging.info('Shutting down simulator %s.', self.simulator_id)
     try:
-      subprocess.check_output(
-          ['xcrun', 'simctl', 'shutdown', self.simulator_id],
-          stderr=subprocess.STDOUT)
+      _RunSimctlCommand(['xcrun', 'simctl', 'shutdown', self.simulator_id])
     except subprocess.CalledProcessError as e:
       if 'Unable to shutdown device in current state: Shutdown' in e.output:
         logging.info('Simulator %s has already shut down.', self.simulator_id)
@@ -147,7 +145,7 @@ def Delete(self):
           'Can only delete the simulator with state SHUTDOWN. The current '
           'state of simulator %s is %s.' % (self._simulator_id, sim_state))
     try:
-      subprocess.check_call(['xcrun', 'simctl', 'delete', self.simulator_id])
+      _RunSimctlCommand(['xcrun', 'simctl', 'delete', self.simulator_id])
     except subprocess.CalledProcessError as e:
       raise ios_errors.SimError(
           'Failed to delete simulator %s: %s' % (self.simulator_id, e.output))
@@ -185,9 +183,9 @@ def GetAppDocumentsPath(self, app_bundle_id):
     """Gets the path of the app's Documents directory."""
     if xcode_info_util.GetXcodeVersionNumber() >= 830:
       try:
-        app_data_container = subprocess.check_output(
+        app_data_container = _RunSimctlCommand(
             ['xcrun', 'simctl', 'get_app_container', self._simulator_id,
-             app_bundle_id, 'data']).strip()
+             app_bundle_id, 'data'])
         return os.path.join(app_data_container, 'Documents')
       except subprocess.CalledProcessError as e:
         raise ios_errors.SimError(
@@ -313,8 +311,8 @@ def CreateNewSimulator(device_type=None, os_version=None, name=None):
                name, os_type, os_version, device_type)
   for i in range(0, _SIM_OPERATION_MAX_ATTEMPTS):
     try:
-      new_simulator_id = subprocess.check_output(
-          ['xcrun', 'simctl', 'create', name, device_type, runtime_id]).strip()
+      new_simulator_id = _RunSimctlCommand(
+          ['xcrun', 'simctl', 'create', name, device_type, runtime_id])
     except subprocess.CalledProcessError as e:
       raise ios_errors.SimError(
           'Failed to create simulator: %s' % e.output)
@@ -371,8 +369,7 @@ def GetSupportedSimDeviceTypes(os_type=None):
   #
   # See more examples in testdata/simctl_list_devicetypes.json
   sim_types_infos_json = ast.literal_eval(
-      subprocess.check_output(
-          ('xcrun', 'simctl', 'list', 'devicetypes', '-j')))
+      _RunSimctlCommand(('xcrun', 'simctl', 'list', 'devicetypes', '-j')))
   sim_types = []
   for sim_types_info in sim_types_infos_json['devicetypes']:
     sim_type = sim_types_info['name']
@@ -438,8 +435,7 @@ def GetSupportedSimOsVersions(os_type=ios_constants.OS.IOS):
   #
   # See more examples in testdata/simctl_list_runtimes.json
   sim_runtime_infos_json = ast.literal_eval(
-      subprocess.check_output(
-          ('xcrun', 'simctl', 'list', 'runtimes', '-j')))
+      _RunSimctlCommand(('xcrun', 'simctl', 'list', 'runtimes', '-j')))
   sim_versions = []
   for sim_runtime_info in sim_runtime_infos_json['runtimes']:
     # Normally, the json does not contain unavailable runtimes. To be safe,
@@ -610,3 +606,14 @@ def IsXctestFailedToLaunchOnSim(sim_sys_log):
   """
   pattern = re.compile(_PATTERN_XCTEST_PROCESS_CRASH_ON_SIM)
   return pattern.search(sim_sys_log) is not None
+
+
+def _RunSimctlCommand(command):
+  """Runs simctl command."""
+  for i in range(2):
+    try:
+      return subprocess.check_output(command, stderr=subprocess.STDOUT).strip()
+    except subprocess.CalledProcessError as e:
+      if i == 0 and ios_constants.CORESIMULATOR_INTERRUPTED_ERROR in e.output:
+        continue
+      raise e

xctestrunner/test_runner/TestProject/TestProject.xcodeproj/xcshareddata/xcschemes/TestProjectXcuitest.xcscheme

@@ -27,6 +27,7 @@
       selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
       selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
       shouldUseLaunchSchemeArgsEnv = "YES"
+      systemAttachmentLifetime = "keepNever"
       disableMainThreadChecker = "YES">
       <Testables>
          <TestableReference

xctestrunner/test_runner/dummy_project.py

@@ -58,12 +58,12 @@ def __init__(self, app_under_test_dir, test_bundle_dir,
 
     Args:
       app_under_test_dir: string, path of the app to be tested in
-          dummy project.
+        dummy project.
       test_bundle_dir: string, path of the test bundle.
       sdk: string, SDKRoot of the dummy project. See supported SDKs in
-          module shared.ios_constants.
+        module shared.ios_constants.
       test_type: string, test type of the test bundle. See supported test types
-          in module shared.ios_constants.
+        in module shared.ios_constants.
       work_dir: string, work directory which contains run files.
     """
     self._app_under_test_dir = app_under_test_dir
@@ -411,7 +411,7 @@ def SetTestBundleProvisioningProfile(self, test_bundle_provisioning_profile):
 
     Args:
       test_bundle_provisioning_profile: string, name/path of the provisioning
-          profile of test bundle.
+        profile of test bundle.
     """
     if not test_bundle_provisioning_profile:
       return
@@ -488,6 +488,28 @@ def SetArgs(self, args):
       arg_element.set('isEnabled', 'YES')
     scheme_tree.write(scheme_path)
 
+  def SetSkipTests(self, skip_tests):
+    """Sets the skip tests in the dummy project's scheme.
+
+    Args:
+     skip_tests: a list of string. The format of each item is
+       Test-Class-Name[/Test-Method-Name].
+    """
+    if not skip_tests:
+      return
+    self.GenerateDummyProject()
+    scheme_path = self.test_scheme_path
+    scheme_tree = ET.parse(scheme_path)
+    test_action_element = scheme_tree.getroot().find('TestAction')
+    testable_reference_element = test_action_element.find(
+        'Testables').find('TestableReference')
+    skip_tests_element = ET.SubElement(
+        testable_reference_element, 'SkippedTests')
+    for skip_test in skip_tests:
+      skip_test_element = ET.SubElement(skip_tests_element, 'Test')
+      skip_test_element.set('Identifier', skip_test)
+    scheme_tree.write(scheme_path)
+
 
 def _GetTestProject(work_dir):
   """Gets the TestProject path."""

xctestrunner/test_runner/test_summaries_util.py

@@ -28,7 +28,9 @@ def GetTestSummariesPaths(derived_data_dir):
   return glob.glob('%s/Logs/Test/*_TestSummaries.plist' % derived_data_dir)
 
 
-def ParseTestSummaries(test_summaries_path, attachments_dir_path):
+def ParseTestSummaries(
+    test_summaries_path, attachments_dir_path,
+    delete_uitest_auto_screenshots=True):
   """Parse the TestSummaries.plist and structure the attachments' files.
 
   Only the screenshots file from failure test methods and .crash files will be
@@ -37,13 +39,17 @@ def ParseTestSummaries(test_summaries_path, attachments_dir_path):
   Args:
     test_summaries_path: string, the path of TestSummaries.plist file.
     attachments_dir_path: string, the path of Attachments directory.
+    delete_uitest_auto_screenshots: bool, whether deletes the auto screenshots.
   """
   test_summaries_plist = plist_util.Plist(test_summaries_path)
   tests_obj = test_summaries_plist.GetPlistField('TestableSummaries:0:Tests:0')
   # Store the required screenshots and crash files under temp directory first.
   # Then use the temp directory to replace the original Attachments directory.
+  # If delete_uitest_auto_screenshots is true, only move crash files to
+  # temp directory and the left screenshots will be deleted.
   temp_dir = tempfile.mkdtemp(dir=os.path.dirname(attachments_dir_path))
-  _ParseTestObject(tests_obj, attachments_dir_path, temp_dir)
+  if not delete_uitest_auto_screenshots:
+    _ParseTestObject(tests_obj, attachments_dir_path, temp_dir)
   for crash_file in glob.glob('%s/*.crash' % attachments_dir_path):
     shutil.move(crash_file, temp_dir)
   shutil.rmtree(attachments_dir_path)

xctestrunner/test_runner/xcodebuild_test_executor.py

@@ -17,6 +17,7 @@
 import io
 import logging
 import os
+import random
 import re
 import shutil
 import subprocess
@@ -32,7 +33,7 @@
 
 
 _XCODEBUILD_TEST_STARTUP_TIMEOUT_SEC = 150
-_SIM_TEST_MAX_ATTEMPTS = 2
+_SIM_TEST_MAX_ATTEMPTS = 3
 _TAIL_SIM_LOG_LINE = 200
 _BACKGROUND_TEST_RUNNER_ERROR = 'Failed to background test runner'
 _PROCESS_EXISTED_OR_CRASHED_ERROR = ('The process did launch, but has since '
@@ -41,6 +42,7 @@
                          '(SBMainWorkspace) for reason')
 _APP_UNKNOWN_TO_FRONTEND_PATTERN = re.compile(
     'Application ".*" is unknown to FrontBoard.')
+_INIT_SIM_SERVICE_ERROR = 'Failed to initiate service connection to simulator'
 
 
 class CheckXcodebuildStuckThread(threading.Thread):
@@ -142,9 +144,15 @@ def Execute(self, return_output=True):
       output = io.BytesIO()
       for stdout_line in iter(process.stdout.readline, ''):
         if not test_started:
+          # Terminates the CheckXcodebuildStuckThread when test has started
+          # or XCTRunner.app has started.
+          # But XCTRunner.app start does not mean test start.
           if ios_constants.TEST_STARTED_SIGNAL in stdout_line:
             test_started = True
             check_xcodebuild_stuck.Terminate()
+          if (self._test_type == ios_constants.TestType.XCUITEST and
+              ios_constants.XCTRUNNER_STARTED_SIGNAL in stdout_line):
+            check_xcodebuild_stuck.Terminate()
         else:
           if self._succeeded_signal and self._succeeded_signal in stdout_line:
             test_succeeded = True
@@ -180,7 +188,7 @@ def Execute(self, return_output=True):
 
           # The following error can be fixed by relaunching the test again.
           try:
-            if sim_log_path:
+            if sim_log_path and os.path.exists(sim_log_path):
               tail_sim_log = _ReadFileTailInShell(
                   sim_log_path, _TAIL_SIM_LOG_LINE)
               if (self._test_type == ios_constants.TestType.LOGIC_TEST and
@@ -190,6 +198,12 @@ def Execute(self, return_output=True):
                 raise ios_errors.SimError('')
             if _PROCESS_EXISTED_OR_CRASHED_ERROR in output_str:
               raise ios_errors.SimError('')
+            if ios_constants.CORESIMULATOR_INTERRUPTED_ERROR in output_str:
+              # Sleep random[0,2] seconds to avoid race condition. It is known
+              # issue that CoreSimulatorService connection will interrupte if
+              # two simulators booting at the same time.
+              time.sleep(random.uniform(0, 2))
+              raise ios_errors.SimError('')
           except ios_errors.SimError:
             if i < max_attempts - 1:
               logging.warning(
@@ -228,6 +242,8 @@ def _NeedRecreateSim(self, output_str):
       return True
     if _REQUEST_DENIED_ERROR in output_str:
       return True
+    if _INIT_SIM_SERVICE_ERROR in output_str:
+      return True
     return False
 
 

xctestrunner/test_runner/xctest_session.py

@@ -26,6 +26,7 @@
 from xctestrunner.shared import xcode_info_util
 from xctestrunner.test_runner import dummy_project
 from xctestrunner.test_runner import logic_test_util
+from xctestrunner.test_runner import runner_exit_codes
 from xctestrunner.test_runner import test_summaries_util
 from xctestrunner.test_runner import xctestrun
 
@@ -62,6 +63,8 @@ def __init__(self, sdk, work_dir=None, output_dir=None):
     self._logic_test_env_vars = None
     self._logic_test_args = None
     self._logic_tests_to_run = None
+    # The following fields are only for XCUITest.
+    self._disable_uitest_auto_screenshots = True
 
   def __enter__(self):
     return self
@@ -70,6 +73,8 @@ def __exit__(self, unused_type, unused_value, unused_traceback):
     """Deletes the temp directories."""
     self.Close()
 
+  # TODO(albertdai): Support bundle id as the value of app_under_test and
+  # test_bundle.
   def Prepare(self, app_under_test=None, test_bundle=None,
               xctestrun_file_path=None, test_type=None, signing_options=None):
     """Prepares the test session.
@@ -177,13 +182,24 @@ def SetLaunchOptions(self, launch_options):
       self._xctestrun_obj.SetTestEnvVars(launch_options.get('env_vars'))
       self._xctestrun_obj.SetTestArgs(launch_options.get('args'))
       self._xctestrun_obj.SetTestsToRun(launch_options.get('tests_to_run'))
+      self._xctestrun_obj.SetSkipTests(launch_options.get('skip_tests'))
       self._xctestrun_obj.SetAppUnderTestEnvVars(
           launch_options.get('app_under_test_env_vars'))
       self._xctestrun_obj.SetAppUnderTestArgs(
           launch_options.get('app_under_test_args'))
+
+      if launch_options.get('uitest_auto_screenshots'):
+        self._disable_uitest_auto_screenshots = False
+        # By default, this SystemAttachmentLifetime field is in the generated
+        # xctestrun.plist.
+        try:
+          self._xctestrun_obj.DeleteXctestrunField('SystemAttachmentLifetime')
+        except ios_errors.PlistError:
+          pass
     elif self._dummy_project_obj:
       self._dummy_project_obj.SetEnvVars(launch_options.get('env_vars'))
       self._dummy_project_obj.SetArgs(launch_options.get('args'))
+      self._dummy_project_obj.SetSkipTests(launch_options.get('skip_tests'))
     elif self._logic_test_bundle:
       self._logic_test_env_vars = launch_options.get('env_vars')
       self._logic_test_args = launch_options.get('args')
@@ -214,11 +230,12 @@ def RunTest(self, device_id):
         try:
           test_summaries_util.ParseTestSummaries(
               test_summaries_path,
-              os.path.join(self._output_dir, 'Logs/Test/Attachments'))
+              os.path.join(self._output_dir, 'Logs/Test/Attachments'),
+              True if self._disable_uitest_auto_screenshots else
+              exit_code == runner_exit_codes.EXITCODE.SUCCEEDED)
         except ios_errors.PlistError as e:
-          logging.warning(
-              'Failed to parse test summaries %s: %s',
-              test_summaries_path, e.message)
+          logging.warning('Failed to parse test summaries %s: %s',
+                          test_summaries_path, e.message)
       return exit_code
     elif self._dummy_project_obj:
       return self._dummy_project_obj.RunXcTest(

xctestrunner/test_runner/xctestrun.py

@@ -128,6 +128,17 @@ def SetTestsToRun(self, tests_to_run):
       return
     self.SetXctestrunField('OnlyTestIdentifiers', tests_to_run)
 
+  def SetSkipTests(self, skip_tests):
+    """Sets the specific test methods/test classes to skip in xctestrun file.
+
+    Args:
+      skip_tests: a list of string. The format of each item is
+          Test-Class-Name[/Test-Method-Name]
+    """
+    if not skip_tests:
+      return
+    self.SetXctestrunField('SkipTestIdentifiers', skip_tests)
+
   def Run(self, device_id, sdk, derived_data_dir):
     """Runs the test with generated xctestrun file in the specific device.
 
@@ -413,8 +424,17 @@ def _GenerateXctestrunFileForXcuitest(self):
     if os.path.exists(xctrunner_plugins_dir):
       shutil.rmtree(xctrunner_plugins_dir)
     os.mkdir(xctrunner_plugins_dir)
-    self._test_bundle_dir = _MoveAndReplaceFile(
-        self._test_bundle_dir, xctrunner_plugins_dir)
+    # The test bundle should not exist under the new generated XCTRunner.app.
+    if os.path.islink(self._test_bundle_dir):
+      # The test bundle under PlugIns can not be symlink since it will cause
+      # app installation error.
+      new_test_bundle_path = os.path.join(
+          xctrunner_plugins_dir, os.path.basename(self._test_bundle_dir))
+      shutil.copytree(self._test_bundle_dir, new_test_bundle_path)
+      self._test_bundle_dir = new_test_bundle_path
+    else:
+      self._test_bundle_dir = _MoveAndReplaceFile(
+          self._test_bundle_dir, xctrunner_plugins_dir)
 
     generated_xctestrun_file_paths = glob.glob('%s/*.xctestrun' %
                                                derived_data_build_products_dir)
@@ -460,8 +480,16 @@ def _GenerateXctestrunFileForXctest(self):
         self._app_under_test_dir, 'PlugIns')
     if not os.path.exists(app_under_test_plugins_dir):
       os.mkdir(app_under_test_plugins_dir)
-    self._test_bundle_dir = _MoveAndReplaceFile(
-        self._test_bundle_dir, app_under_test_plugins_dir)
+    new_test_bundle_path = os.path.join(
+        app_under_test_plugins_dir, os.path.basename(self._test_bundle_dir))
+    # The test bundle under PlugIns can not be symlink since it will cause
+    # app installation error.
+    if os.path.islink(self._test_bundle_dir):
+      shutil.copytree(self._test_bundle_dir, new_test_bundle_path)
+      self._test_bundle_dir = new_test_bundle_path
+    elif new_test_bundle_path != self._test_bundle_dir:
+      self._test_bundle_dir = _MoveAndReplaceFile(
+          self._test_bundle_dir, app_under_test_plugins_dir)
 
     # The xctestrun file are under the build products directory of dummy
     # project's derived data dir.