Robust solution for checking the device SDK in ios_test_runner

Robust solution for checking the device SDK in ios_test_runner.
- Support passing optional argument --platform=[ios_device,ios_simulator].
- If args.platform is not given, use the output `xcrun simctl list devices` and `instruments -s devices` to detect the SDK of the device from given device id.

For new iOS models, such as iPhone Xs, Xs MAX, the uuid is no longer 40 digit. To be more robust, ios_test_runner won't use uuid format to determine the SDK of the device. To be consistent with Google bazel, we add a new argument --platform.

Author: Weiming Dai

xctestrunner/shared/ios_constants.py

@@ -20,12 +20,16 @@ def enum(**enums):
 
 
 SDK = enum(IPHONEOS='iphoneos', IPHONESIMULATOR='iphonesimulator')
+# It is consistent with bazel's apple platform:
+# https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/rules/apple/ApplePlatform.java
+PLATFORM = enum(IOS_DEVICE='ios_device', IOS_SIMULATOR='ios_simulator')
 OS = enum(IOS='iOS', WATCHOS='watchOS', TVOS='tvOS')
 TestType = enum(XCUITEST='xcuitest', XCTEST='xctest', LOGIC_TEST='logic_test')
 SimState = enum(CREATING='Creating', SHUTDOWN='Shutdown', BOOTED='Booted',
                 UNKNOWN='Unknown')
 
 SUPPORTED_SDKS = [SDK.IPHONESIMULATOR, SDK.IPHONEOS]
+SUPPORTED_PLATFORMS = [PLATFORM.IOS_SIMULATOR, PLATFORM.IOS_DEVICE]
 SUPPORTED_TEST_TYPES = [TestType.XCUITEST, TestType.XCTEST, TestType.LOGIC_TEST]
 SUPPORTED_SIM_OSS = [OS.IOS]
 

xctestrunner/simulator_control/simulator_util.py

@@ -124,7 +124,7 @@ def Shutdown(self):
           'Can not shut down the simulator in state CREATING.')
     logging.info('Shutting down simulator %s.', self.simulator_id)
     try:
-      _RunSimctlCommand(['xcrun', 'simctl', 'shutdown', self.simulator_id])
+      RunSimctlCommand(['xcrun', 'simctl', 'shutdown', self.simulator_id])
     except ios_errors.SimError as e:
       if 'Unable to shutdown device in current state: Shutdown' in str(e):
         logging.info('Simulator %s has already shut down.', self.simulator_id)
@@ -143,13 +143,16 @@ def Delete(self):
     Raises:
       ios_errors.SimError: The simulator's state is not SHUTDOWN.
     """
-    sim_state = self.GetSimulatorState()
-    if sim_state != ios_constants.SimState.SHUTDOWN:
-      raise ios_errors.SimError(
-          'Can only delete the simulator with state SHUTDOWN. The current '
-          'state of simulator %s is %s.' % (self._simulator_id, sim_state))
+    # In Xcode 9+, simctl can delete Booted simulator. In prior of Xcode 9,
+    # we have to shutdown the simulator first before deleting it.
+    if xcode_info_util.GetXcodeVersionNumber() < 900:
+      sim_state = self.GetSimulatorState()
+      if sim_state != ios_constants.SimState.SHUTDOWN:
+        raise ios_errors.SimError(
+            'Can only delete the simulator with state SHUTDOWN. The current '
+            'state of simulator %s is %s.' % (self._simulator_id, sim_state))
     try:
-      _RunSimctlCommand(['xcrun', 'simctl', 'delete', self.simulator_id])
+      RunSimctlCommand(['xcrun', 'simctl', 'delete', self.simulator_id])
     except ios_errors.SimError as e:
       raise ios_errors.SimError(
           'Failed to delete simulator %s: %s' % (self.simulator_id, str(e)))
@@ -187,14 +190,14 @@ def GetAppDocumentsPath(self, app_bundle_id):
     """Gets the path of the app's Documents directory."""
     if xcode_info_util.GetXcodeVersionNumber() >= 830:
       try:
-        app_data_container = _RunSimctlCommand(
+        app_data_container = RunSimctlCommand(
             ['xcrun', 'simctl', 'get_app_container', self._simulator_id,
              app_bundle_id, 'data'])
         return os.path.join(app_data_container, 'Documents')
       except ios_errors.SimError as e:
         raise ios_errors.SimError(
-            'Failed to get data container of the app %s in simulator %s: %s',
-            app_bundle_id, self._simulator_id, str(e))
+            'Failed to get data container of the app %s in simulator %s: %s'%
+            (app_bundle_id, self._simulator_id, str(e)))
 
     apps_dir = os.path.join(
         self.simulator_root_dir, 'data/Containers/Data/Application')
@@ -208,13 +211,13 @@ def GetAppDocumentsPath(self, app_bundle_id):
       if current_app_bundle_id == app_bundle_id:
         return os.path.join(apps_dir, sub_dir_name, 'Documents')
     raise ios_errors.SimError(
-        'Failed to get Documents directory of the app %s in simulator %s: %s',
-        app_bundle_id, self._simulator_id)
+        'Failed to get Documents directory of the app %s in simulator %s' %
+        (app_bundle_id, self._simulator_id))
 
   def IsAppInstalled(self, app_bundle_id):
     """Checks if the simulator has installed the app with given bundle id."""
     try:
-      _RunSimctlCommand(
+      RunSimctlCommand(
           ['xcrun', 'simctl', 'get_app_container', self._simulator_id,
            app_bundle_id])
       return True
@@ -325,7 +328,7 @@ def CreateNewSimulator(device_type=None, os_version=None, name=None):
                name, os_type, os_version, device_type)
   for i in range(0, _SIM_OPERATION_MAX_ATTEMPTS):
     try:
-      new_simulator_id = _RunSimctlCommand(
+      new_simulator_id = RunSimctlCommand(
           ['xcrun', 'simctl', 'create', name, device_type, runtime_id])
     except ios_errors.SimError as e:
       raise ios_errors.SimError(
@@ -383,7 +386,7 @@ def GetSupportedSimDeviceTypes(os_type=None):
   #
   # See more examples in testdata/simctl_list_devicetypes.json
   sim_types_infos_json = json.loads(
-      _RunSimctlCommand(('xcrun', 'simctl', 'list', 'devicetypes', '-j')))
+      RunSimctlCommand(('xcrun', 'simctl', 'list', 'devicetypes', '-j')))
   sim_types = []
   for sim_types_info in sim_types_infos_json['devicetypes']:
     sim_type = sim_types_info['name']
@@ -438,7 +441,9 @@ def GetSupportedSimOsVersions(os_type=ios_constants.OS.IOS):
   # {
   # "runtimes" : [
   #   {
-  #     "bundlePath" : "\/Applications\/Xcode10.app\/Contents\/Developer\/Platforms\/iPhoneOS.platform\/Developer\/Library\/CoreSimulator\/Profiles\/Runtimes\/iOS.simruntime",
+  #     "bundlePath" : "\/Applications\/Xcode10.app\/Contents\/Developer\
+  #                     /Platforms\/iPhoneOS.platform\/Developer\/Library\
+  #                     /CoreSimulator\/Profiles\/Runtimes\/iOS.simruntime",
   #     "availabilityError" : "",
   #     "buildversion" : "16A366",
   #     "availability" : "(available)",
@@ -451,7 +456,7 @@ def GetSupportedSimOsVersions(os_type=ios_constants.OS.IOS):
   # See more examples in testdata/simctl_list_runtimes.json
   xcode_version_num = xcode_info_util.GetXcodeVersionNumber()
   sim_runtime_infos_json = json.loads(
-      _RunSimctlCommand(('xcrun', 'simctl', 'list', 'runtimes', '-j')))
+      RunSimctlCommand(('xcrun', 'simctl', 'list', 'runtimes', '-j')))
   sim_versions = []
   for sim_runtime_info in sim_runtime_infos_json['runtimes']:
     # Normally, the json does not contain unavailable runtimes. To be safe,
@@ -501,7 +506,7 @@ def GetLastSupportedSimOsVersion(os_type=ios_constants.OS.IOS,
   supported_os_versions = GetSupportedSimOsVersions(os_type)
   if not supported_os_versions:
     raise ios_errors.SimError(
-        'Can not find supported OS version of %s.', os_type)
+        'Can not find supported OS version of %s.' % os_type)
   if not device_type:
     return supported_os_versions[-1]
 
@@ -648,7 +653,7 @@ def IsCoreSimulatorCrash(sim_sys_log):
   return pattern.search(sim_sys_log) is not None
 
 
-def _RunSimctlCommand(command):
+def RunSimctlCommand(command):
   """Runs simctl command."""
   for i in range(_SIMCTL_MAX_ATTEMPTS):
     process = subprocess.Popen(

xctestrunner/test_runner/ios_test_runner.py

@@ -22,6 +22,7 @@
 import argparse
 import json
 import logging
+import subprocess
 import sys
 
 from xctestrunner.shared import ios_constants
@@ -105,9 +106,9 @@ def _AddTestSubParser(subparsers):
   """Adds sub parser for sub command `test`."""
   def _Test(args):
     """The function of sub command `test`."""
+    sdk = _PlatformToSdk(args.platform) if args.platform else _GetSdk(args.id)
     with xctest_session.XctestSession(
-        sdk=xctest_session.GetSdk(args.id),
-        work_dir=args.work_dir, output_dir=args.output_dir) as session:
+        sdk=sdk, work_dir=args.work_dir, output_dir=args.output_dir) as session:
       session.Prepare(
           app_under_test=args.app_under_test_path,
           test_bundle=args.test_bundle_path,
@@ -126,6 +127,12 @@ def _Test(args):
       '--id',
       required=True,
       help='The device id. The device can be iOS real device or simulator.')
+  optional_arguments = test_parser.add_argument_group('Optional arguments')
+  optional_arguments.add_argument(
+      '--platform',
+      help='The platform of the device. The value can be ios_device or '
+           'ios_simulator.'
+  )
   test_parser.set_defaults(func=_Test)
 
 
@@ -144,7 +151,11 @@ def _RunSimulatorTest(args):
           signing_options=_GetJson(args.signing_options_json_path))
       session.SetLaunchOptions(_GetJson(args.launch_options_json_path))
 
-      simulator_util.QuitSimulatorApp()
+      # In prior of Xcode 9, `xcodebuild test` will launch the Simulator.app
+      # process. If there is Simulator.app before running test, it will cause
+      # error later.
+      if xcode_info_util.GetXcodeVersionNumber() < 900:
+        simulator_util.QuitSimulatorApp()
       max_attempts = 3
       reboot_sim = False
       for i in range(max_attempts):
@@ -175,18 +186,22 @@ def _RunSimulatorTest(args):
               continue
           return exit_code
         finally:
-          # 1. Before Xcode 9, `xcodebuild test` will launch the Simulator.app
-          # process. Quit the Simulator.app to avoid side effect.
+          # 1. In prior of Xcode 9, `xcodebuild test` will launch the
+          # Simulator.app process. Quit the Simulator.app to avoid side effect.
           # 2. Quit Simulator.app can also shutdown the simulator. To make sure
           # the Simulator state to be SHUTDOWN, still call shutdown command
           # later.
           if xcode_info_util.GetXcodeVersionNumber() < 900:
             simulator_util.QuitSimulatorApp()
           simulator_obj = simulator_util.Simulator(simulator_id)
-          # Can only delete the "SHUTDOWN" state simulator.
-          simulator_obj.Shutdown()
-          # Deletes the new simulator to avoid side effect.
-          if not reboot_sim:
+          if reboot_sim:
+            simulator_obj.Shutdown()
+          else:
+            # In Xcode 9+, simctl can delete the Booted simulator.
+            # In prior of Xcode 9, we have to shutdown the simulator first
+            # before deleting it.
+            if xcode_info_util.GetXcodeVersionNumber() < 900:
+              simulator_obj.Shutdown()
             simulator_obj.Delete()
 
   def _SimulatorTest(args):
@@ -245,6 +260,38 @@ def _GetJson(json_path):
   return None
 
 
+def _PlatformToSdk(platform):
+  """Gets the SDK of the given platform."""
+  if platform == ios_constants.PLATFORM.IOS_DEVICE:
+    return ios_constants.SDK.IPHONEOS
+  if platform == ios_constants.PLATFORM.IOS_SIMULATOR:
+    return ios_constants.SDK.IPHONESIMULATOR
+  raise ios_errors.IllegalArgumentError(
+      'The platform %s is not supported. The supported values are %s.' %
+      (platform, ios_constants.SUPPORTED_PLATFORMS))
+
+
+def _GetSdk(device_id):
+  """Gets the sdk of the target device with the given device_id."""
+  # The command `instruments -s devices` is much slower than
+  # `xcrun simctl list devices`. So use `xcrun simctl list devices` to check
+  # IPHONESIMULATOR SDK first.
+  simlist_devices_output = simulator_util.RunSimctlCommand(
+      ['xcrun', 'simctl', 'list', 'devices'])
+  if device_id in simlist_devices_output:
+    return ios_constants.SDK.IPHONESIMULATOR
+
+  known_devices_output = subprocess.check_output(
+      ['instruments', '-s', 'devices'])
+  for line in known_devices_output.split('\n'):
+    if device_id in line and '(Simulator)' not in line:
+      return ios_constants.SDK.IPHONEOS
+
+  raise ios_errors.IllegalArgumentError(
+      'The device with id %s can not be found. The known devices are %s.' %
+      (device_id, known_devices_output))
+
+
 def main(argv):
   args = _BuildParser().parse_args(argv[1:])
   if args.verbose:

xctestrunner/test_runner/xctest_session.py

@@ -264,14 +264,6 @@ def Close(self):
       shutil.rmtree(self._output_dir)
 
 
-def GetSdk(device_id):
-  """Gets the sdk of the target device with the given device_id."""
-  if '-' in device_id:
-    return ios_constants.SDK.IPHONESIMULATOR
-  else:
-    return ios_constants.SDK.IPHONEOS
-
-
 def _PrepareBundles(working_dir, app_under_test_path, test_bundle_path):
   """Prepares the bundles in work directory.
 

xctestrunner/test_runner/xctestrun.py

@@ -271,8 +271,10 @@ def __init__(self, app_under_test_dir, test_bundle_dir,
     self._sdk = sdk
     self._test_type = test_type
     if self._sdk == ios_constants.SDK.IPHONEOS:
+      self._on_device = True
       self._signing_options = signing_options
     else:
+      self._on_device = False
       if signing_options:
         logging.info(
             'The signing options only works on sdk iphoneos, but current sdk '
@@ -373,7 +375,7 @@ def _ValidateArguments(self):
           'The test type %s is not supported. Supported test types are %s.'
           % (self._test_type, ios_constants.SUPPORTED_TEST_TYPES))
     if (self._test_type == ios_constants.TestType.LOGIC_TEST and
-        self._sdk != ios_constants.SDK.IPHONESIMULATOR):
+        self._on_device):
       raise ios_errors.IllegalArgumentError(
           'Only support running logic test on sdk iphonesimulator. '
           'Current sdk is %s' % self._sdk)
@@ -406,7 +408,7 @@ def _GenerateTestRootForXcuitest(self):
 
     self._PrepareUitestInRunerApp(uitest_runner_app)
 
-    if self._sdk == ios_constants.SDK.IPHONEOS:
+    if self._on_device:
       runner_app_embedded_provision = os.path.join(
           uitest_runner_app, 'embedded.mobileprovision')
       use_customized_provision = False
@@ -448,23 +450,22 @@ def _GenerateTestRootForXcuitest(self):
       plist_util.Plist(entitlements_plist_path).SetPlistField(
           None, entitlements_dict)
 
-      app_under_test_signing_identity = bundle_util.GetCodesignIdentity(
-          self._app_under_test_dir)
+      test_bundle_signing_identity = bundle_util.GetCodesignIdentity(
+          self._test_bundle_dir)
       bundle_util.CodesignBundle(
-          xctest_framework, identity=app_under_test_signing_identity)
+          xctest_framework, identity=test_bundle_signing_identity)
       if xcode_info_util.GetXcodeVersionNumber() >= 900:
         bundle_util.CodesignBundle(
-            xct_automation_framework, identity=app_under_test_signing_identity)
+            xct_automation_framework, identity=test_bundle_signing_identity)
       bundle_util.CodesignBundle(
           uitest_runner_app,
           entitlements_plist_path=entitlements_plist_path,
-          identity=app_under_test_signing_identity)
+          identity=test_bundle_signing_identity)
 
       bundle_util.CodesignBundle(self._test_bundle_dir)
       bundle_util.CodesignBundle(self._app_under_test_dir)
 
-    platform_name = ('iPhoneOS' if self._sdk == ios_constants.SDK.IPHONEOS else
-                     'iPhoneSimulator')
+    platform_name = 'iPhoneOS' if self._on_device else 'iPhoneSimulator'
     test_envs = {
         'DYLD_FRAMEWORK_PATH':
             '__TESTROOT__:__PLATFORMS__/%s.platform/Developer/'
@@ -581,7 +582,7 @@ def _GenerateTestRootForXctest(self):
                          'Developer/usr/lib/libXCTestBundleInject.dylib'),
             insert_libs_framework)
 
-    if self._sdk == ios_constants.SDK.IPHONEOS:
+    if self._on_device:
       app_under_test_signing_identity = bundle_util.GetCodesignIdentity(
           self._app_under_test_dir)
       bundle_util.CodesignBundle(
@@ -593,8 +594,7 @@ def _GenerateTestRootForXctest(self):
 
     app_under_test_name = os.path.splitext(
         os.path.basename(self._app_under_test_dir))[0]
-    platform_name = ('iPhoneOS' if self._sdk == ios_constants.SDK.IPHONEOS else
-                     'iPhoneSimulator')
+    platform_name = 'iPhoneOS' if self._on_device else 'iPhoneSimulator'
     if xcode_info_util.GetXcodeVersionNumber() < 1000:
       dyld_insert_libs = ('__PLATFORMS__/%s.platform/Developer/Library/'
                           'PrivateFrameworks/IDEBundleInjection.framework/'