Add simulation flags to groups and tests

davinci_pas_test_kit.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.description   = 'Inferno Test Kit for the Da Vinci Prior Authorization Support IG'
   spec.homepage      = 'https://github.com/inferno-framework/davinci-pas-test-kit'
   spec.license       = 'Apache-2.0'
-  spec.add_dependency 'inferno_core', '~> 1.0', '>= 1.0.2'
+  spec.add_dependency 'inferno_core', '~> 1.0', '>= 1.1.2'
   spec.add_dependency 'smart_app_launch_test_kit', '~> 1.0'
   spec.add_dependency 'subscriptions_test_kit', '~> 0.12.0'
   spec.add_dependency 'udap_security_test_kit', '~> 0.12.0'

lib/davinci_pas_test_kit/custom_groups/v2.0.1/client_tests/pas_client_inquire_response_bundle_validation_test.rb

@@ -10,10 +10,10 @@ class PasClientInquireResponseBundleValidationTest < Inferno::Test
       include ResponseGenerator
 
       id :pas_client_v201_inquire_response_bundle_validation_test
-      title '[USER INPUT VERIFICATION] Inquire Response Bundle is valid'
+      title 'Inquire Response Bundle is valid'
       description %(
-        **USER INPUT VERIFICATION**: This test verifies input provided by the tester instead of the system under test.
-        Errors encountered will be treated as a skip instead of a failure.
+        **USER INPUT VERIFICATION**: This test verifies input provided by the tester instead of
+        the system under test. Errors encountered will be treated as a skip instead of a failure.
 
         This test verifies the conformity of the inquire response sent by Inferno, which will have been
         either:
@@ -44,6 +44,7 @@ class PasClientInquireResponseBundleValidationTest < Inferno::Test
         issue](https://github.com/inferno-framework/davinci-pas-test-kit/issues/11)
         for additional details.
       )
+      simulation_verification
 
       def request_type
         'inquire'

lib/davinci_pas_test_kit/custom_groups/v2.0.1/client_tests/pas_client_response_bundle_validation_test.rb

@@ -10,10 +10,10 @@ class PasClientResponseBundleValidationTest < Inferno::Test
       include ResponseGenerator
 
       id :pas_client_v201_response_bundle_validation_test
-      title '[USER INPUT VERIFICATION] Submit Response Bundle is valid'
+      title 'Submit Response Bundle is valid'
       description %(
-        **USER INPUT VERIFICATION**: This test verifies input provided by the tester instead of the system under test.
-        Errors encountered will be treated as a skip instead of a failure.
+        **USER INPUT VERIFICATION**: This test verifies input provided by the tester instead
+        of the system under test. Errors encountered will be treated as a skip instead of a failure.
 
         This test verifies the conformity of the submit response sent by Inferno, which will have been
         either:
@@ -44,6 +44,7 @@ class PasClientResponseBundleValidationTest < Inferno::Test
         issue](https://github.com/inferno-framework/davinci-pas-test-kit/issues/11)
         for additional details.
       )
+      simulation_verification
 
       def request_type
         'submit'

lib/davinci_pas_test_kit/custom_groups/v2.0.1/notification/pas_subscription_notification_test.rb

@@ -5,6 +5,7 @@ module DaVinciPASV201
     # TODO: none of file name, class name, or ID match
     class PasUpdateNotificationTest < Inferno::Test
       include URLs
+
       id :prior_auth_claim_response_update_notification_validation
       title 'Server notifies the client that the pended claim was updated.'
       description %(

lib/davinci_pas_test_kit/custom_groups/v2.0.1/pas_client_denial_group.rb

@@ -9,6 +9,7 @@ module DaVinciPASTestKit
   module DaVinciPASV201
     class PASClientDenialGroup < Inferno::TestGroup
       include UserInputResponse
+
       id :pas_client_v201_denial_group
       title 'Denial Workflow'
       description %(

lib/davinci_pas_test_kit/custom_groups/v2.0.1/pas_client_pended_group.rb

@@ -11,6 +11,7 @@ module DaVinciPASTestKit
   module DaVinciPASV201
     class PASClientPendedGroup < Inferno::TestGroup
       include UserInputResponse
+
       id :pas_client_v201_pended_group
       title 'Pended Workflow'
       description %(
@@ -70,26 +71,29 @@ class PASClientPendedGroup < Inferno::TestGroup
         title 'Verify notification interaction'
 
         test from: :subscriptions_r4_client_notification_input_verification,
-             title: '[USER INPUT VERIFICATION] Tester-provided event notification Bundle is conformant',
+             title: 'Tester-provided event notification Bundle is conformant',
              description: %(
-               This test checks that the notification Bundle sent to the client, which will be either
-               the tester-provided notification Bundle in the **Claim updated notification JSON** input
-               or mocked by Inferno based on details in the Subscription and submitted Claim, is conformant
-               to Subscription Backport IG requirements.
+               **USER INPUT VERIFICATION**: This test checks that the notification Bundle sent to the
+               client, which will be either the tester-provided notification Bundle in the **Claim
+               updated notification JSON** input or mocked by Inferno based on details in the Subscription
+               and submitted Claim, is conformant to Subscription Backport IG requirements.
              ),
+             simulation_verification: true,
              config: {
                inputs: {
                  notification_bundle: { optional: true } # doesn't use the input (bug in Subscriptions)
                }
              }
         test from: :subscriptions_r4_client_notification_input_payload_verification,
-             title: '[USER INPUT VERIFICATION] Tester-provided event notification Bundle matches the Subscription',
+             title: 'Tester-provided event notification Bundle matches the Subscription',
              description: %(
-               This test checks that the notification Bundle sent to the client, which will be either
-               the tester-provided notification Bundle in the **Claim updated notification JSON** input
-               or mocked by Inferno based on details in the Subscription and submitted Claim, matches the details
-               requested in the Subscription provided during the **2.1** "PAS Subscription Setup" tests.
+               **USER INPUT VERIFICATION**: This test checks that the notification Bundle sent to the client,
+               which will be either the tester-provided notification Bundle in the **Claim updated notification
+               JSON** input or mocked by Inferno based on details in the Subscription and submitted Claim,
+               matches the details requested in the Subscription provided during the **2.1** "PAS Subscription
+               Setup" tests.
              ),
+             simulation_verification: true,
              config: {
                inputs: {
                  notification_bundle: { optional: true } # doesn't use the input (bug in Subscriptions)

lib/davinci_pas_test_kit/custom_groups/v2.0.1/pas_server_subscription_input_conformance.rb

@@ -4,17 +4,19 @@ module DaVinciPASTestKit
   module DaVinciPASV201
     class PASServerSubscriptionInputConformance < Inferno::Test
       include PASSubscriptionVerification
+
       id :pas_server_subscription_input_conformance
-      title '[USER INPUT VERIFICATION] Verify Subscription PAS conformance'
+      title 'Verify Subscription PAS conformance'
       description %(
-        This test accepts a Subscription resource as an input and verifies that it is conformant to PAS requirements on
-        the Subscriptions, including:
+        **USER INPUT VERIFICATION**: This test accepts a Subscription resource as an input and
+        verifies that it is conformant to PAS requirements on the Subscriptions, including:
         - The payload content type must be `id-only`
         - The use of the [PAS-defined Subscription
           Topic](https://hl7.org/fhir/us/davinci-pas/STU2/SubscriptionTopic-PASSubscriptionTopic.html), and
         - Inclusion of filter criteria for the client's organization.
       )
       input :subscription_resource
+      simulation_verification
 
       run do
         omit_if subscription_resource.blank?, 'Did not input a Subscription resource of this type.'

lib/davinci_pas_test_kit/custom_groups/v2.0.1/pas_server_subscription_setup.rb

@@ -16,7 +16,9 @@ class PASServerSubscriptionSetup < Inferno::TestGroup
       verifies_requirements 'hl7.fhir.us.davinci-pas_2.0.1@141'
       run_as_group
 
-      test from: :subscriptions_r4_server_subscription_conformance do
+      test from: :subscriptions_r4_server_subscription_conformance,
+           title: 'Verify Subscription to Send to Server',
+           simulation_verification: true do
         input :subscription_resource,
               title: 'Pended Prior Authorization Subscription',
               description: %(

lib/davinci_pas_test_kit/generated/v2.0.1/pas_inquiry_request_bundle/server_pas_inquiry_request_bundle_validation_test.rb

@@ -6,7 +6,7 @@ class ServerPasInquiryRequestBundleValidationTest < Inferno::Test
       include DaVinciPASTestKit::PasBundleValidation
 
       id :pas_server_v201_pas_inquiry_request_bundle_validation_test
-      title '[USER INPUT VALIDATION] Inquiry Request Bundle is valid'
+      title 'Inquiry Request Bundle is valid'
       description %(
         **USER INPUT VALIDATION**: This test validates input provided by the user instead of the system under test.
         Errors encountered will be treated as a skip instead of a failure.
@@ -39,6 +39,7 @@ class ServerPasInquiryRequestBundleValidationTest < Inferno::Test
         issue](https://github.com/inferno-framework/davinci-pas-test-kit/issues/11)
         for additional details.
       )
+      simulation_verification
 
       input :pa_inquire_request_payload,
             title: 'PAS Inquire Request Payload',

lib/davinci_pas_test_kit/generated/v2.0.1/pas_request_bundle/server_pas_request_bundle_validation_test.rb

@@ -6,7 +6,7 @@ class ServerPasRequestBundleValidationTest < Inferno::Test
       include DaVinciPASTestKit::PasBundleValidation
 
       id :pas_server_v201_pas_request_bundle_validation_test
-      title '[USER INPUT VALIDATION] Request Bundle is valid'
+      title 'Request Bundle is valid'
       description %(
         **USER INPUT VALIDATION**: This test validates input provided by the user instead of the system under test.
         Errors encountered will be treated as a skip instead of a failure.
@@ -39,6 +39,7 @@ class ServerPasRequestBundleValidationTest < Inferno::Test
         issue](https://github.com/inferno-framework/davinci-pas-test-kit/issues/11)
         for additional details.
       )
+      simulation_verification
 
       input :pa_submit_request_payload,
             title: 'PAS Submit Request Payload',

lib/davinci_pas_test_kit/generated/v2.0.1/pas_server_must_support_use_case_group.rb

@@ -112,7 +112,7 @@ def use_case
           - [PAS PractitionerRole](https://hl7.org/fhir/us/davinci-pas/STU2/StructureDefinition-profile-practitionerrole.html)
         )
         run_as_group
-        
+
         group do
           title 'Submission of claims to the $submit operation for must support validation'
 
@@ -143,8 +143,9 @@ def use_case
           test from: :pas_server_v201_pas_response_bundle_validation_test
         end
         group do
-          title '[USER INPUT VALIDATION] Submit Request Must Support'
+          title 'Submit Request Must Support'
           verifies_requirements 'hl7.fhir.us.davinci-pas_2.0.1@35'
+          simulation_verification
 
           test from: :pas_server_submit_v201_must_support_requirement
           test from: :pas_server_submit_request_v201_pas_request_bundle_must_support_test
@@ -205,7 +206,7 @@ def use_case
           - [PAS PractitionerRole](https://hl7.org/fhir/us/davinci-pas/STU2/StructureDefinition-profile-practitionerrole.html)
         )
         run_as_group
-        
+
         group do
           title 'Submission of claims to the $inquire operation for must support validation'
 
@@ -236,8 +237,9 @@ def use_case
           test from: :pas_server_v201_pas_inquiry_response_bundle_validation_test
         end
         group do
-          title '[USER INPUT VALIDATION] Inquiry Request Must Support'
+          title 'Inquiry Request Must Support'
           verifies_requirements 'hl7.fhir.us.davinci-pas_2.0.1@36'
+          simulation_verification
 
           test from: :pas_server_inquire_request_v201_pas_inquiry_request_bundle_must_support_test
           test from: :pas_server_inquire_request_v201_claim_inquiry_must_support_test

lib/davinci_pas_test_kit/generator/group_generator.rb

@@ -66,7 +66,7 @@ def title
         end
       end
 
-      def run_as_group
+      def run_as_group?
         return false if use_case.include?('must_support') && system == 'server'
 
         true
@@ -219,19 +219,27 @@ def grouped_must_support_test_ids
         inquiry_operation = inquiry_request_validation_test_ids +
                             inquiry_operation_test_ids +
                             inquiry_response_validation_test_ids
+        submit_request_key = [
+          'Submit Request Must Support',
+          "#{' ' * 10}verifies_requirements 'hl7.fhir.us.davinci-pas_2.0.1@35'",
+          "#{' ' * 10}simulation_verification"
+        ].join("\n")
+
+        inquiry_request_key = [
+          'Inquiry Request Must Support',
+          "#{' ' * 10}verifies_requirements 'hl7.fhir.us.davinci-pas_2.0.1@36'",
+          "#{' ' * 10}simulation_verification"
+        ].join("\n")
+
         {
           '$submit Element Support' => {
             'Submission of claims to the $submit operation for must support validation' => submit_tests,
-            "[USER INPUT VALIDATION] Submit Request Must Support'" \
-            "\n#{' ' * 10}verifies_requirements " \
-            "'hl7.fhir.us.davinci-pas_2.0.1@35" => submit_request_must_support_test_ids,
+            submit_request_key => submit_request_must_support_test_ids,
             'Submit Response Must Support' => submit_response_must_support_test_ids
           },
           '$inquire Element Support' => {
             'Submission of claims to the $inquire operation for must support validation' => inquiry_operation,
-            "[USER INPUT VALIDATION] Inquiry Request Must Support'" \
-            "\n#{' ' * 10}verifies_requirements " \
-            "'hl7.fhir.us.davinci-pas_2.0.1@36" => inquiry_request_must_support_test_ids,
+            inquiry_request_key => inquiry_request_must_support_test_ids,
             'Inquiry Response Must Support' => inquiry_response_must_support_test_ids
           }
         }

lib/davinci_pas_test_kit/generator/must_support_test_generator.rb

@@ -125,7 +125,7 @@ def build_must_support_list_string(uscdi_only)
           choice[:extension_ids].each { |id| extension_names.delete(id.to_s) } if choice[:extension_ids].present?
 
           element_paths = choice[:paths].map { |path| "#{resource_type}.#{path}" }.join(' or ')
-          extension_ids = choice[:extension_ids].map(&:to_s).join(' or ')
+          extension_ids = choice[:extension_ids].join(' or ')
 
           element_names << "#{element_paths} or #{extension_ids}"
         end

lib/davinci_pas_test_kit/generator/naming.rb

@@ -6,6 +6,7 @@ class Generator
     module Naming
       class << self
         include DaVinciPASTestKit::PASV201::ResourceList
+
         def resources_with_multiple_profiles
           resources = []
           RESOURCE_SUPPORTED_PROFILES.each do |resource, profile_list|

lib/davinci_pas_test_kit/generator/operation_test_generator.rb

@@ -82,7 +82,7 @@ def conformance_expectation
         read_interaction[:expectation]
       end
 
-      def skip_if_empty
+      def skip_if_empty?
         # Return true if a system must demonstrate at least one example of the resource type.
         # This drives omit vs. skip result statuses in this test.
         resource_type != 'Medication'
@@ -141,7 +141,7 @@ def description
           Server SHALL support PAS #{operation_name} requests: a POST interaction to
           the /Claim/$#{operation} endpoint.
           This test submits a Prior Authorization #{operation_name} request to the server and verifies that a
-          response is returned with HTTP status 2XX.#{operation == 'submit' ? "\nThe server SHOULD respond within 15 seconds." : ''}
+          response is returned with HTTP status 2XX.#{"\nThe server SHOULD respond within 15 seconds." if operation == 'submit'}
         DESCRIPTION
       end
     end

lib/davinci_pas_test_kit/generator/templates/group.rb.erb

@@ -12,7 +12,7 @@ module DaVinciPASTestKit
       <%- if verifies_requirements -%>
       verifies_requirements <%= verifies_requirements.map { |r| "'#{r}'" }.join(', ') %>
       <%- end -%>
-      <% if run_as_group %>run_as_group<% end %>
+      <% if run_as_group? %>run_as_group<% end %>
   <% if system == 'client'%>
     <% test_id_list.each do |id| %>
       test from: :<%= id %><% end %>
@@ -112,9 +112,24 @@ module DaVinciPASTestKit
           - [PAS PractitionerRole](https://hl7.org/fhir/us/davinci-pas/STU2/StructureDefinition-profile-practitionerrole.html)
         )<% end %>
         run_as_group
-        <% subgroup.each do |category, id_list| %>
-        group do
-          title '<%= category%>'
+        <%- subgroup.each do |category, id_list| -%>
+<%-
+  title_lines = category.to_s.split("\n").map(&:strip)
+  s = "        group do\n"
+  s << "          title '#{title_lines.shift.gsub("'","\\\\'")}'\n"
+  title_lines.each do |extra|
+    if extra.start_with?("verifies_requirements")
+    s << "          #{extra}\n"
+    elsif extra == 'simulation_verification'
+      s << "          simulation_verification\n"
+    else
+      s << "          #{extra}\n"
+    end
+  end
+  s.rstrip!
+  s << "\n"
+-%>
+<%= "\n" + s -%>
           <% id_list.each do |id|%>
           test from: :<%= id%><% if must_support_rename_input?(id) %> do
             id :<%= alt_test_id(id, use_case) %>

lib/davinci_pas_test_kit/generator/templates/validation.rb.erb

@@ -10,9 +10,12 @@ module DaVinciPASTestKit
       description %(
         <%=description.strip.gsub("\n", "\n" + " "*8) %>
       )
-      <%- if verifies_requirements -%>
+<% if simulation_verification? -%>
+      simulation_verification
+<% end -%>
+<% if verifies_requirements -%>
       verifies_requirements <%= verifies_requirements.map { |r| "'#{r}'" }.join(', ') %>
-      <%- end -%>
+<% end -%>
       <% if request_type.include?('request') %>
       input :pa_<%= request_type %>_payload,
             title: 'PAS <%= request_type.humanize.titleize%> Payload',

lib/davinci_pas_test_kit/generator/validation_test_generator.rb

@@ -87,7 +87,7 @@ def conformance_expectation
         read_interaction[:expectation]
       end
 
-      def skip_if_empty
+      def skip_if_empty?
         # Return true if a system must demonstrate at least one example of the resource type.
         # This drives omit vs. skip result statuses in this test.
         resource_type != 'Medication'
@@ -109,14 +109,17 @@ def generate
       end
 
       def title
-        pref = user_input? ? '[USER INPUT VALIDATION] ' : ''
-        "#{pref}#{group_metadata.title} is valid"
+        "#{group_metadata.title} is valid"
       end
 
       def user_input?
         request_type.include?('request')
       end
 
+      def simulation_verification?
+        user_input?
+      end
+
       def verifies_requirements
         case test_id
         when 'pas_server_v201_pas_inquiry_response_bundle_validation_test'