diff --git a/docs/examples/comparison_workflows.md b/docs/examples/comparison_workflows.md index 0ec9b2a4e..cebffec54 100644 --- a/docs/examples/comparison_workflows.md +++ b/docs/examples/comparison_workflows.md @@ -21,3 +21,24 @@ comparison_experiment_paths: These experiments should have matching assimilation window parameters. By default in this suite, start and end cycle points are not specified, in which case Swell will parse the two experiments to find the matching cycle times between the two. Alternatively, start and end cycle points can be set manually. The experiment can then be created using `swell create compare_variational_marine -o override.yaml` or `swell create compare_variational_atmosphere -o override`, depending on the type of experiments being compared. Launching the experiment will run tasks analyzing the jedi log and generating plots using Eva for increments. Comparison of the log analysis will be placed under the comparison suite's directory in a file named `jedi_log_comparison.txt`, while the eva plots will be located under the cycle directory for each cycle. + +## Comparing JEDI builds + +This section describes how to run `ctests` on JEDI builds. The task `RunJediCtests` can be run in `build_jedi` experiments run by swell to output the results to a text file. `ctests` will be run for bundles specified in `bundles_to_run_ctests`. This field defaults to `fv3-jedi`, but any bundle with ctests can be run by this task. The output of the `ctest` execution is sent to `/ctests/ctest_results-.txt` + + +The `compare_jedi` suite checks the ctest results to ensure the two `build_jedi` experiments listed under `comparison_experiment_paths` pass the same ctests. The `bundles_to_run_ctests` key is also used by the `compare_jedi` suite to specify which bundles should be compared. The `compare_jedi` suite assumes that the task `RunJediCtests` has been run in the `build_jedi` experiments listed under `comparison_experiment_paths`. The task `CompareJediCtests` parses the output to figure out which tests fail for both experiments (the assumption is that some tests will always fail for most bundles, so the condition for zero-diff is ensuring the same tasks fail for both builds). If a mismatch in passed tests is detected, this task generates an error. The log output of this task lists the failed tasks for the two suites, and displays if any pass for one that is not passed for the other. For example, the output for comparing `fv3-jedi`: + +``` +fv3-jedi CTL EXP +fv3jedi_staticb_nicas_gfs Fail Fail +fv3jedi_hofx_nomodel_abi_radii Fail Fail +fv3jedi_staticb_split_nicas_gfs Fail Fail +fv3jedi_hyb Fail Fail +fv3jedi_staticb_dirac_local_gfs_12pe Fail Fail +fv3jedi_staticb_cor_geos Fail Fail +fv3jedi_staticb_dirac_local_gfs_6pe Fail Fail +fv3jedi_staticb_nicas_geos Fail Fail +fv3jedi_staticb_dirac_global_gfs_6pe Fail Fail +fv3jedi_staticb_dirac_global_gfs_12pe Fail Fail +``` diff --git a/src/swell/suites/build_jedi/flow.cylc b/src/swell/suites/build_jedi/flow.cylc index b7e347dee..068f4fdb1 100644 --- a/src/swell/suites/build_jedi/flow.cylc +++ b/src/swell/suites/build_jedi/flow.cylc @@ -22,6 +22,10 @@ CloneJedi => BuildJediByLinking? BuildJediByLinking:fail? => BuildJedi + + {% if bundles_to_run_ctests | length > 0 %} + BuildJedi => RunJediCtests + {% endif %} """ # -------------------------------------------------------------------------------------------------- @@ -53,4 +57,13 @@ --{{key}} = {{value}} {%- endfor %} + [[RunJediCtests]] + script = "swell task RunJediCtests $config" + platform = {{platform}} + execution time limit = {{scheduling["RunJediCtests"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediCtests"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare_jedi/flow.cylc b/src/swell/suites/compare_jedi/flow.cylc new file mode 100644 index 000000000..0e4bade4a --- /dev/null +++ b/src/swell/suites/compare_jedi/flow.cylc @@ -0,0 +1,42 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for comparing two builds of JEDI directly + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + [[graph]] + R1 = """ + CompareJediCtests + """ + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CompareJediCtests]] + script = "swell task CompareJediCtests $config" + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare_jedi/suite_config.py b/src/swell/suites/compare_jedi/suite_config.py new file mode 100644 index 000000000..a037744b5 --- /dev/null +++ b/src/swell/suites/compare_jedi/suite_config.py @@ -0,0 +1,33 @@ +# -------------------------------------------------------------------------------------------------- +# @package configuration +# +# Class containing the configuration. This is a dictionary that is converted from +# an input yaml configuration file. Various function are included for interacting with the +# dictionary. +# +# -------------------------------------------------------------------------------------------------- + + +from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.suites.suite_questions import SuiteQuestions as sq + +from enum import Enum + +from swell.utilities.question_defaults import QuestionDefaults as qd + +# -------------------------------------------------------------------------------------------------- + + +class SuiteConfig(QuestionContainer, Enum): + + # -------------------------------------------------------------------------------------------------- + + compare_jedi = QuestionList( + list_name="compare_jedi", + questions=[ + sq.all_suites, + qd.comparison_experiment_paths() + ] + ) + + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/compare_jedi_ctests.py b/src/swell/tasks/compare_jedi_ctests.py new file mode 100644 index 000000000..6c9c271ec --- /dev/null +++ b/src/swell/tasks/compare_jedi_ctests.py @@ -0,0 +1,145 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +import os +import re + +from swell.utilities.comparisons import comparison_tags +from swell.tasks.base.task_base import taskBase + +# -------------------------------------------------------------------------------------------------- + + +class CompareJediCtests(taskBase): + + def parse_results(self, results_file) -> list: + ''' + Read results from a file containing output from JEDI ctests, + and parse to a list of failed tests + + Parameters: + results_file: path to a file containing output from bundle ctests + (e.g. one generated by the RunJediCtests task) + + Returns: + failed_tests: list of tests failed for the build + ''' + + with open(results_file, 'r') as f: + lines = f.readlines() + + if len(lines) == 0: + raise Exception(f'File {results_file} does not contain results') + + failed_tests = [] + + for line in lines: + if re.search('- .* \(Failed\)', line): # noqa + failed_tests.append(line.split('-')[1].split('(Failed)')[0].strip()) + + failed_tests = list(set(failed_tests)) + + return failed_tests + + def execute(self) -> None: + + # Paths to experiments to compare + experiment_paths = self.config.comparison_experiment_paths() + + # Bundles to consider ctest results for + bundles = self.config.bundles_to_run_ctests() + + # Attach tags to paths, if not present + experiment_tag_paths = comparison_tags(experiment_paths, self.logger) + + experiment_tag_1 = list(experiment_tag_paths.keys())[0] + experiment_tag_2 = list(experiment_tag_paths.keys())[1] + + experiment_path_1 = list(experiment_tag_paths.values())[0] + experiment_path_2 = list(experiment_tag_paths.values())[1] + + # Paths to the ctest results rendered by the RunJediCtests task + ctest_path_1 = os.path.join(os.path.dirname(experiment_path_1), '..', 'ctests') + ctest_path_2 = os.path.join(os.path.dirname(experiment_path_2), '..', 'ctests') + + # Dict tracking all test results + results_dict = {} + + for bundle in bundles: + ctest_file_1 = os.path.join(ctest_path_1, f'ctest_results-{bundle}.txt') + ctest_file_2 = os.path.join(ctest_path_2, f'ctest_results-{bundle}.txt') + + # Parse for failed tests + failed_results_1 = self.parse_results(ctest_file_1) + failed_results_2 = self.parse_results(ctest_file_2) + + results_dict[bundle] = {} + + # Track which tests have failed for both builds + for test in failed_results_1: + if test not in results_dict[bundle].keys(): + results_dict[bundle][test] = {experiment_tag_2: 'Pass'} + results_dict[bundle][test][experiment_tag_1] = 'Fail' + results_dict[bundle][test]['width'] = len(test) + + for test in failed_results_2: + if test not in results_dict[bundle].keys(): + results_dict[bundle][test] = {experiment_tag_1: 'Pass'} + results_dict[bundle][test][experiment_tag_2] = 'Fail' + results_dict[bundle][test]['width'] = len(test) + + # Whether the same number of tests pass + passed = True + + # Format the string for readable output + results_str = 'JEDI CTest Results Comparison\n' + results_str += f'{experiment_tag_1}: {experiment_path_1}\n' + results_str += f'{experiment_tag_2}: {experiment_path_2}\n\n' + + width_col_1 = max(len('Fail'), len(experiment_tag_1)) + 2 + width_col_2 = max(len('Fail'), len(experiment_tag_2)) + 2 + + for bundle in bundles: + max_width = max(len(bundle), max([results_dict[bundle][test]['width'] + for test in results_dict[bundle]])) + 2 + results_str += bundle + ' ' * (max_width - len(bundle)) + results_str += experiment_tag_1 + ' ' * (width_col_1 - len(experiment_tag_1)) + results_str += experiment_tag_2 + ' ' * (width_col_2 - len(experiment_tag_2)) + results_str += '\n' + + # Specify if all tests have passed + if len(results_dict[bundle].keys()) == 0: + results_str += 'All tests passed.\n' + + for test in results_dict[bundle].keys(): + results_str += test + ' ' * (max_width - len(test)) + result_1 = results_dict[bundle][test][experiment_tag_1] + result_2 = results_dict[bundle][test][experiment_tag_2] + if result_1 != result_2: + passed = False + results_str += result_1 + ' ' * (width_col_1 - len(result_1)) + results_str += result_2 + ' ' * (width_col_2 - len(result_2)) + results_str += '\n' + + results_str += '\n' + + self.logger.info(results_str) + + out_path = os.path.join(self.experiment_path(), 'ctests') + os.makedirs(out_path, exist_ok=True) + with open(os.path.join(out_path, 'ctest_comparison.txt'), 'w') as f: + f.write(results_str) + + if not passed: + # Send the result to job.err as well + self.logger.error(results_str) + raise Exception(f'Differing tests passed between experiments') + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_ctests.py b/src/swell/tasks/run_jedi_ctests.py new file mode 100644 index 000000000..9bd5c6462 --- /dev/null +++ b/src/swell/tasks/run_jedi_ctests.py @@ -0,0 +1,55 @@ + +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os +import subprocess + +from swell.tasks.base.task_base import taskBase + +# -------------------------------------------------------------------------------------------------- + + +class RunJediCtests(taskBase): + + def execute(self) -> None: + + # Locate the experiment's jedi build dir, must be built or linked first + build_dir = os.path.join(self.experiment_path(), 'jedi_bundle', 'build') + + # Identify the bundles to run ctests on + bundles = self.config.bundles_to_run_ctests() + + for bundle in bundles: + bundle_dir = os.path.join(build_dir, bundle) + + # Run the ctests + cwd = os.getcwd() + os.chdir(bundle_dir) + command = ['ctest', '-V'] + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + # Record the output + results, error = process.communicate() + os.chdir(cwd) + + # Get the output file name + out_name = f'ctest_results-{bundle}.txt' + + # Make the output directory + out_path = os.path.join(self.experiment_path(), 'ctests') + os.makedirs(out_path, exist_ok=True) + + # Write the results + with open(os.path.join(out_path, out_name), 'w') as f: + f.write(f'CTest results for {bundle} bundle located at: {bundle_dir}\n') + f.write(results.decode('utf-8')) + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index d6cd00a41..d7273dde8 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -197,6 +197,16 @@ class TaskQuestions(QuestionContainer, Enum): # -------------------------------------------------------------------------------------------------- + CompareJediCtests = QuestionList( + list_name="CompareJediCtests", + questions=[ + qd.bundles_to_run_ctests(), + qd.comparison_experiment_paths() + ] + ) + + # -------------------------------------------------------------------------------------------------- + ConvertObsToIoda = QuestionList( list_name="ConvertObsToIoda", questions=[ @@ -633,6 +643,15 @@ class TaskQuestions(QuestionContainer, Enum): # -------------------------------------------------------------------------------------------------- + RunJediCtests = QuestionList( + list_name="RunJediCtests", + questions=[ + qd.bundles_to_run_ctests() + ] + ) + + # -------------------------------------------------------------------------------------------------- + RunJediEnsembleMeanVariance = QuestionList( list_name="RunJediEnsembleMeanVariance", questions=[ diff --git a/src/swell/test/test_driver.py b/src/swell/test/test_driver.py index 16060acaa..98d0eeef1 100644 --- a/src/swell/test/test_driver.py +++ b/src/swell/test/test_driver.py @@ -32,3 +32,6 @@ def test_wrapper(test: str) -> None: # -------------------------------------------------------------------------------------------------- + +if __name__ == '__main__': + test_wrapper('code_tests') diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index ac9c4c1b5..088a0ff5d 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -367,6 +367,27 @@ class bundles(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class bundles_to_run_ctests(TaskQuestion): + default_value: list[str] = mutable_field([ + "fv3-jedi" + ]) + question_name: str = "bundles_to_run_ctests" + ask_question: bool = True + options: list[str] = mutable_field([ + "fv3-jedi", + "soca", + "iodaconv", + "ufo", + "ioda", + "oops", + "saber" + ]) + prompt: str = "Which JEDI bundles to you wish to run ctests on?" + widget_type: WType = WType.STRING_CHECK_LIST + + # -------------------------------------------------------------------------------------------------- + @dataclass class check_for_obs(TaskQuestion): default_value: bool = True diff --git a/src/swell/utilities/slurm.py b/src/swell/utilities/slurm.py index 787650d81..4ee855217 100644 --- a/src/swell/utilities/slurm.py +++ b/src/swell/utilities/slurm.py @@ -88,6 +88,7 @@ def prepare_scheduling_dict( 'RunJediObsfiltersExecutable', 'RunJediUfoTestsExecutable', 'RunJediVariationalExecutable', + 'RunJediCtests' } # Throw an error if a user tries to set SLURM directives for a task that