Set stall timeout from swell launch

src/swell/deployment/create_experiment.py

@@ -589,9 +589,18 @@ def prepare_cylc_suite_jinja2(
     render_dictionary['scheduling']['BuildJedi']['execution_time_limit'] = 'PT3H'
     render_dictionary['scheduling']['EvaObservations']['execution_time_limit'] = 'PT30M'
 
+    # Set jinja templated string to use upon runtime
+    # ----------------------------------------------
+    render_dictionary['scheduling']['stall_timeout'] = """\
+    {% if environ.get('SWELL_CYLC_TIMEOUT') %}
+    [[events]]
+    stall timeout = {{environ['SWELL_CYLC_TIMEOUT']}}
+    {% endif %}"""
+
     # Render the template
     # -------------------
-    new_suite_file = template_string_jinja2(logger, suite_file, render_dictionary, False)
+    new_suite_file = template_string_jinja2(logger, suite_file, render_dictionary,
+                                            allow_unresolved=True)
 
     # Write suite file to experiment
     # ------------------------------

src/swell/deployment/launch_experiment.py

@@ -24,14 +24,16 @@ def __init__(
         suite_path: str,
         experiment_name: str,
         no_detach: bool,
-        log_path: str
+        log_path: str,
+        cylc_timeout: str
     ) -> None:
 
         self.logger = get_logger('DeployWorkflow')
         self.suite_path = suite_path
         self.experiment_name = experiment_name
         self.no_detach = no_detach
         self.log_path = log_path
+        self.cylc_timeout = cylc_timeout
 
     # ----------------------------------------------------------------------------------------------
 
@@ -50,6 +52,10 @@ def cylc_run_experiment(self) -> None:  # NB: Could be a factory based on workfl
         if os.path.exists(self.suite_path + 'global.cylc'):
             os.environ['CYLC_CONF_PATH'] = self.suite_path
 
+        # Tell cylc to exit a stall quickly
+        if self.cylc_timeout is not None:
+            os.environ['SWELL_CYLC_TIMEOUT'] = self.cylc_timeout
+
         # Install the suite
         if self.log_path:
             # Add optional path for workflow engine logging.
@@ -105,7 +111,8 @@ def cylc_run_experiment(self) -> None:  # NB: Could be a factory based on workfl
 def launch_experiment(
     suite_path: str,
     no_detach: bool,
-    log_path: str
+    log_path: str,
+    cylc_timeout: bool
 ) -> None:
 
     # Get the path to where the suite files are located
@@ -121,7 +128,7 @@ def launch_experiment(
 
     # Create the deployment object
     # ----------------------------
-    deploy_workflow = DeployWorkflow(suite_path, experiment_name, no_detach, log_path)
+    deploy_workflow = DeployWorkflow(suite_path, experiment_name, no_detach, log_path, cylc_timeout)
 
     # Write some info for the user
     # ----------------------------

src/swell/suites/3dfgat_atmos/flow.cylc

@@ -1,3 +1,4 @@
+#!jinja2
 # (C) Copyright 2021- United States Government as represented by the Administrator of the
 # National Aeronautics and Space Administration. All Rights Reserved.
 #
@@ -14,6 +15,8 @@
     UTC mode = True
     allow implicit tasks = False
 
+{{scheduling['stall_timeout']}}
+
 # --------------------------------------------------------------------------------------------------
 
 [scheduling]

src/swell/suites/3dfgat_marine_cycle/flow.cylc

@@ -1,3 +1,4 @@
+#!jinja2
 # (C) Copyright 2023 United States Government as represented by the Administrator of the
 # National Aeronautics and Space Administration. All Rights Reserved.
 #
@@ -14,6 +15,8 @@
     UTC mode = True
     allow implicit tasks = False
 
+{{scheduling['stall_timeout']}}
+
 # --------------------------------------------------------------------------------------------------
 
 [scheduling]

src/swell/suites/3dvar_atmos/flow.cylc

@@ -1,3 +1,4 @@
+#!jinja2
 # (C) Copyright 2021- United States Government as represented by the Administrator of the
 # National Aeronautics and Space Administration. All Rights Reserved.
 #

src/swell/suites/3dvar_cf/flow.cylc

@@ -1,3 +1,4 @@
+#!jinja2
 # (C) Copyright 2021- United States Government as represented by the Administrator of the
 # National Aeronautics and Space Administration. All Rights Reserved.
 #

src/swell/suites/3dvar_marine/flow.cylc

@@ -1,3 +1,4 @@
+#!jinja2
 # (C) Copyright 2021- United States Government as represented by the Administrator of the
 # National Aeronautics and Space Administration. All Rights Reserved.
 #
@@ -14,6 +15,8 @@
     UTC mode = True
     allow implicit tasks = False
 
+{{scheduling['stall_timeout']}}
+
 # --------------------------------------------------------------------------------------------------
 
 [scheduling]

src/swell/suites/3dvar_marine_cycle/flow.cylc

@@ -1,3 +1,4 @@
+#!jinja2
 # (C) Copyright 2023 United States Government as represented by the Administrator of the
 # National Aeronautics and Space Administration. All Rights Reserved.
 #
@@ -14,6 +15,8 @@
     UTC mode = True
     allow implicit tasks = False
 
+{{scheduling['stall_timeout']}}
+
 # --------------------------------------------------------------------------------------------------
 
 [scheduling]

src/swell/suites/build_geos/flow.cylc

@@ -1,3 +1,4 @@
+#!jinja2
 # (C) Copyright 2021- United States Government as represented by the Administrator of the
 # National Aeronautics and Space Administration. All Rights Reserved.
 #
@@ -13,6 +14,8 @@
 [scheduler]
     allow implicit tasks = False
 
+{{scheduling['stall_timeout']}}
+
 # --------------------------------------------------------------------------------------------------
 
 [scheduling]

src/swell/suites/build_jedi/flow.cylc

@@ -1,3 +1,4 @@
+#!jinja2
 # (C) Copyright 2021- United States Government as represented by the Administrator of the
 # National Aeronautics and Space Administration. All Rights Reserved.
 #
@@ -13,6 +14,8 @@
 [scheduler]
     allow implicit tasks = False
 
+{{scheduling['stall_timeout']}}
+
 # --------------------------------------------------------------------------------------------------
 
 [scheduling]

src/swell/suites/compare/flow.cylc

@@ -1,3 +1,4 @@
+#!jinja2
 # (C) Copyright 2021- United States Government as represented by the Administrator of the
 # National Aeronautics and Space Administration. All Rights Reserved.
 #
@@ -17,6 +18,8 @@
     UTC mode = True
     allow implicit tasks = False
 
+{{scheduling['stall_timeout']}}
+
 # --------------------------------------------------------------------------------------------------
 
 [scheduling]

src/swell/suites/convert_bufr/flow.cylc

@@ -1,3 +1,4 @@
+#!jinja2
 # (C) Copyright 2021- United States Government as represented by the Administrator of the
 # National Aeronautics and Space Administration. All Rights Reserved.
 #
@@ -14,6 +15,8 @@
     UTC mode = True
     allow implicit tasks = False
 
+{{scheduling['stall_timeout']}}
+
 # --------------------------------------------------------------------------------------------------
 
 [scheduling]

src/swell/suites/convert_ncdiags/flow.cylc

@@ -1,3 +1,4 @@
+#!jinja2
 # (C) Copyright 2021- United States Government as represented by the Administrator of the
 # National Aeronautics and Space Administration. All Rights Reserved.
 #
@@ -14,6 +15,8 @@
     UTC mode = True
     allow implicit tasks = False
 
+{{scheduling['stall_timeout']}}
+
 # --------------------------------------------------------------------------------------------------
 
 [scheduling]

src/swell/suites/eva_capabilities/flow.cylc

@@ -1,3 +1,4 @@
+#!jinja2
 # (C) Copyright 2021- United States Government as represented by the Administrator of the
 # National Aeronautics and Space Administration. All Rights Reserved.
 #
@@ -14,6 +15,8 @@
     UTC mode = True
     allow implicit tasks = False
 
+{{scheduling['stall_timeout']}}
+
 # --------------------------------------------------------------------------------------------------
 
 [scheduling]

src/swell/suites/forecast_coupled_geos/flow.cylc

@@ -1,3 +1,4 @@
+#!jinja2
 # (C) Copyright 2021- United States Government as represented by the Administrator of the
 # National Aeronautics and Space Administration. All Rights Reserved.
 #
@@ -14,6 +15,8 @@
     UTC mode = True
     allow implicit tasks = False
 
+{{scheduling['stall_timeout']}}
+
 # --------------------------------------------------------------------------------------------------
 
 [scheduling]

src/swell/suites/geosadas/flow.cylc

@@ -1,3 +1,4 @@
+#!jinja2
 # (C) Copyright 2021- United States Government as represented by the Administrator of the
 # National Aeronautics and Space Administration. All Rights Reserved.
 #
@@ -14,6 +15,8 @@
     UTC mode = True
     allow implicit tasks = False
 
+{{scheduling['stall_timeout']}}
+
 # --------------------------------------------------------------------------------------------------
 
 [scheduling]

src/swell/suites/hofx/flow.cylc

@@ -1,3 +1,4 @@
+#!jinja2
 # (C) Copyright 2021- United States Government as represented by the Administrator of the
 # National Aeronautics and Space Administration. All Rights Reserved.
 #
@@ -14,6 +15,8 @@
     UTC mode = True
     allow implicit tasks = False
 
+{{scheduling['stall_timeout']}}
+
 # --------------------------------------------------------------------------------------------------
 
 [scheduling]

src/swell/suites/hofx_cf/flow.cylc

@@ -1,3 +1,4 @@
+#!jinja2
 # (C) Copyright 2021- United States Government as represented by the Administrator of the
 # National Aeronautics and Space Administration. All Rights Reserved.
 #
@@ -14,6 +15,8 @@
     UTC mode = True
     allow implicit tasks = False
 
+{{scheduling['stall_timeout']}}
+
 # --------------------------------------------------------------------------------------------------
 
 [scheduling]

src/swell/suites/ingest_obs/flow.cylc

@@ -1,7 +1,10 @@
+#!jinja2
 [scheduler]
     UTC mode = True
     allow implicit tasks = False
 
+{{scheduling['stall_timeout']}}
+
 [scheduling]
     initial cycle point = {{start_cycle_point}}
     final cycle point = {{final_cycle_point}}
@@ -29,4 +32,4 @@
         script = "swell task IngestObs $config -d $datetime -m {{model_component}}"
         execution time limit = PT10M
 
-    {% endfor %}
+    {% endfor %}

src/swell/suites/localensembleda/flow.cylc

@@ -1,3 +1,4 @@
+#!jinja2
 # (C) Copyright 2021- United States Government as represented by the Administrator of the
 # National Aeronautics and Space Administration. All Rights Reserved.
 #
@@ -14,6 +15,8 @@
     UTC mode = True
     allow implicit tasks = False
 
+{{scheduling['stall_timeout']}}
+
 # --------------------------------------------------------------------------------------------------
 
 [scheduling]

src/swell/suites/ufo_testing/flow.cylc

@@ -1,3 +1,4 @@
+#!jinja2
 # (C) Copyright 2021- United States Government as represented by the Administrator of the
 # National Aeronautics and Space Administration. All Rights Reserved.
 #
@@ -14,6 +15,8 @@
     UTC mode = True
     allow implicit tasks = False
 
+{{scheduling['stall_timeout']}}
+
 # --------------------------------------------------------------------------------------------------
 
 [scheduling]

src/swell/swell.py

@@ -20,6 +20,7 @@
 from swell.suites.all_suites import AllSuites
 from swell.utilities.welcome_message import write_welcome_message
 from swell.utilities.scripts.utility_driver import get_utilities, utility_wrapper
+from swell.utilities.datetime_util import is_duration
 
 
 # --------------------------------------------------------------------------------------------------
@@ -84,6 +85,9 @@ def swell_driver() -> None:
 
 skip_r2d2_help = """Skip registering this experiment and storing products in R2D2."""
 
+cylc_timeout_help = """
+Set the cylc stall timeout manually for experiment. If unset, defaults to user value in
+ ~/.cylc/flow/global.cylc, or the Cylc default of 1 hour. Uses ISO duration format (e.g. PT30S)"""
 
 # --------------------------------------------------------------------------------------------------
 
@@ -163,10 +167,12 @@ def clone(
 @click.argument('suite_path')
 @click.option('-b', '--no-detach', 'no_detach', is_flag=True, default=False, help=no_detach_help)
 @click.option('-l', '--log_path', 'log_path', default=None, help=log_path_help)
+@click.option('-t', '--cylc-timeout', 'cylc_timeout', default=None, help=cylc_timeout_help)
 def launch(
     suite_path: str,
     no_detach: bool,
-    log_path: str
+    log_path: str,
+    cylc_timeout: bool
 ) -> None:
     """
     Launch an experiment with the cylc workflow manager
@@ -177,7 +183,12 @@ def launch(
         suite_path (str): Path to where the flow.cylc and associated suite files are located. \n
 
     """
-    launch_experiment(suite_path, no_detach, log_path)
+
+    if cylc_timeout is not None:
+        if not is_duration(cylc_timeout):
+            raise ValueError(f'Specified cylc timeout does not match ISO duration format')
+
+    launch_experiment(suite_path, no_detach, log_path, cylc_timeout)
 
 
 # --------------------------------------------------------------------------------------------------