Ingest suite to download raw obs and convert it to ioda format, and ingest it to R2D2

src/swell/configuration/jedi/interfaces/geos_cf/convert_observations/omps_o3_nm_total.yaml

@@ -0,0 +1,23 @@
+# OMPS Nadir Mapper Total Ozone L2 — IODA converter configuration
+# Converter: omps_o3_nm_h52ioda.py (Must be available either in
+#            `converter_path`, set in suite_config.py or
+#            or in jedi_bundle/build/bin/)
+#
+# Invocation (one call per cycle, all granules passed together):
+#   python3 <jedi_bundle>/build/bin/omps_o3_nm_h52ioda.py
+#       -i <granule1.h5> <granule2.h5> ...
+#       -o <cycle_dir>/ioda/omps_o3_nm/omps_o3_nm_YYYYMMDDHH.nc
+#       -q 128
+#       -e atbd
+
+# name of the IODA-converterPython
+converter_script: omps_o3_nm_h52ioda.py
+
+# Output filename template (must be the same as 
+# source in ingest_observations/omps_o3_nm_total.yaml)
+output_filename_template: "omps_o3_nm_total_%Y%m%d%H.nc"
+
+# Additional IODA converter flags passed verbatim after -i and -o
+extra_flags:
+  -q: 128      # quality flag threshold
+  -e: atbd     # error estimation method

src/swell/configuration/jedi/interfaces/geos_cf/download_observations/omps_o3_nm_total.yaml

@@ -0,0 +1,19 @@
+# OMPS Nadir Mapper Total Ozone L2 — download configuration
+# Instrument: OMPS-NM on Suomi NPP
+# Product:    OMPS_NPP_NMTO3_L2 v2.1
+# Source:     NASA GES DISC (Earthdata authentication required in ~/.netrc)
+#
+# Files are named: OMPS-NPP_NMTO3-L2_v2.1_YYYYmMMDDtHH*.h5
+# where HH is the UTC hour at the start of the orbit granule.
+
+# Requires earthdata token in `~/.netrc'. 
+# Follow the instruction in the documentations
+remote_host: https://snpp-omps.gesdisc.eosdis.nasa.gov
+remote_path_template: /data/SNPP_OMPS_Level2/OMPS_NPP_NMTO3_L2.2/YYYY/JJJ/
+filename_pattern: OMPS-NPP_NMTO3-L2_v2.1_YYYYmMMDDtHH*.h5
+
+# Maximum duration of a single orbit granule. Used to extend the file
+# search window backwards so that orbits starting before window_begin
+# but containing data within the DA window are not missed.
+# OMPS-NM orbits are approximately 101 minutes; PT2H provides a safe margin.
+max_orbit_duration: PT2H

src/swell/configuration/jedi/interfaces/geos_cf/ingest_observations/omps_o3_nm_total.yaml

@@ -0,0 +1,5 @@
+# OMPS Nadir Mapper Total Ozone L2 — R2D2 ingestion configuration
+# Source: output of ConvertObsToIoda, relative to the cycle work directory.
+
+acquisition_method: local
+source: ioda/omps_o3_nm_total/omps_o3_nm_total_%Y%m%d%H.nc # in CYCLE_DIR

src/swell/configuration/jedi/interfaces/geos_marine/ingest_observations/adt_cryosat2n.yaml

@@ -2,5 +2,5 @@
 # time_begin: '2023-07-01T00:00:00Z'
 # time_end: '2999-12-31T23:59:59Z'
 
-retrieval_method: 'cp'
+acquisition_method: 'cp'
 cp_source: '/discover/nobackup/projects/gmao/soca/obs/ioda/ocean/adt_cryosat2n/YYYY/MM/ioda-obs-YYYYMMDDHH-adt_cryosat2n.nc'

src/swell/configuration/jedi/interfaces/geos_marine/ingest_observations/adt_sentinel6a.yaml

@@ -3,5 +3,5 @@
 # time_begin: '2023-07-01T00:00:00Z'
 # time_end: '2999-12-31T23:59:59Z'
 
-retrieval_method: 'cp'
+acquisition_method: 'cp'
 cp_source: '/discover/nobackup/projects/gmao/soca/obs/ioda/ocean/adt_sentinel6a/YYYY/MM'

src/swell/configuration/jedi/observation_ioda_names.yaml

@@ -344,17 +344,21 @@ ioda instrument names:
   # Compo
   - ioda name: tempo_no2_tropo
     full name: TEMPO tropospheric column NO2
-    inst type: no2
+    inst type: retrieval
     provider :  nasa_VCD
   - ioda name: tempo_no2_total
     full name: TEMPO total column NO2
-    inst type: no2
+    inst type: retrieval
     provider : nasa_VCD
   - ioda name: tropomi_s5p_no2_tropo
     full name: TROPOMI tropospheric column NO2
-    inst type: no2
+    inst type: retrieval
     provider : esa
   - ioda name: tropomi_s5p_co_tropo
     full name: TROPOMI tropospheric column CO
-    inst type: co
+    inst type: retrieval
     provider : esa
+  - ioda name: omps_o3_nm_total
+    full name: OMPS total column O3 (Nadir Mapper)
+    inst type: retrieval
+    provider : nasa

src/swell/deployment/platforms/nccs_discover_sles15/modules

@@ -61,5 +61,3 @@ PYTHONPATH={{swell_lib_path}}:$PYTHONPATH
 ulimit -S -s unlimited
 ulimit -S -v unlimited
 umask 022
-
-

src/swell/suites/ingest_obs/flow.cylc

@@ -1,21 +1,58 @@
+# (C) Copyright 2021- United States Government as represented by the Administrator of the
+# National Aeronautics and Space Administration. All Rights Reserved.
+#           
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+
+# --------------------------------------------------------------------------------------------------
+
+# Cylc suite for ingesting observations
+
+# --------------------------------------------------------------------------------------------------
+
 [scheduler]
     UTC mode = True
     allow implicit tasks = False
 
+# --------------------------------------------------------------------------------------------------
+
 [scheduling]
     initial cycle point = {{start_cycle_point}}
     final cycle point = {{final_cycle_point}}
 
     [[graph]]
+        {% if download_convert_pipeline %}
+        R1 = """
+            # Triggers for non cycle time dependent tasks
+            # -------------------------------------------
+            # Clone JEDI source code
+            CloneJedi
+
+            # Build JEDI source code by linking
+            CloneJedi => BuildJediByLinking?
+
+            # If not able to link to build create the build
+            BuildJediByLinking:fail? => BuildJedi
+
+        """
+        {% endif %}
 
         {% for cycle_time in cycle_times %}
             {{cycle_time.cycle_time}} = """
                 {% for model_component in model_components %}
+                {% if download_convert_pipeline %}
+                DownloadObs-{{model_component}} => ConvertObsToIoda-{{model_component}}
+                BuildJediByLinking[^]? | BuildJedi[^]  => ConvertObsToIoda-{{model_component}}
+                ConvertObsToIoda-{{model_component}} => IngestObs-{{model_component}}
+                {% else %}
                 IngestObs-{{model_component}}
+                {% endif %}
                 {% endfor %}
             """
         {% endfor %}
 
+# --------------------------------------------------------------------------------------------------
+
 [runtime]
     [[root]]
         pre-script = "source $CYLC_SUITE_DEF_PATH/modules"
@@ -24,9 +61,40 @@
             datetime = $CYLC_TASK_CYCLE_POINT
             config = $CYLC_SUITE_DEF_PATH/experiment.yaml
 
+    # Tasks
+    # -----
+    {% if download_convert_pipeline %}
+    [[CloneJedi]]
+        script = "swell task CloneJedi $config"
+
+    [[BuildJediByLinking]]
+        script = "swell task BuildJediByLinking $config"
+
+    [[BuildJedi]]
+        script = "swell task BuildJedi $config"
+        platform = {{platform}}
+        execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}}
+        [[[directives]]]
+        {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %}
+            --{{key}} = {{value}}
+        {%- endfor %}
+
+    {% endif %}
+
     {% for model_component in model_components %}
+
+    {% if download_convert_pipeline %}
+    [[DownloadObs-{{model_component}}]]
+        script = "swell task DownloadObs $config -d $datetime -m {{model_component}}"
+        execution time limit = PT30M
+
+    [[ConvertObsToIoda-{{model_component}}]]
+        script = "swell task ConvertObsToIoda $config -d $datetime -m {{model_component}}"
+        execution time limit = PT15M
+    {% endif %}
+
     [[IngestObs-{{model_component}}]]
         script = "swell task IngestObs $config -d $datetime -m {{model_component}}"
         execution time limit = PT10M
 
-    {% endfor %}
+    {% endfor %}

src/swell/suites/ingest_obs/suite_config.py

@@ -17,6 +17,7 @@ class SuiteConfig(QuestionContainer, Enum):
         list_name="ingest_obs",
         questions=[
             sq.common,
+            qd.download_convert_pipeline(False)
         ],
     )
     # This name should be unique and not conflict with other suites
@@ -38,3 +39,26 @@ class SuiteConfig(QuestionContainer, Enum):
             qd.dry_run(True),
         ]
     )
+
+    ingest_obs_cf = QuestionList(
+        list_name="ingest_obs_cf",
+        questions=[
+            ingest_obs,
+            qd.start_cycle_point("2023-08-10T00:00:00Z"),
+            qd.final_cycle_point("2023-08-11T00:00:00Z"),
+            qd.model_components(['geos_cf']),
+            qd.runahead_limit("P5"),
+            qd.download_convert_pipeline(True),
+            qd.jedi_build_method("use_existing"),  # For pyioda
+        ],
+        geos_cf=[
+            qd.window_length("PT6H"),
+            qd.obs_to_download(['omps_o3_nm_total']),
+            qd.obs_to_ingest(['omps_o3_nm_total']),
+            qd.converter_path(
+                "/discover/nobackup/projects/jcsda/s2127/maryamao/"
+                "jedi-bundle/build-intel-1.9/bin/"
+            ),
+            qd.dry_run(False),
+        ]
+    )