diff --git a/.circleci/config.yml b/.circleci/config.yml index b80ed505..1dbf15e8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,7 @@ version: 2.1 checkout: + method: full post: - ./ci-support/checkout_merge_commit.sh @@ -145,7 +146,8 @@ jobs: PYTHON_VERSION: << parameters.python_version >> CHANNELS: "-c conda-forge" steps: - - checkout + - checkout: + method: full - attach_workspace: at: . - run: *setup_env @@ -168,7 +170,8 @@ jobs: USER: "pcmdi" LABEL: "nightly" steps: - - checkout + - checkout: + method: full - attach_workspace: at: . - run: *setup_env @@ -184,7 +187,8 @@ jobs: USER: "pcmdi" LABEL: "testing" steps: - - checkout + - checkout: + method: full - attach_workspace: at: . - run: *setup_env diff --git a/.gitmodules b/.gitmodules index 2d1dd23e..562c920a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "mip-cmor-tables"] path = mip-cmor-tables url = https://github.com/PCMDI/mip-cmor-tables.git +[submodule "cmip7-cmor-tables"] + path = cmip7-cmor-tables + url = https://github.com/WCRP-CMIP/cmip7-cmor-tables diff --git a/Src/cmor.c b/Src/cmor.c index 5d949da5..16fc86de 100644 --- a/Src/cmor.c +++ b/Src/cmor.c @@ -2931,28 +2931,29 @@ int cmor_setDefaultGblAttr(int ref_table_id) /* -------------------------------------------------------------------- */ /* Set further_info_url template if required and not already set. */ -/* Set data_specs_version and mip_era if they are in the CV. */ /* -------------------------------------------------------------------- */ for (k = 0; k < required_attrs->anElements; k++) { if(strcmp(required_attrs->aszValue[k], GLOBAL_ATT_FURTHERINFOURL) == 0 && cmor_current_dataset.furtherinfourl[0] == '\0') { ierr |= cmor_set_cur_dataset_attribute_internal(GLOBAL_ATT_FURTHERINFOURLTMPL, CMOR_DEFAULT_FURTHERURL_TEMPLATE, 0); - } else if(strcmp(required_attrs->aszValue[k], GLOBAL_ATT_DATASPECSVERSION) == 0) { - CV_value = cmor_CV_rootsearch(cmor_tables[ref_table_id].CV, CV_KEY_DATASPECSVERSION); - if (CV_value != NULL && CV_value->szValue[0] != '\0') { - cmor_set_cur_dataset_attribute_internal(GLOBAL_ATT_DATASPECSVERSION, - CV_value->szValue, 0); - } - } else if(strcmp(required_attrs->aszValue[k], GLOBAL_ATT_MIP_ERA) == 0) { - CV_value = cmor_CV_rootsearch(cmor_tables[ref_table_id].CV, CV_KEY_MIP_ERA); - if (CV_value != NULL && CV_value->szValue[0] != '\0') { - cmor_set_cur_dataset_attribute_internal(GLOBAL_ATT_MIP_ERA, - CV_value->szValue, 0); - } } } +/* -------------------------------------------------------------------- */ +/* Set data_specs_version and mip_era if they are in the CV. */ +/* -------------------------------------------------------------------- */ + CV_value = cmor_CV_rootsearch(cmor_tables[ref_table_id].CV, CV_KEY_DATASPECSVERSION); + if (CV_value != NULL && CV_value->szValue[0] != '\0') { + cmor_set_cur_dataset_attribute_internal(GLOBAL_ATT_DATASPECSVERSION, + CV_value->szValue, 0); + } + CV_value = cmor_CV_rootsearch(cmor_tables[ref_table_id].CV, CV_KEY_MIP_ERA); + if (CV_value != NULL && CV_value->szValue[0] != '\0') { + cmor_set_cur_dataset_attribute_internal(GLOBAL_ATT_MIP_ERA, + CV_value->szValue, 0); + } + /* -------------------------------------------------------------------- */ /* Set the path and file name template if they are in the CV and */ /* the user didn't define them. */ diff --git a/Test/test_cmor_CMIP7.py b/Test/test_cmor_CMIP7.py index c6f24427..15e01da9 100644 --- a/Test/test_cmor_CMIP7.py +++ b/Test/test_cmor_CMIP7.py @@ -5,19 +5,22 @@ from netCDF4 import Dataset -DATASET_INFO = { - "_AXIS_ENTRY_FILE": "Tables/CMIP6_coordinate.json", - "_FORMULA_VAR_FILE": "Tables/CMIP6_formula_terms.json", +CMIP7_TABLES_PATH = "cmip7-cmor-tables/tables" +CV_PATH = "TestTables/CMIP7_CV.json" + +USER_INPUT = { + "_AXIS_ENTRY_FILE": "CMIP7_coordinate.json", + "_FORMULA_VAR_FILE": "CMIP7_formula_terms.json", "_cmip7_option": 1, - "_controlled_vocabulary_file": "TestTables/CMIP7_CV.json", + "_controlled_vocabulary_file": CV_PATH, "activity_id": "CMIP", "branch_method": "standard", "branch_time_in_child": 30.0, "branch_time_in_parent": 10800.0, "calendar": "360_day", "cv_version": "6.2.19.0", - "experiment": "1 percent per year increase in CO2", - "experiment_id": "1pctCO2", + "experiment": "Simulation of the pre-industrial climate", + "experiment_id": "piControl", "forcing_index": "f30", "grid": "N96", "grid_label": "gn", @@ -52,11 +55,11 @@ def setUp(self): Write out a simple file using CMOR """ # Set up CMOR - cmor.setup(inpath="TestTables", netcdf_file_action=cmor.CMOR_REPLACE) + cmor.setup(inpath=CMIP7_TABLES_PATH, netcdf_file_action=cmor.CMOR_REPLACE) - # Define dataset using DATASET_INFO + # Define dataset using USER_INPUT with open("Test/input_cmip7.json", "w") as input_file_handle: - json.dump(DATASET_INFO, input_file_handle, sort_keys=True, indent=4) + json.dump(USER_INPUT, input_file_handle, sort_keys=True, indent=4) # read dataset info error_flag = cmor.dataset_json("Test/input_cmip7.json") @@ -77,7 +80,7 @@ def test_cmip7(self): ]) time = numpy.array([15.5, 45]) time_bnds = numpy.array([0, 31, 60]) - cmor.load_table("CMIP7_ocean2d.json") + cmor.load_table("CMIP7_ocean.json") cmorlat = cmor.axis("latitude", coord_vals=lat, cell_bounds=lat_bnds, @@ -113,9 +116,9 @@ def test_cmip7(self): } for attr, val in test_attrs.items(): - self.assertTrue(attr in attrs) + self.assertIn(attr, attrs) self.assertEqual(val, ds.getncattr(attr)) - institution_id = DATASET_INFO["institution_id"] + institution_id = USER_INPUT["institution_id"] license_id = "CC BY 4.0" license_type = "Creative Commons Attribution 4.0 International" license_url = "https://creativecommons.org/licenses/by/4.0/" @@ -153,7 +156,7 @@ def test_secondary_modeling_realm(self): ]) time = numpy.array([15.5, 45]) time_bnds = numpy.array([0, 31, 60]) - cmor.load_table("CMIP7_atmos2d.json") + cmor.load_table("CMIP7_atmos.json") cmorlat = cmor.axis("latitude", coord_vals=lat, cell_bounds=lat_bnds, @@ -167,7 +170,7 @@ def test_secondary_modeling_realm(self): cell_bounds=time_bnds, units="days since 2018") axes = [cmortime, cmorlat, cmorlon] - cmorpr = cmor.variable("pr_tavg-u-hxy-u", "kg m-2 s-1", axes) + cmorpr = cmor.variable("prra_tavg-u-hxy-is", "kg m-2 s-1", axes) self.assertEqual(cmor.write(cmorpr, pr), 0) filename = cmor.close(cmorpr, file_name=True) self.assertEqual(cmor.close(), 0) @@ -175,22 +178,22 @@ def test_secondary_modeling_realm(self): ds = Dataset(filename) attrs = ds.ncattrs() test_attrs = { - 'branding_suffix': 'tavg-u-hxy-u', + 'branding_suffix': 'tavg-u-hxy-is', 'temporal_label': 'tavg', 'vertical_label': 'u', 'horizontal_label': 'hxy', - 'area_label': 'u', + 'area_label': 'is', 'region': 'glb', 'frequency': 'mon', 'archive_id': 'WCRP', 'mip_era': 'CMIP7', 'data_specs_version': 'CMIP-7.0.0.0', 'host_collection': 'CMIP7', - 'realm': 'atmos ocean', + 'realm': 'atmos land landIce', } for attr, val in test_attrs.items(): - self.assertTrue(attr in attrs) + self.assertIn(attr, attrs) self.assertEqual(val, ds.getncattr(attr)) ds.close() diff --git a/Test/test_cmor_check_cv_structure.py b/Test/test_cmor_check_cv_structure.py index f9f00add..f779cf35 100644 --- a/Test/test_cmor_check_cv_structure.py +++ b/Test/test_cmor_check_cv_structure.py @@ -4,23 +4,26 @@ import unittest from base_CMIP6_CV import BaseCVsTest +CMIP7_TABLES_PATH = "cmip7-cmor-tables/tables" +CV_PATH = "TestTables/CMIP7_CV.json" + USER_INPUT = { - "_AXIS_ENTRY_FILE": "Tables/CMIP6_coordinate.json", - "_FORMULA_VAR_FILE": "Tables/CMIP6_formula_terms.json", + "_AXIS_ENTRY_FILE": "CMIP7_coordinate.json", + "_FORMULA_VAR_FILE": "CMIP7_formula_terms.json", "_cmip7_option": 1, - "_controlled_vocabulary_file": "", + "_controlled_vocabulary_file": None, "activity_id": "CMIP", "branch_method": "standard", "branch_time_in_child": 30.0, "branch_time_in_parent": 10800.0, "calendar": "360_day", "cv_version": "6.2.19.0", - "experiment": "1 percent per year increase in CO2", - "experiment_id": "1pctCO2", - "forcing_index": "3", + "experiment": "Simulation of the pre-industrial climate", + "experiment_id": "piControl", + "forcing_index": "f30", "grid": "N96", "grid_label": "gn", - "initialization_index": "1", + "initialization_index": "i000001d", "institution_id": "PCMDI", "license_id": "CC BY 4.0", "nominal_resolution": "250 km", @@ -31,8 +34,8 @@ "parent_source_id": "PCMDI-test-1-0", "parent_experiment_id": "piControl", "parent_variant_label": "r1i1p1f3", - "physics_index": "1", - "realization_index": "9", + "physics_index": "p1", + "realization_index": "r009", "source_id": "PCMDI-test-1-0", "source_type": "AOGCM CHEM BGC", "tracking_prefix": "hdl:21.14100", @@ -51,11 +54,11 @@ def test_check_for_cv_json_key(self): cv_path = f"TestTables/CMIP7_CV_{test_name}.json" input_path = f"Test/input_{test_name}.json" - cmor.setup(inpath="TestTables", + cmor.setup(inpath=CMIP7_TABLES_PATH, netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) - with open("TestTables/CMIP7_CV.json", "r") as cv_infile: + with open(CV_PATH, "r") as cv_infile: cv = json.load(cv_infile) no_cv_key = cv["CV"] with open(cv_path, "w") as cv_outfile: @@ -71,7 +74,7 @@ def test_check_for_cv_json_key(self): raise RuntimeError("CMOR dataset_json call failed") with self.assertRaises(cmor.CMORError): - cmor.load_table("CMIP7_ocean2d.json") + cmor.load_table("CMIP7_ocean.json") self.assertCV("CV section was not found in table: ") @@ -84,11 +87,11 @@ def test_check_cv_attribute_values(self): cv_path = f"TestTables/CMIP7_CV_{test_name}.json" input_path = f"Test/input_{test_name}.json" - cmor.setup(inpath="TestTables", + cmor.setup(inpath=CMIP7_TABLES_PATH, netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) - with open("TestTables/CMIP7_CV.json", "r") as cv_infile: + with open(CV_PATH, "r") as cv_infile: cv = json.load(cv_infile) cv["CV"]["branding_suffix"] = ["template"] cv["CV"]["mip_era"] = {"mip_era": "CMIP7"} @@ -107,7 +110,7 @@ def test_check_cv_attribute_values(self): if error_flag: raise RuntimeError("CMOR dataset_json call failed") - cmor.load_table("CMIP7_ocean2d.json") + cmor.load_table("CMIP7_ocean.json") self.assertCV( "must be a string", @@ -143,11 +146,11 @@ def test_check_cv_single_value_pairs(self): cv_path = f"TestTables/CMIP7_CV_{test_name}.json" input_path = f"Test/input_{test_name}.json" - cmor.setup(inpath="TestTables", + cmor.setup(inpath=CMIP7_TABLES_PATH, netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) - with open("TestTables/CMIP7_CV.json", "r") as cv_infile: + with open(CV_PATH, "r") as cv_infile: cv = json.load(cv_infile) cv["CV"]["institution_id"]["PCMDI"] = {"LLNL": "Livermore, CA"} with open(cv_path, "w") as cv_outfile: @@ -162,7 +165,7 @@ def test_check_cv_single_value_pairs(self): if error_flag: raise RuntimeError("CMOR dataset_json call failed") - cmor.load_table("CMIP7_ocean2d.json") + cmor.load_table("CMIP7_ocean.json") self.assertCV( "in attribute \"institution_id\" cannot be an object", @@ -178,7 +181,7 @@ def test_check_cv_array_values(self): cv_path = f"TestTables/CMIP7_CV_{test_name}.json" input_path = f"Test/input_{test_name}.json" - cmor.setup(inpath="TestTables", + cmor.setup(inpath=CMIP7_TABLES_PATH, netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) @@ -201,7 +204,7 @@ def test_check_cv_array_values(self): if error_flag: raise RuntimeError("CMOR dataset_json call failed") - cmor.load_table("CMIP7_ocean2d.json") + cmor.load_table("CMIP7_ocean.json") self.assertCV( "has elements in its array that are not strings", @@ -217,7 +220,7 @@ def test_check_cv_nested_objects(self): cv_path = f"TestTables/CMIP7_CV_{test_name}.json" input_path = f"Test/input_{test_name}.json" - cmor.setup(inpath="TestTables", + cmor.setup(inpath=CMIP7_TABLES_PATH, netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) @@ -244,7 +247,7 @@ def test_check_cv_nested_objects(self): if error_flag: raise RuntimeError("CMOR dataset_json call failed") - cmor.load_table("CMIP7_ocean2d.json") + cmor.load_table("CMIP7_ocean.json") self.assertCV( "has elements in its array that are not strings", diff --git a/Test/test_cmor_license_attributes.py b/Test/test_cmor_license_attributes.py index d7e951fd..fb417590 100644 --- a/Test/test_cmor_license_attributes.py +++ b/Test/test_cmor_license_attributes.py @@ -5,23 +5,26 @@ from netCDF4 import Dataset -DATASET_INFO = { - "_AXIS_ENTRY_FILE": "Tables/CMIP6_coordinate.json", - "_FORMULA_VAR_FILE": "Tables/CMIP6_formula_terms.json", +CMIP7_TABLES_PATH = "cmip7-cmor-tables/tables" +CV_PATH = "TestTables/CMIP7_CV.json" + +USER_INPUT = { + "_AXIS_ENTRY_FILE": "CMIP7_coordinate.json", + "_FORMULA_VAR_FILE": "CMIP7_formula_terms.json", "_cmip7_option": 1, - "_controlled_vocabulary_file": "TestTables/CMIP7_CV.json", + "_controlled_vocabulary_file": None, "activity_id": "CMIP", "branch_method": "standard", "branch_time_in_child": 30.0, "branch_time_in_parent": 10800.0, "calendar": "360_day", "cv_version": "6.2.19.0", - "experiment": "1 percent per year increase in CO2", - "experiment_id": "1pctCO2", - "forcing_index": "f3", + "experiment": "Simulation of the pre-industrial climate", + "experiment_id": "piControl", + "forcing_index": "f30", "grid": "N96", "grid_label": "gn", - "initialization_index": "i1", + "initialization_index": "i000001d", "institution_id": "PCMDI", "license_id": "CC BY 4.0", "nominal_resolution": "250 km", @@ -33,7 +36,7 @@ "parent_experiment_id": "piControl", "parent_variant_label": "r1i1p1f3", "physics_index": "p1", - "realization_index": "r9", + "realization_index": "r009", "source_id": "PCMDI-test-1-0", "source_type": "AOGCM CHEM BGC", "tracking_prefix": "hdl:21.14100", @@ -52,21 +55,21 @@ def setUp(self): Write out a simple file using CMOR """ # Set up CMOR - cmor.setup(inpath="TestTables", netcdf_file_action=cmor.CMOR_REPLACE) + cmor.setup(inpath=CMIP7_TABLES_PATH, netcdf_file_action=cmor.CMOR_REPLACE) # Add 'license_id' and 'license_url' to required attributes of CV updated_cv_path = "TestTables/CMIP7_CV_license_attrs.json" - with open("TestTables/CMIP7_CV.json", "r") as cv_infile: + with open(CV_PATH, "r") as cv_infile: cv = json.load(cv_infile) cv["CV"]["required_global_attributes"].append("license_id") cv["CV"]["required_global_attributes"].append("license_url") with open(updated_cv_path, "w") as cv_outfile: json.dump(cv, cv_outfile, sort_keys=True, indent=4) - # Define dataset using DATASET_INFO - DATASET_INFO["_controlled_vocabulary_file"] = updated_cv_path + # Define dataset using USER_INPUT + USER_INPUT["_controlled_vocabulary_file"] = updated_cv_path with open("Test/input_cmip7.json", "w") as input_file_handle: - json.dump(DATASET_INFO, input_file_handle, sort_keys=True, indent=4) + json.dump(USER_INPUT, input_file_handle, sort_keys=True, indent=4) # read dataset info error_flag = cmor.dataset_json("Test/input_cmip7.json") @@ -92,7 +95,7 @@ def test_cmip7_with_license_id_and_url(self): ]) time = numpy.array([15.5, 45]) time_bnds = numpy.array([0, 31, 60]) - cmor.load_table("CMIP7_ocean2d.json") + cmor.load_table("CMIP7_ocean.json") cmorlat = cmor.axis("latitude", coord_vals=lat, cell_bounds=lat_bnds, @@ -130,7 +133,7 @@ def test_cmip7_with_license_id_and_url(self): for attr, val in test_attrs.items(): self.assertTrue(attr in attrs) self.assertEqual(val, ds.getncattr(attr)) - institution_id = DATASET_INFO["institution_id"] + institution_id = USER_INPUT["institution_id"] license_id = "CC BY 4.0" license_type = "Creative Commons Attribution 4.0 International" license_url = "https://creativecommons.org/licenses/by/4.0/" diff --git a/Test/test_cmor_path_and_file_templates.py b/Test/test_cmor_path_and_file_templates.py index 2fae015b..de6114a8 100644 --- a/Test/test_cmor_path_and_file_templates.py +++ b/Test/test_cmor_path_and_file_templates.py @@ -4,25 +4,26 @@ import os import numpy -CV_PATH = "TestTables/CMIP7_CV_DRS.json" +CMIP7_TABLES_PATH = "cmip7-cmor-tables/tables" +CV_PATH = "TestTables/CMIP7_CV.json" -DATASET_INFO = { - "_AXIS_ENTRY_FILE": "Tables/CMIP6_coordinate.json", - "_FORMULA_VAR_FILE": "Tables/CMIP6_formula_terms.json", +USER_INPUT = { + "_AXIS_ENTRY_FILE": "CMIP7_coordinate.json", + "_FORMULA_VAR_FILE": "CMIP7_formula_terms.json", "_cmip7_option": 1, - "_controlled_vocabulary_file": CV_PATH, + "_controlled_vocabulary_file": None, "activity_id": "CMIP", "branch_method": "standard", "branch_time_in_child": 30.0, "branch_time_in_parent": 10800.0, "calendar": "360_day", "cv_version": "6.2.19.0", - "experiment": "1 percent per year increase in CO2", - "experiment_id": "1pctCO2", - "forcing_index": "f3", + "experiment": "Simulation of the pre-industrial climate", + "experiment_id": "piControl", + "forcing_index": "f30", "grid": "N96", "grid_label": "gn", - "initialization_index": "i1", + "initialization_index": "i000001d", "institution_id": "PCMDI", "license_id": "CC BY 4.0", "nominal_resolution": "250 km", @@ -34,7 +35,7 @@ "parent_experiment_id": "piControl", "parent_variant_label": "r1i1p1f3", "physics_index": "p1", - "realization_index": "r9", + "realization_index": "r009", "source_id": "PCMDI-test-1-0", "source_type": "AOGCM CHEM BGC", "tracking_prefix": "hdl:21.14100", @@ -66,7 +67,7 @@ def gen_cmor_file(self): ]) time = numpy.array([15.5, 45]) time_bnds = numpy.array([0, 31, 60]) - cmor.load_table("CMIP7_ocean2d.json") + cmor.load_table("CMIP7_ocean.json") cmorlat = cmor.axis("latitude", coord_vals=lat, cell_bounds=lat_bnds, @@ -86,11 +87,11 @@ def gen_cmor_file(self): filepath = cmor.close(cmortos, file_name=True) # Attributes for reconstructing file path - attrs = DATASET_INFO.copy() + attrs = USER_INPUT.copy() variant_label = ('{realization_index}{initialization_index}' '{physics_index}{forcing_index}').format(**attrs) attrs['mip_era'] = 'CMIP7' - attrs['table_id'] = 'ocean2d' + attrs['table_id'] = 'ocean' attrs['variable_id'] = 'tos' attrs['version'] = version attrs['variant_label'] = variant_label @@ -99,21 +100,27 @@ def gen_cmor_file(self): return (filepath, attrs) def test_default_path_and_file_templates(self): + + test_name = "default_path_and_file_templates" + test_cv_path = f"TestTables/CMIP7_CV_{test_name}.json" + # Set up CMOR - cmor.setup(inpath="TestTables", + cmor.setup(inpath=CMIP7_TABLES_PATH, netcdf_file_action=cmor.CMOR_REPLACE) # Remove "DRS" section from CV to use default # directory path and file name templates in CMOR - with open("TestTables/CMIP7_CV.json", "r") as cv_infile: + with open(CV_PATH, "r") as cv_infile: cv = json.load(cv_infile) cv["CV"].pop("DRS") - with open(CV_PATH, "w") as cv_outfile: + with open(test_cv_path, "w") as cv_outfile: json.dump(cv, cv_outfile, sort_keys=True, indent=4) - # Define dataset using DATASET_INFO + # Define dataset using USER_INPUT with open("Test/input_drs.json", "w") as input_file: - json.dump(DATASET_INFO, input_file, sort_keys=True, indent=4) + user_input = USER_INPUT.copy() + user_input["_controlled_vocabulary_file"] = test_cv_path + json.dump(user_input, input_file, sort_keys=True, indent=4) # read dataset info error_flag = cmor.dataset_json("Test/input_drs.json") @@ -133,22 +140,30 @@ def test_default_path_and_file_templates(self): self.assertEqual(filepath, predicted_filepath) os.remove(filepath) - os.remove(CV_PATH) + os.remove(test_cv_path) def test_cv_path_and_file_templates(self): + + test_name = "cv_path_and_file_templates" + test_cv_path = f"TestTables/CMIP7_CV_{test_name}.json" + # Set up CMOR - cmor.setup(inpath="TestTables", + cmor.setup(inpath=CMIP7_TABLES_PATH, netcdf_file_action=cmor.CMOR_REPLACE) # Use "DRS" section from CV - with open("TestTables/CMIP7_CV.json", "r") as cv_infile: + drs = {} + with open(CV_PATH, "r") as cv_infile: cv = json.load(cv_infile) - with open(CV_PATH, "w") as cv_outfile: + drs = cv["CV"]["DRS"] + with open(test_cv_path, "w") as cv_outfile: json.dump(cv, cv_outfile, sort_keys=True, indent=4) - # Define dataset using DATASET_INFO + # Define dataset using USER_INPUT with open("Test/input_drs.json", "w") as input_file: - json.dump(DATASET_INFO, input_file, sort_keys=True, indent=4) + user_input = USER_INPUT.copy() + user_input["_controlled_vocabulary_file"] = test_cv_path + json.dump(user_input, input_file, sort_keys=True, indent=4) # read dataset info error_flag = cmor.dataset_json("Test/input_drs.json") @@ -157,33 +172,42 @@ def test_cv_path_and_file_templates(self): filepath, attrs = self.gen_cmor_file() - predicted_path = ('./{mip_era}/{activity_id}/{source_id}/{region}/' - '{frequency}/{experiment_id}/{variant_label}/' - '{variable_id}/{branding_suffix}/{grid_label}/' - '{version}/').format(**attrs) - predicted_file = ('{variable_id}_{branding_suffix}_{frequency}_' - '{region}_{grid_label}_{source_id}_{experiment_id}_' - '{variant_label}_201801-201802.nc').format(**attrs) + predicted_path_template = drs["directory_path_template"].replace("/","") + predicted_path_template = predicted_path_template.replace("<","{").replace(">","}") + predicted_path_template = predicted_path_template.replace("}{","}/{") + + predicted_file_template = drs["filename_template"] + predicted_file_template = predicted_file_template.replace("<","{").replace(">","}") + predicted_file_template = predicted_file_template.replace("}{","}_{") + + predicted_path = f'./{predicted_path_template.format(**attrs)}' + predicted_file = f'{predicted_file_template.format(**attrs)}_201801-201802.nc' predicted_filepath = os.path.join(predicted_path, predicted_file) - self.assertEqual(filepath, predicted_filepath) + + self.assertEqual(filepath.replace('//','/'), predicted_filepath) os.remove(filepath) - os.remove(CV_PATH) + os.remove(test_cv_path) def test_user_input_path_and_file_templates(self): + + test_name = "user_input_path_and_file_templates" + test_cv_path = f"TestTables/CMIP7_CV_{test_name}.json" + # Set up CMOR - cmor.setup(inpath="TestTables", + cmor.setup(inpath=CMIP7_TABLES_PATH, netcdf_file_action=cmor.CMOR_REPLACE) # Use "DRS" section from CV - with open("TestTables/CMIP7_CV.json", "r") as cv_infile: + with open(CV_PATH, "r") as cv_infile: cv = json.load(cv_infile) - with open(CV_PATH, "w") as cv_outfile: + with open(test_cv_path, "w") as cv_outfile: json.dump(cv, cv_outfile, sort_keys=True, indent=4) # Use directory path and file name templates from user input with open("Test/input_drs.json", "w") as input_file: - user_input = DATASET_INFO.copy() + user_input = USER_INPUT.copy() + user_input["_controlled_vocabulary_file"] = test_cv_path user_input["output_path_template"] = ( "" "" @@ -209,7 +233,7 @@ def test_user_input_path_and_file_templates(self): self.assertEqual(filepath, predicted_filepath) os.remove(filepath) - os.remove(CV_PATH) + os.remove(test_cv_path) if __name__ == '__main__': diff --git a/Test/test_cmor_time_value_and_bounds_mismatch.py b/Test/test_cmor_time_value_and_bounds_mismatch.py index b65c4c19..50d62a6a 100644 --- a/Test/test_cmor_time_value_and_bounds_mismatch.py +++ b/Test/test_cmor_time_value_and_bounds_mismatch.py @@ -5,11 +5,12 @@ import numpy import base_CMIP6_CV +CMIP7_TABLES_PATH = "cmip7-cmor-tables/tables" CV_PATH = "TestTables/CMIP7_CV.json" USER_INPUT = { - "_AXIS_ENTRY_FILE": "Tables/CMIP6_coordinate.json", - "_FORMULA_VAR_FILE": "Tables/CMIP6_formula_terms.json", + "_AXIS_ENTRY_FILE": "CMIP7_coordinate.json", + "_FORMULA_VAR_FILE": "CMIP7_formula_terms.json", "_cmip7_option": 1, "_controlled_vocabulary_file": CV_PATH, "activity_id": "CMIP", @@ -18,12 +19,12 @@ "branch_time_in_parent": 10800.0, "calendar": "360_day", "cv_version": "6.2.19.0", - "experiment": "1 percent per year increase in CO2", - "experiment_id": "1pctCO2", - "forcing_index": "f3", + "experiment": "Simulation of the pre-industrial climate", + "experiment_id": "piControl", + "forcing_index": "f30", "grid": "N96", "grid_label": "gn", - "initialization_index": "i1", + "initialization_index": "i000001d", "institution_id": "PCMDI", "license_id": "CC BY 4.0", "nominal_resolution": "250 km", @@ -35,12 +36,12 @@ "parent_experiment_id": "piControl", "parent_variant_label": "r1i1p1f3", "physics_index": "p1", - "realization_index": "r9", + "realization_index": "r009", "source_id": "PCMDI-test-1-0", "source_type": "AOGCM CHEM BGC", "tracking_prefix": "hdl:21.14100", "host_collection": "CMIP7", - "frequency": "xxxxxxxxxxxx", + "frequency": "mon", "region": "glb", "archive_id": "WCRP" } @@ -54,7 +55,7 @@ def setUp(self): frequency = "mon" self.input_path = f"Test/input_making_{frequency}_axis.json" - cmor.setup(inpath="TestTables", + cmor.setup(inpath=CMIP7_TABLES_PATH, netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) @@ -68,7 +69,7 @@ def setUp(self): if error_flag: raise RuntimeError("CMOR dataset_json call failed") - cmor.load_table("CMIP7_ocean2d.json") + cmor.load_table("CMIP7_ocean.json") def tearDown(self): os.remove(self.input_path) @@ -140,7 +141,7 @@ def setUp(self): frequency = "6hr" self.input_path = f"Test/input_making_{frequency}_axis.json" - cmor.setup(inpath="TestTables", + cmor.setup(inpath=CMIP7_TABLES_PATH, netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) @@ -154,7 +155,7 @@ def setUp(self): if error_flag: raise RuntimeError("CMOR dataset_json call failed") - cmor.load_table("CMIP7_ocean2d.json") + cmor.load_table("CMIP7_ocean.json") def tearDown(self): os.remove(self.input_path) @@ -209,7 +210,7 @@ def setUp(self): frequency = "yr" self.input_path = f"Test/input_making_{frequency}_axis.json" - cmor.setup(inpath="TestTables", + cmor.setup(inpath=CMIP7_TABLES_PATH, netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) @@ -223,7 +224,7 @@ def setUp(self): if error_flag: raise RuntimeError("CMOR dataset_json call failed") - cmor.load_table("CMIP7_ocean2d.json") + cmor.load_table("CMIP7_ocean.json") def tearDown(self): os.remove(self.input_path) @@ -318,7 +319,7 @@ def test_time_value_bounds_mismatch_times_passed(self): input_path = f"Test/input_{test_name}.json" frequency = "mon" - cmor.setup(inpath="TestTables", + cmor.setup(inpath=CMIP7_TABLES_PATH, netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) @@ -340,7 +341,7 @@ def test_time_value_bounds_mismatch_times_passed(self): time_bnds = numpy.array([0, 31, 60]) tos_shape = (time.shape[0], lat.shape[0], lon.shape[0]) tos = numpy.full(tos_shape, 27) - cmor.load_table("CMIP7_ocean2d.json") + cmor.load_table("CMIP7_ocean.json") cmorlat = cmor.axis("latitude", coord_vals=lat, cell_bounds=lat_bnds, diff --git a/TestTables/CMIP7_CV.json b/TestTables/CMIP7_CV.json index 697fe7c9..4e012762 100644 --- a/TestTables/CMIP7_CV.json +++ b/TestTables/CMIP7_CV.json @@ -6,6 +6,32 @@ "filename_example":"tas_tavg-h2m-hxy-u_mon_glb_gn_PCMDI-test-1-0_historical_r1i1p1f3_185001-186912.nc", "filename_template":"" }, + "activity_id":{ + "AerChemMIP":"Aerosols and Chemistry Model Intercomparison Project", + "C4MIP":"Coupled Climate Carbon Cycle Model Intercomparison Project", + "CDRMIP":"Carbon Dioxide Removal Model Intercomparison Project", + "CFMIP":"Cloud Feedback Model Intercomparison Project", + "CMIP":"CMIP DECK: 1pctCO2, abrupt4xCO2, amip, esm-piControl, esm-historical, historical, and piControl experiments", + "CORDEX":"Coordinated Regional Climate Downscaling Experiment", + "DAMIP":"Detection and Attribution Model Intercomparison Project", + "DCPP":"Decadal Climate Prediction Project", + "DynVarMIP":"Dynamics and Variability Model Intercomparison Project", + "FAFMIP":"Flux-Anomaly-Forced Model Intercomparison Project", + "GMMIP":"Global Monsoons Model Intercomparison Project", + "GeoMIP":"Geoengineering Model Intercomparison Project", + "HighResMIP":"High-Resolution Model Intercomparison Project", + "ISMIP6":"Ice Sheet Model Intercomparison Project for CMIP6", + "LS3MIP":"Land Surface, Snow and Soil Moisture", + "LUMIP":"Land-Use Model Intercomparison Project", + "OMIP":"Ocean Model Intercomparison Project", + "PAMIP":"Polar Amplification Model Intercomparison Project", + "PMIP":"Palaeoclimate Modelling Intercomparison Project", + "RFMIP":"Radiative Forcing Model Intercomparison Project", + "SIMIP":"Sea Ice Model Intercomparison Project", + "ScenarioMIP":"Scenario Model Intercomparison Project", + "VIACSAB":"Vulnerability, Impacts, Adaptation and Climate Services Advisory Board", + "VolMIP":"Volcanic Forcings Model Intercomparison Project" + }, "archive_id":{ "WCRP":"a collection of datasets from the AMIP and CMIP project phases, along with project supporting datasets from the input4MIPs (forcing datasets used to drive CMIP simulations) and obs4MIPs (observational datasets used to evaluate CMIP simulations, and numerous other supporting activities" }, @@ -38,33 +64,56 @@ }, "branding_suffix":"", "data_specs_version":"CMIP-7.0.0.0", - "experiment_id":{ - "1pctCO2":{ - "activity_id":[ + "experiment_id": { + "historical": { + "activity_id": [ "CMIP" ], - "additional_allowed_model_components":[ + "additional_allowed_model_components": [ "AER", "CHEM", "BGC" ], - "description":"DECK: 1pctCO2", - "end_year":"", - "experiment":"1 percent per year increase in CO2", - "experiment_id":"1pctCO2", - "host_collection":"CMIP7", - "min_number_yrs_per_sim":"150", - "parent_activity_id":[ + "description": "DECK: historical simulation of the recent past (TODO: add details)", + "end_year": "2021 (TODO make clear it is end of 2021, not start)", + "experiment": "Simulation of the recent past", + "experiment_id": "historical", + "host_collection": "CMIP7", + "min_number_yrs_per_sim": 171, + "parent_activity_id": [ "CMIP" ], - "parent_experiment_id":[ + "parent_experiment_id": [ "piControl" ], - "required_model_components":[ + "required_model_components": [ + "AOGCM" + ], + "start_year": "1850", + "tier": 0 + }, + "piControl": { + "activity_id": [ + "CMIP" + ], + "additional_allowed_model_components": [ + "AER", + "CHEM", + "BGC" + ], + "description": "DECK: pre-industrial control simulation (TODO: add details)", + "end_year": "", + "experiment": "Simulation of the pre-industrial climate", + "experiment_id": "piControl", + "host_collection": "CMIP7", + "min_number_yrs_per_sim": 150, + "parent_activity_id": [], + "parent_experiment_id": [], + "required_model_components": [ "AOGCM" ], - "start_year":"", - "tier":"1" + "start_year": "", + "tier": 0 } }, "frequency":{ diff --git a/cmip7-cmor-tables b/cmip7-cmor-tables new file mode 160000 index 00000000..48e5bdfc --- /dev/null +++ b/cmip7-cmor-tables @@ -0,0 +1 @@ +Subproject commit 48e5bdfc1fd1c0f16925e58d9935cfce1989cba4