diff --git a/.circleci/config.yml b/.circleci/config.yml index dd039e0c5..88c46c994 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,22 +12,20 @@ jobs: - checkout - run: - name: Install MiniForge + name: Install micromamba shell: /bin/bash -xl command: | apt update --yes && apt upgrade --yes apt install --yes --no-install-recommends wget ca-certificates git - cd ${HOME} - wget -O Miniforge3.sh \ - "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" - bash Miniforge3.sh -b -p "${HOME}/conda" - - # Put mamba's shell init into a profile.d file, in order to activate - # it even in non-interactive terminals - "${HOME}/conda/bin/mamba" shell hook --shell bash --root-prefix "${HOME}/conda" \ - > /etc/profile.d/mamba-raider-circleci.sh - echo "mamba activate RAiDER" >> /etc/profile.d/mamba-raider-circleci.sh + # Put micromamba's shell init script into a profile.d file instead + # of .bashrc, in order to activate it even in CircleCI's + # non-interactive terminals. + mv ${HOME}/.bashrc ${HOME}/.bashrc.tmp &> /dev/null + # From https://mamba.readthedocs.io/en/latest/installation/micromamba-installation.html#automatic-install + curl -L micro.mamba.pm/install.sh | "${SHELL}" + mv ${HOME}/.bashrc /etc/profile.d/mamba-raider-circleci.sh + mv ${HOME}/.bashrc.tmp ${HOME}/.bashrc &> /dev/null - run: name: Create RAiDER environment @@ -35,8 +33,10 @@ jobs: shell: /bin/bash -xl command: | PYTHON_VERSION="<< parameters.python-version >>" - sed -i "/python>=/c\ - python=${PYTHON_VERSION}" environment.yml - mamba env create -f environment.yml + # Create RAiDER enivronment + micromamba env create --yes -f environment.yml python=${PYTHON_VERSION} + # Activate the RAiDER env for the rest of the script's shells + echo "micromamba activate RAiDER" >> /etc/profile.d/mamba-raider-circleci.sh - run: name: Install RAiDER and check environment @@ -69,6 +69,7 @@ jobs: name: Report coverage shell: /bin/bash -xl command: | + # Only run the coverage test on the latest supported version (3.12) PYTHON_VERSION="<< parameters.python-version >>" if [ "${PYTHON_VERSION}" == "3.12" ]; then python -m pip install coveralls diff --git a/CHANGELOG.md b/CHANGELOG.md index 58f1af55a..f5c54a946 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,8 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). * [743](https://github.com/dbekaert/RAiDER/pull/743) - Switched from HTTPS to DAP4 for retrieving MERRA2 data, and suppressed a warning for using DAP4 for GMAO data where doing so is not possible. ### Fixed -* [759](https://github.com/dbekaert/RAiDER/pull/759) - Added a `browse` S3 tag for `.png` files when uploaded to AWS. +* [759](https://github.com/dbekaert/RAiDER/pull/759) - Added a `browse_images` S3 tag for `.png` files when uploaded to AWS. +* [757](https://github.com/dbekaert/RAiDER/pull/757) - Switched to micromamba in CircleCI tests to fix configuration issue experienced with MiniForge. * [741](https://github.com/dbekaert/RAiDER/pull/741) - Updated mamba setup commands in CircleCI for mamba 2.0.0. * [738](https://github.com/dbekaert/RAiDER/pull/738) - Resolved an InsecureRequestWarning when fetching data through CDS, such as for ERA5. * [734](https://github.com/dbekaert/RAiDER/pull/734) - Resolved a deprecation warning from Rasterio about the use of `statistics()`. diff --git a/environment.yml b/environment.yml index 469942781..058c9d643 100644 --- a/environment.yml +++ b/environment.yml @@ -9,62 +9,62 @@ channels: - nodefaults dependencies: - python>=3.8 - - pip + - pip>=25 # For running - - asf_search - - boto3 + - asf_search>=9 + - boto3>=1 - cartopy - cdsapi - cfgrib - dask - dem_stitcher>=2.5.8 - - ecmwf-api-client - - h5netcdf - - h5py + - ecmwf-api-client>=1 + - h5netcdf>=1 + - h5py>=2 - herbie-data<2025.2.1 - isce3>=0.15.0 - - jsonschema==3.2.0 # this is for ASF DAAC ingest schema validation - - lxml - - matplotlib - - netcdf4 - - numpy - - pandas - - progressbar - - pydap>3.2.2 + - jsonschema>=3.2.0 # this is for ASF DAAC ingest schema validation + - lxml>=6 + - matplotlib>=3 + - netcdf4>=1 + - numpy>=1 + - pandas>=2 + - progressbar>=2 + - pydap>=3.2.3 - pyproj>=2.2.0 - - pyyaml + - pyyaml>=6 - rasterio>=1.3.0 - - requests + - requests>=2 - rioxarray - s3fs - - scipy>1.10.0 + - scipy>=1.10.1 - s1_orbits - - shapely - - tqdm + - shapely>=2 + - tqdm>=4 - xarray # For packaging and testing - - autopep8 - - cmake - - cxx-compiler - - cython - - pybind11 - - pytest - - pytest-console-scripts - - pytest-cov - - pytest-mock - - pytest-timeout + - autopep8>=2 + - cmake>=4 + - cxx-compiler>=1 + - cython>=3 + - pybind11>=2 + - pytest>=8 + - pytest-console-scripts>=1 + - pytest-cov>=6 + - pytest-mock>=3 + - pytest-timeout>=2 - setuptools_scm>=6.2 - - sysroot_linux-64 + - sysroot_linux-64>=2 # For docs website - - mkdocs - - mkdocs-macros-plugin - - mkdocs-material - - mkdocs-material-extensions + - mkdocs>=1 + - mkdocs-macros-plugin>=1 + - mkdocs-material>=9 + - mkdocs-material-extensions>=1 - mkdocstrings - - mkdocstrings-python + - mkdocstrings-python>=1 # For RAiDER-docs - jupyter_contrib_nbextensions - - jupyterlab + - jupyterlab>=3 - wand # For development - - boto3-stubs + - boto3-stubs>=1 diff --git a/test/gunw_test_data/gunw_schema.json b/test/gunw_test_data/gunw_schema.json index c05028d91..b8288e3d0 100644 --- a/test/gunw_test_data/gunw_schema.json +++ b/test/gunw_test_data/gunw_schema.json @@ -1,187 +1,212 @@ { - "type": "object", - "required": [ - "label", - "location", - "metadata", - "creation_timestamp", - "version" - ], - "properties": { - "label": { - "type": "string", - "minLength": 1 - }, - "creation_timestamp": { - "$ref": "#/definitions/iso_datetime" - }, - "version": { - "type": "string" - }, - "location": { - "type": "object", - "required": [ - "type", - "coordinates" - ], - "properties": { - "type": { - "type": "string", - "enum": ["Polygon"] - }, - "coordinates": { - "type": "array", - "minItems": 1, - "items": { - "type": "array", - "minItems": 4, - "items": { - "$ref": "#/definitions/coordinate" - } - } - } - } - }, - "metadata": { - "type": "object", - "required": [ - "ogr_bbox", - "reference_scenes", - "secondary_scenes", - "sensing_start", - "sensing_stop", - "orbit_number", - "platform", - "beam_mode", - "orbit_direction", - "dataset_type", - "product_type", - "polarization", - "look_direction", - "track_number", - "perpendicular_baseline" - ], - "properties": { - "ogr_bbox": { + "type": "object", + "required": [ + "label", + "location", + "metadata", + "creation_timestamp", + "version" + ], + "properties": { + "label": { + "type": "string", + "minLength": 1 + }, + "creation_timestamp": { + "$ref": "#/definitions/iso_datetime" + }, + "version": { + "type": "string" + }, + "location": { + "type": "object", + "required": [ + "type", + "coordinates" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "Polygon" + ] + }, + "coordinates": { + "type": "array", + "minItems": 1, + "items": { "type": "array", "minItems": 4, - "maxItems": 4, "items": { - "#ref": "#/definitions/coordinate" + "$ref": "#/definitions/coordinate" } - }, - "reference_scenes": { - "$ref": "#/definitions/granule_list" - }, - "secondary_scenes": { - "$ref": "#/definitions/granule_list" - }, - "sensing_start": { - "$ref": "#/definitions/iso_datetime" - }, - "sensing_stop": { - "$ref": "#/definitions/iso_datetime" - }, - "orbit_number": { - "type": "array", - "minItems": 1, - "items": { - "type": "integer", - "minimum": 1 - } - }, - "platform": { - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "enum": ["Sentinel-1A", "Sentinel-1B"] - } - }, - "beam_mode": { - "type": "string", - "enum": ["IW", "EW", "WV", "S1", "S2", "S3", "S4", "S5", "S6"] - }, - "orbit_direction": { - "type": "string", - "enum": ["ascending", "descending"] - }, - "dataset_type": { - "type": "string" - }, - "product_type": { - "type": "string" - }, - "polarization": { - "type": "string", - "enum": ["VV", "HH", "VV+VH", "HH+HV"] - }, - "look_direction": { - "type": "string", - "enum": ["right", "left"] - }, - "track_number": { - "type": "integer", - "minimum": 1, - "maximum": 175 - }, - "perpendicular_baseline": { - "type": "number" - }, - "weather_model": { - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "enum": [ - "ERA5", - "ERA5T", - "GMAO", - "HRES", - "HRRR" - ] - } - }, - "temporal_baseline_days": { - "type": "integer", - "minimum": 1 - }, - "frame_number": { - "type": "integer", - "minimum": 0 } } } }, - "definitions": { - "granule_list": { - "type": "array", - "minItems": 1, - "items": { + "metadata": { + "type": "object", + "required": [ + "ogr_bbox", + "reference_scenes", + "secondary_scenes", + "sensing_start", + "sensing_stop", + "orbit_number", + "platform", + "beam_mode", + "orbit_direction", + "dataset_type", + "product_type", + "polarization", + "look_direction", + "track_number", + "perpendicular_baseline" + ], + "properties": { + "ogr_bbox": { + "type": "array", + "minItems": 4, + "maxItems": 4, + "items": { + "#ref": "#/definitions/coordinate" + } + }, + "reference_scenes": { + "$ref": "#/definitions/granule_list" + }, + "secondary_scenes": { + "$ref": "#/definitions/granule_list" + }, + "sensing_start": { + "$ref": "#/definitions/iso_datetime" + }, + "sensing_stop": { + "$ref": "#/definitions/iso_datetime" + }, + "orbit_number": { + "type": "array", + "minItems": 1, + "items": { + "type": "integer", + "minimum": 1 + } + }, + "platform": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "Sentinel-1A", + "Sentinel-1B" + ] + } + }, + "beam_mode": { "type": "string", - "minLength": 1 - } - }, - "coordinate": { - "type": "array", - "minItems": 2, - "maxItems": 2, - "items": [ - { - "type": "number", - "minimum": -180, - "maximum": 180 - }, - { - "type": "number", - "minimum": -90, - "maximum": 90 + "enum": [ + "IW", + "EW", + "WV", + "S1", + "S2", + "S3", + "S4", + "S5", + "S6" + ] + }, + "orbit_direction": { + "type": "string", + "enum": [ + "ascending", + "descending" + ] + }, + "dataset_type": { + "type": "string" + }, + "product_type": { + "type": "string" + }, + "polarization": { + "type": "string", + "enum": [ + "VV", + "HH", + "VV+VH", + "HH+HV" + ] + }, + "look_direction": { + "type": "string", + "enum": [ + "right", + "left" + ] + }, + "track_number": { + "type": "integer", + "minimum": 1, + "maximum": 175 + }, + "perpendicular_baseline": { + "type": "number" + }, + "weather_model": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "ERA5", + "ERA5T", + "GMAO", + "HRES", + "HRRR" + ] } - ] - }, - "iso_datetime": { + }, + "temporal_baseline_days": { + "type": "integer", + "minimum": 1 + }, + "frame_number": { + "type": "integer", + "minimum": 0 + } + } + } + }, + "definitions": { + "granule_list": { + "type": "array", + "minItems": 1, + "items": { "type": "string", - "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(3[0-1]|[1-2][0-9]|0[1-9])T(2[0-3]|[0-1][0-9]):[0-5][0-9]:[0-5][0-9]\\.[0-9]{6}Z$" + "minLength": 1 } + }, + "coordinate": { + "type": "array", + "prefixItems": [ + { + "type": "number", + "minimum": -180, + "maximum": 180 + }, + { + "type": "number", + "minimum": -90, + "maximum": 90 + } + ], + "items": false + }, + "iso_datetime": { + "type": "string", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(3[0-1]|[1-2][0-9]|0[1-9])T(2[0-3]|[0-1][0-9]):[0-5][0-9]:[0-5][0-9]\\.[0-9]{6}Z$" } - } \ No newline at end of file + } +} \ No newline at end of file diff --git a/test/test_temporal_interpolate.py b/test/test_temporal_interpolate.py index d738ee383..5000e37db 100644 --- a/test/test_temporal_interpolate.py +++ b/test/test_temporal_interpolate.py @@ -2,7 +2,6 @@ import glob import shutil import os -import subprocess import numpy as np import xarray as xr @@ -13,6 +12,7 @@ from RAiDER.logger import logger from RAiDER.utilFcns import write_yaml +from RAiDER.cli.raider import calcDelays wm = 'ERA5' if WM == 'ERA-5' else WM @@ -42,19 +42,13 @@ def test_cube_timemean(): grp['time_group'].update({'time': f'{hr}:00:00'}) ## generate the default run config file and overwrite it with new parms cfg = write_yaml(grp, 'temp.yaml') - ## run raider for the default date - cmd = f'raider.py {cfg}' - proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True) - assert np.isclose(proc.returncode, 0) + calcDelays([str(cfg)]) ## run interpolation in the middle of the two grp['time_group'] = {'time': ti, 'interpolate_time': 'center_time'} cfg = write_yaml(grp, 'temp.yaml') - - cmd = f'raider.py {cfg}' - proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True) - assert np.isclose(proc.returncode, 0) + calcDelays([str(cfg)]) with xr.open_dataset(os.path.join(SCENARIO_DIR, f'{WM}_tropo_{date}T{hr1}0000_ztd.nc')) as ds: @@ -106,16 +100,12 @@ def test_cube_weighting(): cfg = write_yaml(grp, 'temp.yaml') ## run raider for the default date - cmd = f'raider.py {cfg}' - proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True) - assert np.isclose(proc.returncode, 0) + calcDelays([str(cfg)]) ## run interpolation very near the first grp['time_group'] = {'time': ti, 'interpolate_time': 'center_time'} cfg = write_yaml(grp, 'temp.yaml') - - cmd = f'raider.py {cfg}' - proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True) + calcDelays([str(cfg)]) ## double check on weighting diff --git a/tools/RAiDER/models/merra2.py b/tools/RAiDER/models/merra2.py index c35f0b4e9..8416e95e9 100755 --- a/tools/RAiDER/models/merra2.py +++ b/tools/RAiDER/models/merra2.py @@ -109,7 +109,7 @@ def _fetch(self, out: Path) -> None: + '.nc4' ) - stream = pydap.client.open_url(url) + stream = xr.open_dataset(url, decode_times=False, engine='pydap') q = stream['QV'][0, :, lat_min_ind : lat_max_ind + 1, lon_min_ind : lon_max_ind + 1].data.squeeze() p = stream['PL'][0, :, lat_min_ind : lat_max_ind + 1, lon_min_ind : lon_max_ind + 1].data.squeeze()