Merge pull request #93 from oberonia78/s1c_update

S1c update

Author: oberonia78

.github/workflows/test.yml

@@ -41,30 +41,33 @@ jobs:
 
       # Set the conda environment up using Mamba and install dependencies
       - name: Setup Environment
-        uses: mamba-org/setup-micromamba@main
+        uses: mamba-org/setup-micromamba@v2
         with:
           environment-file: ${{ matrix.env-file.file }}
           environment-name: RTC
+          init-shell: bash                          # let the action touch ~/.bash_profile
+          log-level: debug                          # shows solver problems early
+          post-cleanup: none
 
       # Install the S1-Reader OPERA-ADT project.
       - name: Install S1-Reader
         run: |
           curl -sSL \
-            https://github.com/opera-adt/s1-reader/archive/refs/tags/v0.2.2.tar.gz \
+            https://github.com/opera-adt/s1-reader/archive/refs/tags/v0.2.5.tar.gz \
             -o s1_reader_src.tar.gz \
           && tar -xvf s1_reader_src.tar.gz \
-          && ln -s s1-reader-0.2.2 s1-reader \
+          && ln -s s1-reader-0.2.5 s1-reader \
           && rm s1_reader_src.tar.gz \
-          && python -m pip install ./s1-reader      
+          && python -m pip install -e ./s1-reader      
 
       # Setup the project
       - name: Install Project
         run: python -m pip install .
 
-      # Test the project.
+      # # Test the project.
       - name: Test Project
         run: |
-          pytest -vrpP
+          pytest -vrpP tests/
 
   build_docker:
 
@@ -100,4 +103,4 @@ jobs:
       # Build the image.
       - name: Build docker image
         run: |
-          sh build_docker_image.sh
+          bash build_docker_image.sh

Docker/Dockerfile

@@ -1,4 +1,9 @@
-FROM oraclelinux:8
+ARG BASE_IMAGE=oraclelinux:8.8
+
+FROM ${BASE_IMAGE}
+
+# ARG is out of scope here unless re-declared
+ARG BASE_IMAGE
 
 LABEL author="OPERA ADT" \
     description="RTC cal/val release R4" \
@@ -37,9 +42,9 @@ SHELL ["conda", "run", "-n", "RTC", "/bin/bash", "-c"]
 WORKDIR /home/rtc_user/OPERA
 
 # installing OPERA s1-reader
-RUN curl -sSL https://github.com/isce-framework/s1-reader/archive/refs/tags/v0.2.4.tar.gz -o s1_reader_src.tar.gz &&\
+RUN curl -sSL https://github.com/isce-framework/s1-reader/archive/refs/tags/v0.2.5.tar.gz -o s1_reader_src.tar.gz &&\
     tar -xvf s1_reader_src.tar.gz &&\
-    ln -s s1-reader-0.2.4 s1-reader &&\
+    ln -s s1-reader-0.2.5 s1-reader &&\
     rm s1_reader_src.tar.gz &&\
     python -m pip install ./s1-reader
 

Docker/environment.yml

@@ -1,15 +1,15 @@
 name: rtc_s1_sas_final
 channels:
   - conda-forge
+  - nodefaults
 dependencies:
   - python>=3.9,<3.10
   - gdal>=3.0
-  - s1reader>=0.2.4
+  - s1reader>=0.2.5
   - numpy>=1.20
   - pybind11>=2.5
   - pyre>=1.11.2
-  
   - scipy!=1.10.0
   - isce3==0.15.0
   # Workaround for the issue with `libabseil` (09/11/2023)
-  - libabseil=20230125.3
+  - libabseil=20230125.3

Docker/lockfile.lock

@@ -155,5 +155,4 @@ https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.7.1-h880a63b_9.conda
 https://conda.anaconda.org/conda-forge/noarch/pooch-1.7.0-pyha770c72_3.conda
 https://conda.anaconda.org/conda-forge/linux-64/gdal-3.7.1-py39h41b90d8_9.conda
 https://conda.anaconda.org/conda-forge/linux-64/scipy-1.11.2-py39h6183b62_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/isce3-0.15.0-py39h431996e_0.conda
-
+https://conda.anaconda.org/conda-forge/linux-64/isce3-0.15.0-py39h431996e_0.conda

build_docker_image.sh

@@ -4,6 +4,15 @@ REPO=opera
 IMAGE=rtc
 TAG=final_1.0.4
 
+docker_build_args=(--rm --force-rm --network host -t $REPO/$IMAGE:$TAG -f Docker/Dockerfile)
+
+if [ $# -eq 0 ]; then
+    echo "Base image was not specified. Using the default image specified in the Dockerfile."
+else
+    echo "Using $1 as the base image."
+    docker_build_args+=(--build-arg BASE_IMAGE=$1)
+fi
+
 echo "IMAGE is $REPO/$IMAGE:$TAG"
 
 # fail on any non-zero exit codes
@@ -13,8 +22,7 @@ set -ex
 python3 setup.py sdist
 
 # build image
-docker build --rm --force-rm --network host -t $REPO/$IMAGE:$TAG -f Docker/Dockerfile .
-
+docker build "${docker_build_args[@]}" .
 # run tests - to be worked on when the RTC test module is in place
 #docker run --rm -u "$(id -u):$(id -g)" -v "$PWD:/mnt" -w /mnt -it --network host "${IMAGE}:$t" pytest /mnt/tests/
 

tests/test_rtc_s1_workflow.py

@@ -4,6 +4,8 @@
 import requests
 import tarfile
 from osgeo import gdal
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 
 from rtc.runconfig import RunConfig, load_parameters
 from rtc.core import create_logger
@@ -149,28 +151,35 @@ def test_workflow():
         os.makedirs(test_data_directory, exist_ok=True)
 
     dataset_name = 's1b_los_angeles'
-    dataset_url = ('https://zenodo.org/record/7753472/files/'
+    dataset_url = ('https://zenodo.org/records/7753472/files/'
                    's1b_los_angeles.tar.gz?download=1')
 
     tests_dir = os.path.dirname(__file__)
     dataset_dir = os.path.join(test_data_directory, dataset_name)
     if FLAG_ALWAYS_DOWNLOAD or not os.path.isdir(dataset_dir):
 
-
-
         print(f'Test dataset {dataset_name} not found. Downloading'
               f' file {dataset_url}.')
-        response = requests.get(dataset_url)
-        response.raise_for_status()
-
-        compressed_filename = os.path.join(test_data_directory,
-                                           os.path.basename(dataset_url))
-
-        open(compressed_filename, 'wb').write(response.content)
-
-        print(f'Extracting downloaded file {compressed_filename}')
-        with tarfile.open(compressed_filename) as compressed_file:
-            compressed_file.extractall(test_data_directory)
+        # To avoid the issue in downloading, try again. 
+        session = requests.Session()
+        retries = Retry(
+            total=5,                    # up to 5 attempts
+            backoff_factor=2,           # 2 s, 4 s, 8 s, …
+            status_forcelist=[502, 503, 504],
+        )
+        session.mount("https://", HTTPAdapter(max_retries=retries))
+
+        compressed_path = os.path.join(test_data_directory,
+                                       f"{dataset_name}.tar.gz")
+        with session.get(dataset_url, stream=True, timeout=120) as r:
+            r.raise_for_status()
+            with open(compressed_path, "wb") as f:
+                for chunk in r.iter_content(chunk_size=1024 * 1024):  # 1 MB
+                    f.write(chunk)
+
+        print(f"Extracting {compressed_path}")
+        with tarfile.open(compressed_path, "r:gz") as tf:
+            tf.extractall(test_data_directory)
 
     # create logger
     log_file = os.path.join('data', 'log.txt')