Feature/fix sql ingestions

.github/workflows/lint.yml

@@ -0,0 +1,33 @@
+name: Lint
+
+on:
+  push:
+    branches: [ '**' ]
+  pull_request:
+    branches: [ '**' ]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Build Docker image
+      run: |
+        docker build -f Dockerfile.github -t sen2like-lint .
+
+    - name: Run linting tools in Docker
+      run: |
+        docker run sen2like-lint bash -c "
+          echo '=== Running Black check ===' &&
+          black --check --diff --line-length=120 . &&
+          echo '=== Running isort check ===' &&
+          isort --check --diff --profile black . &&
+          echo '=== Running Flake8 ===' &&
+          flake8 --max-line-length=120 --statistics . &&
+          echo '=== Running Bandit ===' &&
+          bandit -r --severity-level high . &&
+          echo '=== Running Pylint ===' &&
+          pylint --extension-pkg-whitelist=numpy,cv2 --max-line-length=120 -j 1 sen2like/**/*.py
+        "

.github/workflows/test.yml

@@ -0,0 +1,46 @@
+name: Unit Tests
+
+on:
+  push:
+    branches: [ '**' ]
+  pull_request:
+    branches: [ '**' ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Build Docker image
+      run: |
+        docker build -f Dockerfile.github -t sen2like-test .
+
+    - name: Run tests with coverage in Docker
+      run: |
+        docker run sen2like-test bash -c "
+          cd /app/sen2like &&
+          echo '=== Running tests with coverage ===' &&
+          [ -d reports ] || mkdir reports &&
+          PYTHONPATH=/app/sen2like:/app/sen2like/sen2like:/app/sen2like/sen2like/aux_data coverage run --source=sen2like --branch -m pytest -v -k 'not test_landsat and not test_s2_l1c and not test_s2_l2a and not test_sentinel2_maja and not test_landsat_maja and not test_S2B_band01 and not test_S2B_band09 and not test_landsat_intercalibration' --junitxml=reports/junit.xml &&
+          coverage xml -o reports/coverage.xml &&
+          coverage report -m > reports/coverage.report &&
+          echo '=== Test coverage report ===' &&
+          cat reports/coverage.report
+        "
+
+    - name: Upload test results
+      uses: actions/upload-artifact@v4
+      if: success() || failure()
+      with:
+        name: test-results
+        path: sen2like/reports/
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v4
+      with:
+        file: ./sen2like/reports/coverage.xml
+        flags: unittests
+        name: codecov-umbrella
+        fail_ci_if_error: false

Dockerfile.github

@@ -0,0 +1,35 @@
+FROM docker.io/continuumio/miniconda3
+
+# Install system dependencies including GDAL
+RUN apt-get update && \
+    apt-get install -y gdal-bin libgdal-dev build-essential && \
+    rm -rf /var/lib/apt/lists/*
+
+# Copy the environment file
+COPY sen2like/environment.yml /tmp/environment.yml
+
+# Create the conda environment
+RUN conda env create -f /tmp/environment.yml
+
+# Make RUN commands use the new environment
+SHELL ["conda", "run", "-n", "sen2like", "/bin/bash", "-c"]
+
+# Activate the environment by default
+ENV CONDA_DEFAULT_ENV=sen2like
+ENV PATH="/opt/conda/envs/sen2like/bin:${PATH}"
+
+# Copy the project
+WORKDIR /app
+COPY . .
+
+# Install test and linting dependencies using conda
+RUN conda run -n sen2like bash -c "cd /app/sen2like && conda install -c conda-forge --file requirements_dev.txt && conda install --file requirements_tests.txt"
+
+# Set PYTHONPATH to include the project
+ENV PYTHONPATH=/app/sen2like:/app/sen2like/aux_data:/app:$PYTHONPATH
+
+# Set working directory to the sen2like project
+WORKDIR /app/sen2like
+
+# Default command
+CMD ["bash", "-c", "echo 'Docker image ready for linting and testing'"]

prisma4sen2like/prisma/adapter.py

@@ -35,6 +35,7 @@
 from numpy.typing import NDArray
 from PIL import Image
 from prisma_product import PrismaProduct
+
 from sen2like.image_file import S2L_ImageFile
 
 logger = logging.getLogger(__name__)

prisma4sen2like/prisma/geometry.py

@@ -21,6 +21,7 @@
 from dataclasses import asdict, dataclass
 
 from osgeo import gdal
+
 from sen2like.grids import MGRSGeoInfo
 
 logger = logging.getLogger()

prisma4sen2like/prisma/mgrs_util.py

@@ -23,9 +23,10 @@
 """
 import mgrs
 from geometry import LatLong
-from sen2like.grids import GridsConverter, MGRSGeoInfo
 from shapely.wkt import loads
 
+from sen2like.grids import GridsConverter, MGRSGeoInfo
+
 _MGRS = mgrs.MGRS()
 
 

prisma4sen2like/prisma/sen2like/grids.py

@@ -40,8 +40,9 @@ def __init__(self):
     def _get_roi(self, tilecode):
         # search tilecode in "s2tiles" and return row as a pandas dataframe
         return pd.read_sql_query(
-            f'SELECT TILE_ID, EPSG, UTM_WKT, MGRS_REF, LL_WKT FROM s2tiles WHERE TILE_ID="{tilecode}"',  # nosec B608
+            "SELECT TILE_ID, EPSG, UTM_WKT, MGRS_REF, LL_WKT FROM s2tiles WHERE TILE_ID=?",
             self.conn,
+            params=(tilecode,),
         )
 
     def close(self):

sen2like/README.md

@@ -110,7 +110,7 @@ python sen2like.py
 That should display
 
 ```
-Sen2like 4.5.2 with Python 3.12.7 | packaged by conda-forge | (main, Oct  4 2024, 15:47:54) [MSC v.1941 64 bit (AMD64)]
+Sen2like 4.5.3 with Python 3.12.7 | packaged by conda-forge | (main, Oct  4 2024, 15:47:54) [MSC v.1941 64 bit (AMD64)]
 usage: sen2like.py [-h] [--version] {product-mode,single-tile-mode,multi-tile-mode,roi-based-mode} ...
 
 positional arguments:
@@ -211,7 +211,7 @@ You can run it directly without entering into the container:
 ```bash
 docker run --rm my-internal-docker-registry-url/sen2like/sen2like:4.5
 
-Sen2like 4.5.2 with Python 3.12.7 | packaged by conda-forge | (main, Oct  4 2024, 15:47:54) [MSC v.1941 64 bit (AMD64)]
+Sen2like 4.5.3 with Python 3.12.7 | packaged by conda-forge | (main, Oct  4 2024, 15:47:54) [MSC v.1941 64 bit (AMD64)]
 usage: sen2like.py [-h] [--version] {product-mode,single-tile-mode,multi-tile-mode,roi-based-mode} ...
 
 positional arguments:

sen2like/aux_data/dem/dem_downloader.py

@@ -222,7 +222,7 @@ def compute_tile_extent(self) -> list[tuple[str, str]]:
             LOGGER.debug(latitudes)
 
             # For now assume we never have this case.
-            # to fix it, retrieve code history to see how it was made, 
+            # to fix it, retrieve code history to see how it was made,
             # but it was not working well
             self.cross_dateline = False
             lon_min = int(lon_min)
@@ -233,13 +233,9 @@ def compute_tile_extent(self) -> list[tuple[str, str]]:
                 longitudes = range(lon_min, lon_max)
             LOGGER.debug(longitudes)
 
-            latitudes = [
-                f"N{latitude}" if (0 < latitude <= 90) else f"S{abs(latitude-1)}"
-                for latitude in latitudes
-            ]
+            latitudes = [f"N{latitude}" if (0 < latitude <= 90) else f"S{abs(latitude-1)}" for latitude in latitudes]
             longitudes = [
-                f"E{longitude:03}" if (0 < longitude <= 180) else f"W{abs(longitude-1):03}"
-                for longitude in longitudes
+                f"E{longitude:03}" if (0 < longitude <= 180) else f"W{abs(longitude-1):03}" for longitude in longitudes
             ]
             LOGGER.debug(latitudes)
             LOGGER.debug(longitudes)
@@ -279,9 +275,7 @@ def resolve_dem_file_urls(self, locations: list[tuple[str, str]], local=True) ->
 
         if locations is not None:
             for latitude, longitude in locations:
-                dem_url = url.format(
-                    dem_dataset_name=self.config.dem_dataset_name, dem_product_name=dem_product_name
-                )
+                dem_url = url.format(dem_dataset_name=self.config.dem_dataset_name, dem_product_name=dem_product_name)
                 # format `{dem_product_name}` put in dem_url by previous instruction
                 dem_url = dem_url.format(arcsec=arcsec, latitude=latitude, longitude=longitude)
 
@@ -350,9 +344,7 @@ def _create_dem_mosaic(self, dem_mosaic_file: str, dem_files: list[str]):
             if self.cross_dateline:
                 gdal.SetConfigOption("CENTER_LONG", "180")
 
-            options = gdal.WarpOptions(
-                dstNodata=no_data, outputType=gdal.GDT_Int16, dstSRS="EPSG:4326"
-            )
+            options = gdal.WarpOptions(dstNodata=no_data, outputType=gdal.GDT_Int16, dstSRS="EPSG:4326")
 
             dataset = gdal.Warp(dem_mosaic_file, dem_files, options=options)
 

sen2like/conf/config.ini

@@ -26,7 +26,7 @@ scl_dir = /data/AUX_DATA/SCL_maps_2.10
 [InputProductArchive]
 # global InputProductArchive params
 coverage = 0.5
-cloud_cover = 11
+cloud_cover = 20
 
 ##################################################################
 # Local only
@@ -36,15 +36,15 @@ url_parameters_pattern_Landsat8 = {base_url}/{mission}/{path}/{row}
 url_parameters_pattern_Landsat9 = {base_url}/{mission}/{path}/{row}
 
 ##################################################################
-# Creodias only
-; base_url = https://datahub.creodias.eu/resto/api/collections
-; location_Landsat8 = path={path}&row={row}&productType=L1TP
-; location_Landsat9 = path={path}&row={row}&productType=L1TP
-; location_Sentinel2 = productIdentifier=%25T{tile}%25
-; url_parameters_pattern = {base_url}/{mission}/search.json?maxRecords=1000&cloudCover=%5B0%2C{cloud_cover}%5D&startDate={start_date}&completionDate={end_date}&sortParam=startDate&sortOrder=ascending&status=all&{location}&dataset=ESA-DATASET
-; thumbnail_property = properties/productIdentifier
-; cloud_cover_property = properties/cloudCover
-; gml_geometry_property = properties/gmlgeometry
+# Creodias only (ODATA)
+;base_url = https://datahub.creodias.eu/odata/v1/Products?$filter=(
+;location_Sentinel2 = ((Collection/Name eq 'SENTINEL-2')) and (contains(Name,%27{tile}%27))
+;location_Landsat8 = ((Collection/Name eq 'LANDSAT-8')) and (Attributes/OData.CSC.StringAttribute/any(i0:i0/Name eq %27wrsPath%27 and i0/Value eq %27{path}%27)) and (Attributes/OData.CSC.StringAttribute/any(i0:i0/Name eq %27wrsRow%27 and i0/Value eq %27{row}%27))
+;location_Landsat9 = ((Collection/Name eq 'LANDSAT-9')) and (Attributes/OData.CSC.StringAttribute/any(i0:i0/Name eq %27wrsPath%27 and i0/Value eq %27{path}%27)) and (Attributes/OData.CSC.StringAttribute/any(i0:i0/Name eq %27wrsRow%27 and i0/Value eq %27{row}%27))
+;url_parameters_pattern = {base_url}{location} and (Attributes/OData.CSC.DoubleAttribute/any(i0:i0/Name eq 'cloudCover' and i0/Value le {cloud_cover})) and (ContentDate/Start ge {start_date} and ContentDate/Start le {end_date}) and (Online eq true))&$orderby=ContentDate/Start%20asc&$top=1000&$expand=Attributes
+;thumbnail_property = properties/productIdentifier
+;cloud_cover_property = properties/cloudCover
+;gml_geometry_property = properties/gmlgeometry
 
 ##################################################################
 # Mixed archive sample: local landsat and remote S2
@@ -54,9 +54,9 @@ url_parameters_pattern_Landsat9 = {base_url}/{mission}/{path}/{row}
 ;url_parameters_pattern_Landsat9 = {base_url_landsat}/{mission}/{path}/{row}
 
 # remote S2 part
-;base_url_s2 = https://datahub.creodias.eu/resto/api/collections
-;location_Sentinel2 = productIdentifier=%25T{tile}%25
-;url_parameters_pattern = {base_url_s2}/{mission}/search.json?maxRecords=1000&cloudCover=%5B0%2C{cloud_cover}%5D&startDate={start_date}&completionDate={end_date}&sortParam=startDate&sortOrder=ascending&status=all&{location}&dataset=ESA-DATASET
+;base_url_s2 = https://datahub.creodias.eu/odata/v1/Products?$filter=(
+;location_Sentinel2 = ((Collection/Name eq 'SENTINEL-2')) and (contains(Name,%27{tile}%27))
+;url_parameters_pattern = {base_url_s2}{location} and (Attributes/OData.CSC.DoubleAttribute/any(i0:i0/Name eq 'cloudCover' and i0/Value le {cloud_cover})) and (ContentDate/Start ge {start_date} and ContentDate/Start le {end_date}) and (Online eq true))&$orderby=ContentDate/Start%20asc&$top=1000&$expand=Attributes
 ;thumbnail_property = properties/productIdentifier
 ;cloud_cover_property = properties/cloudCover
 ;gml_geometry_property = properties/gmlgeometry
@@ -68,10 +68,10 @@ url_parameters_pattern_Landsat9 = {base_url}/{mission}/{path}/{row}
 ;url_parameters_pattern_Sentinel2 = {base_url_s2}/{mission}/{tile}
 
 # remote landsat part
-;base_url_landsat = https://datahub.creodias.eu/resto/api/collections
-;location_Landsat8 = path={path}&row={row}&productType=L1TP
-;location_Landsat9 = path={path}&row={row}&productType=L1TP
-;url_parameters_pattern = {base_url_landsat}/{mission}/search.json?maxRecords=1000&cloudCover=%5B0%2C{cloud_cover}%5D&startDate={start_date}&completionDate={end_date}&sortParam=startDate&sortOrder=ascending&status=all&{location}&dataset=ESA-DATASET
+;base_url_landsat = https://datahub.creodias.eu/odata/v1/Products?$filter=(
+;location_Landsat8 = ((Collection/Name eq 'LANDSAT-8')) and (Attributes/OData.CSC.StringAttribute/any(i0:i0/Name eq %27wrsPath%27 and i0/Value eq %27{path}%27)) and (Attributes/OData.CSC.StringAttribute/any(i0:i0/Name eq %27wrsRow%27 and i0/Value eq %27{row}%27))
+;location_Landsat9 = ((Collection/Name eq 'LANDSAT-9')) and (Attributes/OData.CSC.StringAttribute/any(i0:i0/Name eq %27wrsPath%27 and i0/Value eq %27{path}%27)) and (Attributes/OData.CSC.StringAttribute/any(i0:i0/Name eq %27wrsRow%27 and i0/Value eq %27{row}%27))
+;url_parameters_pattern = {base_url_landsat}{location} and (Attributes/OData.CSC.DoubleAttribute/any(i0:i0/Name eq 'cloudCover' and i0/Value le {cloud_cover})) and (ContentDate/Start ge {start_date} and ContentDate/Start le {end_date}) and (Online eq true))&$orderby=ContentDate/Start%20asc&$top=1000&$expand=Attributes
 ;thumbnail_property = properties/productIdentifier
 ;cloud_cover_property = properties/cloudCover
 ;gml_geometry_property = properties/gmlgeometry

sen2like/docs/source/conf.py

@@ -14,8 +14,8 @@
 import sys
 from datetime import date
 
-sys.path.insert(0, os.path.abspath(os.path.join('..', '..', 'sen2like')))
-sys.path.insert(0, os.path.abspath(os.path.join('..', '..')))
+sys.path.insert(0, os.path.abspath(os.path.join("..", "..", "sen2like")))
+sys.path.insert(0, os.path.abspath(os.path.join("..", "..")))
 
 # import version from sen2like
 # pylint: disable=wrong-import-position
@@ -24,9 +24,9 @@
 # -- Project information -----------------------------------------------------
 
 # pylint: disable=invalid-name
-project = 'Sen2Like'
-author = 'Telespazio'
-copyright = f'{date.today().year}, {author}' # pylint: disable=redefined-builtin
+project = "Sen2Like"
+author = "Telespazio"
+copyright = f"{date.today().year}, {author}"  # pylint: disable=redefined-builtin
 version = version.__version__
 release = version
 
@@ -35,19 +35,20 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['sphinx.ext.todo',
-              'sphinx.ext.viewcode',
-              'sphinx.ext.autodoc',
-              'sphinx.ext.autosummary',
-              'sphinx.ext.coverage',
-              'sphinx_rtd_theme',
-              'm2r2'
-              ]
+extensions = [
+    "sphinx.ext.todo",
+    "sphinx.ext.viewcode",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.coverage",
+    "sphinx_rtd_theme",
+    "m2r2",
+]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
-source_suffix = ['.rst', '.md']
+source_suffix = [".rst", ".md"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -59,7 +60,7 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,

sen2like/release-notes.md

@@ -1,5 +1,12 @@
 # Sen2Like Release Notes
 
+## v4.5.3
+
+### New features
+
+* Add support for CreoDIAS ODATA API
+* Support CAMS CF-1.7 convention and previous CF-1.6 convention
+
 ## v4.5.2
 
 ### Fix

sen2like/run_tests.sh

@@ -8,7 +8,7 @@ export PYTHONPATH=.:${PYTHONPATH}
 
 if hash coverage 2> /dev/null; then
     echo **Running tests
-    PYTHONPATH=sen2like:aux_data coverage run --branch -m pytest -source=sen2like --log-level=DEBUG --junitxml reports/junit.xml "$@" |& tee reports/tests.report
+    PYTHONPATH=sen2like:aux_data coverage run --branch --source=sen2like -m pytest --log-level=DEBUG --junitxml reports/junit.xml "$@" |& tee reports/tests.report
     out=${PIPESTATUS[0]}
     if [ $out -ne 0 ];
     then

sen2like/sen2like/__init__.py

@@ -14,4 +14,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-

sen2like/sen2like/atmcor/__init__.py

@@ -14,4 +14,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-