diff --git a/CHANGELOG.md b/CHANGELOG.md
index c7fcff62..f77fbb80 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,6 @@
# Sen2VM Release history
-## Merged in main
+## Merged in main (1.3.0.rc2)
### Main feature
@@ -12,11 +12,17 @@
* Feature:
- *
+ * Handle mosaic DEM (multiple square degrees): #62
* Doc:
- *
+ * Notebooks improvements:
+
+ * Separate static input download in a dedicated notebook (#56)
+ * Allow more flexible input structure with separated input locations (#56)
+ * Creation of a Notebook for Inverse location (#54)
+ * Allow using OTB instead of gdal in direct location Notebook (through a new notebook)
+
## 1.2.0 (2026-05-04)
diff --git a/Dockerfile b/Dockerfile
index 8bc7ccff..63c98c84 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,6 @@
FROM ghcr.io/sen2vm/sen2vm-build-env:latest AS launcher
-ENV SEN2VM_VERSION=1.3.0.rc1
+ENV SEN2VM_VERSION=1.3.0.rc2
WORKDIR /Sen2vm
diff --git a/pom.xml b/pom.xml
index dc7155ff..e9152b8a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -7,7 +7,7 @@
esa.sen2vm
sen2vm-core
- 1.3.0.rc1
+ 1.3.0.rc2
sen2vm-core
https://github.com/sen2vm/sen2vm-core
diff --git a/sen2vm-notebook/README_Notebooks.md b/sen2vm-notebook/README_Notebooks.md
index 78c56020..6e9ba052 100644
--- a/sen2vm-notebook/README_Notebooks.md
+++ b/sen2vm-notebook/README_Notebooks.md
@@ -16,8 +16,22 @@
# Sen2VM Notebook Processing Workflow
This repository provides a complete workflow to run **Sen2VM** inside Docker and generate orthorectified and mosaicked outputs from Sentinel-2 L1B data.
+
+Four notebooks are provided:
+
+* **inputs-download-notebook.ipynb**
+ Downloads all required input data (GEOID and GIPP) from the Git repository.
+
+* **notebook-direct-grid.ipynb**
+ Runs Sen2VM with the **direct** mode, using the downloaded inputs, along with the IERS file and L1B product, to produce orthorectified and mosaicked images using gdal functions.
+
+* **notebook-direct-grid-otb.ipynb**
+ Runs Sen2vm with the **direct** mode, using the downloaded inputs, along with the IERS file and L1B product, to product orthorectified and mosaicked images using otb functions.
+
+* **notebook-inverse-grid.ipynb**
+ Runs Sen2VM with the **inverse** mode, using the downloaded inputs, along with the IERS file and L1B product, to produce orthorectified and mosaicked images.
+
The project is designed and tested on **Linux**. It may not work reliably on **Windows**.
-Actually, this notebook can generate an inverse grid but can not use it to apply the orthorectification.
## Prerequisites
@@ -25,35 +39,74 @@ Actually, this notebook can generate an inverse grid but can not use it to apply
* **Python 3.x**
* **the full sen2vm-notebook folder shall be present:**
- * notebook.ipynb
+ * inputs-download-notebook.ipynb
+ * notebook-direct-grid.ipynb
+ * notebook-direct-grid-otb.ipynb
+ * notebook-inverse-grid.ipynb
+
* requirements.txt
* gdal-latest folder
* Required data:
- * **Sentinel-2 L1B product** : /DATASTRIP and /GRANULE.
- * **DEM files** placed in the appropriate directory
+ * **Sentinel-2 L1B product**: /DATASTRIP and /GRANULE.
+ * **DEM files**
## Mandatory Directory Structure
+Only a `WORKDIR` folder is required. All outputs, as well as intermediate files generated during execution, will be stored there.
+
+## Inputs
+
+Several inputs are needed:
+* L1B S2 product under EUP.SAFE format:
+```bash
+ /
+ ├── DATASTRIP # Required
+ ├── GRANULE # Required
+ ├── S2*OPER_MTD_SAFL1B_PDMC*.xml #Required
+ └── ...
+```
+* Digital Elevation Model (DEM)
+
+* IERS (prediction of Earth orientation, IERS can be donwloaded using **notebook.ipynb** (cell number 2))
+
+* GIPP (GIPP can be downloaded from the GIT repositery using **inputs-download-notebook.ipynb**)
+
+* GEOID (GEOID can also be downloaded from the GIT repositery using **inputs-download-notebook.ipynb** )
+
+Thanks to a recent update (#issue 56), now the differents inputs path are absolute path.
+
+## Directory Structure after execution
```bash
WORKDIR
│
-├── DATA/
-│ ├── DEM/ # Put your DEM files here
-| ├── GEOID/ # Put your GEOID files here (Optional)
-│ └── / # Place the full L1B product here
-| ├── DATASTRIP # Required
-| ├── GRANULE # Required
-| └── ...
-└── ...
+├── bulletin*.txt # IERS prediction of earth exploration (downloaded by the cell n°3)
+│
+├── output/ # output after orthorectification and mosaic (.tif)
+│ ├── DIRECT_gdal
+│ │ ├─ GDAL_OUTPUT_ORTHO # .tif orthorectified, one for each band and detector
+│ │ └─ GDAL_OUTPUT_MOSAIC # .tif orthorectified and merged by band, one for each band
+│ ├── DIRECT_otb
+│ │ ├─ INVERSE_GRID # inverse gelocation grid, generated from direct grids by sen2vminvlocfromdirlocgrid.py, these grids encode the transformation between sensor geometry and map geometry
+│ │ ├─ raw # contains the raw L1B images extracted per detector and per band, expressed in the sensor geometry (i.e., before any orthorectification or map projection)
+│ │ ├─ otb_no_georef # contains images resampled using OTB with the inverse grids. The data are projected onto a regular grid but are not yet fully georeferenced
+│ │ ├─ output_georef # contains the final orthorectificatied and georeferenced images
+│ │ └─ mosaic # contains mosaicked images generated per spectral band by merging all detector-level orthorectified
+│ └── INVERSE
+│ ├─ INVERSE_GRID # inverse gelocation grid, generated by sen2vm, these grids encode the transformation between sensor geometry and map geometry
+│ ├─ raw # contains the raw L1B images extracted per detector and per band, expressed in the sensor geometry (i.e., before any orthorectification or map projection)
+│ ├─ otb_no_georef # contains images resampled using OTB with the inverse grids. The data are projected onto a regular grid but are not yet fully georeferenced
+│ ├─ output_georef # contains the final orthorectificatied and georeferenced images
+│ └─ mosaic # contains mosaicked images generated per spectral band by merging all detector-level orthorectified images
+│
+├── src/ # contains a .sh used to run in the gdal docker generated by the notebook
+└── UserConf # contains .txt config files generated by the notebook
```
-If the GEOID folder is empty, the notebopok will automaticaly use the geoid provided by sen2vm-core
-
## Python Environment Setup
-In the notebook directory create your venv :
+In the notebook directory create your venv:
```bash
python3 -m venv .venv
@@ -63,38 +116,85 @@ pip install -r requirements.txt
Select the virtual environment kernel in your Jupyter session.
-## Notebook Configuration
+## Notebooks Configuration
+
+In **inputs-download-notebook.ipynb**:
+In the first cell of the notebook:
+
+1. Set the paths to:
+ * The directory where you want the GIPP files to be downloaded
+ * The directory where you want the GEOID files to be downloaded
+
+In **notebook-direct-grid.ipynb** or **notebook-direct-grid-otb.ipynb** or **notebook-inverse-grid.ipynb**:
In the first cell of the notebook:
-1. Set the path to:
+1. Set the absolute paths to:
* The working directory
* The L1B product
- * The output directory
+ * The GIPP directory
+ * The GEOID directory
+ * The DEM directory
+ * The IERS file (if you do not have the IERS file, put `""`, and one will be automatically ed to the working directory)
+ * NB : The output directory will be created in the working directory
2. Adjust configuration parameters for:
* Sen2VM
* Orthorectification settings
+
+3. Specify whether the docker images should be removed
## Processing Steps
-Execute the notebook cell by cell in the following order:
+If you do not have your own GIPP and GEOID files, or if they are not already downloaded,
+execute the **inputs-download-notebook.ipynb** cell by cell in the following order:
-1. Variable definitions
+1. Path definitions
2. Clone `sen2vm-gipp`, and manage GIPP assets
-3. Automatic download of the IERS bulletin
-4. Generation of `config.json` in: `/WORKDIR/UserConf`
-5. Generation of `params.json` in: `/WORKDIR/UserConf`
-6. Execution of sen2vm inside Docker
-7. Generation of a `.sh` script, then execution inside a second Docker container running the latest GDAL:
-
- * Orthorectification by band
+3. Copy `Geoid` folder from `"/sen2vm-core/src/test/resources/DEM_GEOID"`
+
+Then if you want to use the direct grids with gdal functions only, execute the **notebook-direct-grid.ipynb** cell by cell:
+
+1. Variable and path definitions
+2. Automatic download of the IERS bulletin if none is provided by the user
+3. Generation of `config.json` in: `/WORKDIR/UserConf`
+4. Generation of `params.json` in: `/WORKDIR/UserConf`
+5. Execution of sen2vm inside Docker to generatre the direct grids
+6. Generation of a `.sh` script, then execution inside a second Docker container running the latest GDAL:
+
+ * Orthorectification by band that generate images in `WORKDIR/DIRECT/GDAL_OUTPUT_ORTHO` using the function **gdalwarp**
+ * Mosaicking by band that generate an image in `WORKDIR/DIRECT/GDAL_OUTPUT_MOSAIC` using the function **gdalmerge.py**
+
+If you want to use the direct grids with otb functions for the orthorectification, execute the **notebook-direct-grid-otb.ipynb** cell by cell:
+
+1. Variable and path definitions
+2. Automatic download of the IERS bulletin if none is provided by the user
+3. Generation of `config.json` in: `/WORKDIR/UserConf`
+4. Generation of `params.json` in: `/WORKDIR/UserConf`
+5. Execution of sen2vm inside Docker to generate the direct grids
+6. Generation of inverse grids in `WORKDIR/output/INVERSE/INVERSE_GRID` from the direct direct grids using **sen2vminvlocfromdirlocgrid**
+7. Orthorectification using OTB inside a docker then mosaicking using GDAL:
+
+ * Orthorectification by band
* Mosaicking
-Docker images are automatically cleaned up after each execution.
+Docker images can be removed or kept depending on the value of `REMOVE_DOCKER_IMAGE` in step 1.
+
+If you want to use the inverse grids, execute the **notebook-inverse-grid.ipynb** cell by cell:
+
+1. Variable and path definitions
+2. Automatic download of the IERS bulletin if none is provided by the user
+3. Generation of `config.json` in: `/WORKDIR/UserConf`
+4. Generation of `params.json` in: `/WORKDIR/UserConf`
+5. Execution of sen2vm inside Docker to generate the inverse grids in `WORKDIR/output/INVERSE/INVERSE_GRID`
+6. Orthorectification using OTB inside a docker then mosaicking using GDAL:
+
+ * Orthorectification by band
+ * Mosaicking
+Docker images can be removed or kept depending on the value of `REMOVE_DOCKER_IMAGE` in step 1.
## Execution
Run all notebook cells sequentially.
diff --git a/sen2vm-notebook/src/inputs-download-notebook.ipynb b/sen2vm-notebook/src/inputs-download-notebook.ipynb
new file mode 100644
index 00000000..cb5327b8
--- /dev/null
+++ b/sen2vm-notebook/src/inputs-download-notebook.ipynb
@@ -0,0 +1,150 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "e71d30a3-b1dc-48f9-b7a0-1ee82f8cc775",
+ "metadata": {},
+ "source": [
+ "# === USER CONFIGURATION ==="
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "0d377684-c4b1-4faa-a16f-02d7c8a58105",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#In this cell, you may indicate where you want the inputs to be donwloaded\n",
+ "\n",
+ "# Path where to put the GIPP directory\n",
+ "PATH_TO_GIPP = \"PATH/WHERE/TO/PUT/GIPP\"\n",
+ "\n",
+ "# Path where to put the GEOID directory\n",
+ "PATH_TO_GEOID = \"PATH/WHERE/TO/PUT/GEOID\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "93bc15d2-9c05-4cc2-82e5-3533e1bbcfd0",
+ "metadata": {},
+ "source": [
+ "# === GIPP database download ==="
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "52f77292-b1d3-42cd-817f-24b9bea7e520",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# === GIPPs ===\n",
+ "# This step is optional if the Database was already downloaded or if you want to use your own GIPP,\n",
+ "# It will donwload from GITHUB the directory sen2vm-gipp-database at PATH_GIPP\n",
+ "\n",
+ "import os\n",
+ "import shutil\n",
+ "import tarfile\n",
+ "import re\n",
+ "from datetime import datetime\n",
+ "import sys\n",
+ "\n",
+ "if not os.path.exists(PATH_TO_GIPP):\n",
+ " print(\"The folder where you want to download the GIPP is missing\")\n",
+ " sys.exit() \n",
+ "\n",
+ "# =====================================================================\n",
+ "# 1) CLONE sen2vm-gipp-database\n",
+ "# =====================================================================\n",
+ "\n",
+ "gipp_repo_name = \"sen2vm-gipp-database\"\n",
+ "PATH_GIPP = os.path.join(PATH_TO_GIPP, gipp_repo_name)\n",
+ "\n",
+ "# Delete repository if exists\n",
+ "if os.path.exists(PATH_GIPP):\n",
+ " print(f\"Removing existing repository: {PATH_GIPP}\")\n",
+ " shutil.rmtree(PATH_GIPP)\n",
+ "\n",
+ "# Clone repository fresh\n",
+ "print(\"Cloning sen2vm-gipp-database...\")\n",
+ "!git clone https://github.com/sen2vm/sen2vm-gipp-database.git {PATH_GIPP}\n",
+ "print(\"Clone complete.\\n\")\n",
+ "\n",
+ "print(\"GIPP processing finished successfully.\")\n",
+ "print(\"GIPP downloaded in: \", PATH_GIPP)\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "0e8b2d6d-5e67-4145-aa04-42d99671d36b",
+ "metadata": {},
+ "source": [
+ "# === GEOID folder copy from github ==="
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c781bbdc-3aa7-48d1-9284-5f5c507ed265",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# === GEOID ===\n",
+ "# This step is optional if the GEOID are already copied, or if you want to you use your own GEOID,\n",
+ "# It will copy the folder GEOID from \"*/sen2vm-core/src/test/resources/DEM_GEOID\" to PATH_TO_GEOID\n",
+ "\n",
+ "import os\n",
+ "import sys\n",
+ "import shutil\n",
+ "\n",
+ "if not os.path.exists(PATH_TO_GEOID):\n",
+ " print(\"The folder where you want to download the GEOID is missing\")\n",
+ " sys.exit() \n",
+ "\n",
+ "GEOID_DIR = os.path.join(PATH_TO_GEOID, \"GEOID\")\n",
+ "\n",
+ "if os.path.exists(GEOID_DIR):\n",
+ " shutil.rmtree(GEOID_DIR)\n",
+ "\n",
+ "os.makedirs(GEOID_DIR)\n",
+ "\n",
+ "# Notebook location (NOT relative to CWD)\n",
+ "notebook_dir = os.getcwd()\n",
+ "\n",
+ "# Relative path to DEM_GEOID from notebook\n",
+ "internal_geoid_dir = os.path.abspath(os.path.join(\n",
+ " notebook_dir,\n",
+ " \"..\", \"..\", \"src\", \"test\", \"resources\", \"DEM_GEOID\"\n",
+ "))\n",
+ "\n",
+ "for f in os.listdir(internal_geoid_dir):\n",
+ " src = os.path.join(internal_geoid_dir, f)\n",
+ " dst = os.path.join(GEOID_DIR, f)\n",
+ " shutil.copy(src, dst)\n",
+ "print(\"Done copying\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.13.11"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/sen2vm-notebook/src/notebook-direct-grid-otb.ipynb b/sen2vm-notebook/src/notebook-direct-grid-otb.ipynb
new file mode 100644
index 00000000..fbafa227
--- /dev/null
+++ b/sen2vm-notebook/src/notebook-direct-grid-otb.ipynb
@@ -0,0 +1,925 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "8e5ed8d8-c718-4f80-a220-464bbc67c748",
+ "metadata": {},
+ "source": [
+ "# === USER CONFIGURATION ==="
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2b816cd9-5254-452b-bced-2943eb6b7aec",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# # === USER CONFIGURATION ===\n",
+ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
+ "# /!\\/!\\/!\\ CAREFUL, please read the README_Notebooks.md file before\n",
+ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
+ "\n",
+ "WORKDIR = \"PATH/TO/WORKDIR\" # Working directory for sen2vm processing\n",
+ "\n",
+ "# Path to downloaded product\n",
+ "PATH_L1B_DATA = \"PATH/TO/L1B_PRODUCT\"\n",
+ "\n",
+ "# Path to GIPP directory\n",
+ "PATH_GIPP = \"PATH/TO/GIPP\"\n",
+ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
+ "# /!\\/!\\/!\\ If you do not have your own GIPP folder, you may use the inputs-download-notebook to download it, then you may indicate the path were you \n",
+ "# /!\\/!\\/!\\ downloaded it here in PATH_GIPP\n",
+ "# /!\\/!\\/!\\ If you have your own GIPP folder, please note that this current notebook will search for a subfolder with mission S2[A/B/C] inside the GIPP folder\n",
+ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
+ "\n",
+ "# Path to DEM directory \n",
+ "PATH_DEM = \"PATH/TO/DEM\"\n",
+ "\n",
+ "# Path to GEOID directory\n",
+ "PATH_GEOID = \"PATH/TO/GEOID\"\n",
+ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
+ "# /!\\/!\\/!\\ If you do not have your own GEOID folder, you may use the inputs-download-notebook to download it, then you may indicate the path were you\n",
+ "# /!\\/!\\/!\\ downloaded it here in \n",
+ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
+ "\n",
+ "# Path to IERS file\n",
+ "PATH_IERS = \"PATH/TO/IERS_FILE\"\n",
+ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
+ "# /!\\/!\\/!\\ If you do not have your own IERS, you may let this path empy (i.e. \"\") and execute the cell number 2 named #IERS Download\n",
+ "# /!\\/!\\/!\\ In this case, the IERS will be downloaded in the WORKDIR\n",
+ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
+ "\n",
+ "# The output folder is put in the WORKDIR\n",
+ "OUTPUT_FOLDER = WORKDIR + \"/output/DIRECT_otb\"\n",
+ "\n",
+ "# === SEN2VM OPTIONS ===\n",
+ "UTM_EPSG = 32628 # UTM zone EPSG code for the ROI\n",
+ "LOCATION = {\n",
+ " \"ul_x\": 283910,\n",
+ " \"ul_y\": 3641660,\n",
+ " \"lr_x\": 354280,\n",
+ " \"lr_y\": 3608316.0\n",
+ "}\n",
+ "\n",
+ "# Grid step in pixels for sen2vm grid generation \n",
+ "# These values represent the grid step in pixels\n",
+ "# In operation, this step is linked to DEM step which is not constant over the world, pending the Latitude\n",
+ "STEPS = {\n",
+ " \"10m_bands\": 4.5, # Around 45m in operation (pending laittude) with 30m and also 90m DEM\n",
+ " \"20m_bands\": 4.5, # Around 90m in operation (pending laittude) with 30m and also 90m DEM\n",
+ " \"60m_bands\": 3.0 # Around 180m in operation (pending laittude) with 30m and also 90m DEM\n",
+ "}\n",
+ "\n",
+ "# === GDAL ORTHO OPTIONS ===\n",
+ "\n",
+ "ORTHO_SETTINGS = {\n",
+ " \"keep_bands\": [\"B01\", \"B02\", \"B03\", \"B04\", \"B05\", \"B06\", \"B07\", \"B08\", \"B8A\", \"B09\", \"B10\", \"B11\", \"B12\"], # list of bands to keep for orthorectification\n",
+ " \"keep_detectors\": [\"07\",\"08\",\"09\",\"10\",\"11\"] # list of the detectors to keep \n",
+ "}\n",
+ "\n",
+ "# === Docker Options ===\n",
+ "# Building Docker images may take several minutes.\n",
+ "# By setting REMOVE_DOCKER_IMAGE = False, the images are kept,\n",
+ "# which significantly speeds up subsequent executions.\n",
+ "#\n",
+ "# However, Docker images use disk space (~4 GB in this case).\n",
+ "#\n",
+ "# Docker images can be manually removed using:\n",
+ "# docker images\n",
+ "# docker rmi \n",
+ "#\n",
+ "REMOVE_DOCKER_IMAGE = False # True = remove Docker images after execution"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "63f616a3",
+ "metadata": {},
+ "source": [
+ "# IERS Download"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8ec4d6f6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# === DOWNLOAD IERS ===\n",
+ "import os\n",
+ "import re\n",
+ "import requests\n",
+ "import sys\n",
+ "from datetime import datetime\n",
+ "\n",
+ "if len(PATH_IERS) != 0: # Safety in case the PATH_IERS isn't empty\n",
+ " print(\"Stopping because PATH_IERS is not empty\")\n",
+ " sys.exit(0)\n",
+ "\n",
+ "if not os.path.exists(WORKDIR):\n",
+ " raise RuntimeError(f\"WORKDIR directory not found: {WORKDIR}\")\n",
+ "\n",
+ "print(\"Bulletin output directory: \", WORKDIR)\n",
+ "\n",
+ "PATH_IERS = WORKDIR # then our file will be there\n",
+ "# =====================================================================\n",
+ "# REMOVE EXISTING IERS BULLETINS\n",
+ "# =====================================================================\n",
+ "\n",
+ "for f in os.listdir(WORKDIR):\n",
+ " if f.startswith(\"bulletina-\") and f.endswith(\".txt\"):\n",
+ " os.remove(os.path.join(WORKDIR, f))\n",
+ " print(\"Removed old bulletin A: \", f)\n",
+ "\n",
+ " if f.startswith(\"bulletinb-\") and f.endswith(\".txt\"):\n",
+ " os.remove(os.path.join(WORKDIR, f))\n",
+ " print(\"Removed old bulletin B: \", f)\n",
+ "\n",
+ "print(\"Cleanup of old bulletins complete.\\n\")\n",
+ "# =====================================================================\n",
+ "# EXTRACT PRODUCT DATE (FROM DATASTRIP)\n",
+ "# =====================================================================\n",
+ "\n",
+ "datastrip_dir = os.path.join(PATH_L1B_DATA, \"DATASTRIP\")\n",
+ "\n",
+ "if not os.path.isdir(datastrip_dir):\n",
+ " raise RuntimeError(f\"DATASTRIP directory not found: {datastrip_dir}\")\n",
+ "\n",
+ "datastrip_entries = os.listdir(datastrip_dir)\n",
+ "if not datastrip_entries:\n",
+ " raise RuntimeError(f\"No DATASTRIP found in: {datastrip_dir}\")\n",
+ "\n",
+ "# Take the first DATASTRIP product\n",
+ "datastrip_name = datastrip_entries[0]\n",
+ "\n",
+ "match = re.search(r\"_S(\\d{8})T\\d{6}_\", datastrip_name)\n",
+ "if not match:\n",
+ " raise RuntimeError(\"Could not extract product date from DATASTRIP name.\")\n",
+ "\n",
+ "product_date_str = match.group(1)\n",
+ "\n",
+ "year = int(product_date_str[:4])\n",
+ "month = int(product_date_str[4:6])\n",
+ "day = int(product_date_str[6:8])\n",
+ "\n",
+ "product_date = datetime(year, month, day)\n",
+ "\n",
+ "print(\"Product date extracted from DATASTRIP: \", product_date.date(), \"\\n\")\n",
+ "# =====================================================================\n",
+ "# Roman conversion \n",
+ "# =====================================================================\n",
+ "\n",
+ "def int_to_roman(n):\n",
+ " vals = [\n",
+ " (1000, 'm'), (900, 'cm'), (500, 'd'), (400, 'cd'),\n",
+ " (100, 'c'), (90, 'xc'), (50, 'l'), (40, 'xl'),\n",
+ " (10, 'x'), (9, 'ix'), (5, 'v'), (4, 'iv'), (1, 'i')\n",
+ " ]\n",
+ " res = \"\"\n",
+ " for v, s in vals:\n",
+ " while n >= v:\n",
+ " res += s\n",
+ " n -= v\n",
+ " return res\n",
+ "# =====================================================================\n",
+ "# Bulletin A\n",
+ "# =====================================================================\n",
+ "\n",
+ "roman_year = int_to_roman(year - 1987)\n",
+ "doy = product_date.timetuple().tm_yday\n",
+ "index = (doy - 1) // 7 + 1\n",
+ "\n",
+ "print(f\"Bulletin A Roman year: {roman_year}\")\n",
+ "print(f\"Initial weekly index: {index}\\n\")\n",
+ "\n",
+ "found = False\n",
+ "\n",
+ "while index > 0:\n",
+ " index_str = f\"{index:03d}\"\n",
+ " url = f\"https://datacenter.iers.org/data/6/bulletina-{roman_year}-{index_str}.txt\"\n",
+ " print(\"Trying bulletin: \", url)\n",
+ "\n",
+ " response = requests.get(url)\n",
+ "\n",
+ " if response.status_code == 200:\n",
+ " print(\"Bulletin found: \", index_str)\n",
+ " dest_file = os.path.join(WORKDIR, f\"bulletina-{roman_year}-{index_str}.txt\")\n",
+ " found = True\n",
+ " break\n",
+ "\n",
+ " index -= 1\n",
+ "\n",
+ "if not found:\n",
+ " raise RuntimeError(\"No Bulletin A available for current or previous weeks.\")\n",
+ "\n",
+ "with open(dest_file, \"wb\") as f:\n",
+ " f.write(response.content)\n",
+ "\n",
+ "print(\"Downloaded: \", dest_file)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9e85608d",
+ "metadata": {},
+ "source": [
+ "# Sen2VM configuration"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "276592b5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# === GENERATE CONFIG.JSON ===\n",
+ "\n",
+ "import os\n",
+ "import json\n",
+ "import re\n",
+ "import shutil\n",
+ "from numpy import double\n",
+ "\n",
+ "USERCONF_DIR = os.path.join(WORKDIR, \"UserConf\")\n",
+ "os.makedirs(USERCONF_DIR, exist_ok=True)\n",
+ "\n",
+ "print(\"UserConf directory: \", USERCONF_DIR)\n",
+ "print(\"Geoid directory: \", PATH_GEOID)\n",
+ "\n",
+ "# =====================================================\n",
+ "# 1. Extract mission from DATASTRIP\n",
+ "# =====================================================\n",
+ "\n",
+ "datastrip_dir = os.path.join(PATH_L1B_DATA, \"DATASTRIP\")\n",
+ "\n",
+ "if not os.path.isdir(datastrip_dir):\n",
+ " raise RuntimeError(f\"DATASTRIP directory not found: {datastrip_dir}\")\n",
+ "\n",
+ "datastrip_entries = os.listdir(datastrip_dir)\n",
+ "if not datastrip_entries:\n",
+ " raise RuntimeError(f\"No DATASTRIP found in: {datastrip_dir}\")\n",
+ "\n",
+ "datastrip_name = datastrip_entries[0]\n",
+ "\n",
+ "match = re.match(r\"(S2[A-C])_OPER_\", datastrip_name)\n",
+ "if not match:\n",
+ " raise RuntimeError(\"Cannot extract mission (S2A/S2B/S2C) from DATASTRIP name.\")\n",
+ "\n",
+ "mission = match.group(1)\n",
+ "# =====================================================\n",
+ "# 2. Docker paths inside /workspace\n",
+ "# =====================================================\n",
+ "\n",
+ "docker_l1b = \"/data/L1B\"\n",
+ "docker_dem = \"/data/DEM\"\n",
+ "docker_gipp = f\"/data/GIPP/{mission}\"\n",
+ "# =====================================================\n",
+ "# 3. Geoid management\n",
+ "# =====================================================\n",
+ "\n",
+ "# Detect .gtx inside PATH_GEOID\n",
+ "geoid_files = [f for f in os.listdir(PATH_GEOID) if f.lower().endswith(\".gtx\")]\n",
+ "if len(geoid_files) == 0:\n",
+ " raise RuntimeError(\"No .gtx geoid file found in PATH_GEOID.\")\n",
+ "\n",
+ "docker_geoid = f\"/data/GEOID/{geoid_files[0]}\"\n",
+ "# =====================================================\n",
+ "# 4. Locate IERS bulletin on host\n",
+ "# =====================================================\n",
+ "\n",
+ "iers_host = None\n",
+ "\n",
+ "if os.path.isfile(PATH_IERS):\n",
+ " iers_host = PATH_IERS\n",
+ " PATH_IERS = os.path.dirname(PATH_IERS)\n",
+ "elif os.path.isdir(PATH_IERS):\n",
+ " for f in os.listdir(PATH_IERS):\n",
+ " if f.startswith(\"bulletin\"):\n",
+ " iers_host = os.path.join(PATH_IERS, f)\n",
+ " break\n",
+ "\n",
+ "if iers_host is None:\n",
+ " raise RuntimeError(\"IERS bulletin not found inside PATH_IERS directory.\")\n",
+ "\n",
+ "docker_iers = f\"/data/IERS/{os.path.basename(iers_host)}\"\n",
+ "print(\"IERS: \", docker_iers)\n",
+ "# =====================================================\n",
+ "# 5. Build config dictionary\n",
+ "# =====================================================\n",
+ "\n",
+ "config = {\n",
+ " \"l1b_product\": docker_l1b,\n",
+ " \"gipp_folder\": docker_gipp,\n",
+ " \"auto_gipp_selection\": True,\n",
+ " \"grids_overwriting\": True,\n",
+ " \"dem\": docker_dem,\n",
+ " \"geoid\": docker_geoid,\n",
+ " \"iers\": docker_iers,\n",
+ " \"operation\": \"direct\",\n",
+ " \"deactivate_available_refining\": False,\n",
+ " \"steps\": {\n",
+ " \"10m_bands\": STEPS[\"10m_bands\"],\n",
+ " \"20m_bands\": STEPS[\"20m_bands\"],\n",
+ " \"60m_bands\": STEPS[\"60m_bands\"]\n",
+ " },\n",
+ " \"export_alt\": True\n",
+ "}\n",
+ "# =====================================================\n",
+ "# Save config.json\n",
+ "# =====================================================\n",
+ "\n",
+ "config_path = os.path.join(USERCONF_DIR, \"config.json\")\n",
+ "\n",
+ "with open(config_path, \"w\") as f:\n",
+ " json.dump(config, f, indent=4)\n",
+ "\n",
+ "print(\"Configuration file generated: \")\n",
+ "print(config_path)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "23b1da25",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# === GENERATE PARAMS.JSON ===\n",
+ "\n",
+ "import os\n",
+ "import json\n",
+ "import re\n",
+ "\n",
+ "USERCONF_DIR = os.path.join(WORKDIR, \"UserConf\")\n",
+ "os.makedirs(USERCONF_DIR, exist_ok=True)\n",
+ "\n",
+ "print(\"UserConf directory: \", USERCONF_DIR)\n",
+ "# =====================================================\n",
+ "# Locate GRANULE folders\n",
+ "# =====================================================\n",
+ "\n",
+ "GR_TARGET_DIR = os.path.join(PATH_L1B_DATA, \"GRANULE\")\n",
+ "\n",
+ "if not os.path.exists(GR_TARGET_DIR):\n",
+ " raise RuntimeError(\"GRANULE directory not found inside L1B SAFE.\")\n",
+ "\n",
+ "granule_folders = [\n",
+ " os.path.join(GR_TARGET_DIR, d)\n",
+ " for d in os.listdir(GR_TARGET_DIR)\n",
+ " if os.path.isdir(os.path.join(GR_TARGET_DIR, d))\n",
+ "]\n",
+ "\n",
+ "print(\"Found\", len(granule_folders), \"granule folders.\")\n",
+ "# =====================================================\n",
+ "# Extract detectors and bands from JP2\n",
+ "# =====================================================\n",
+ "\n",
+ "detectors = set()\n",
+ "bands = set()\n",
+ "\n",
+ "pattern = r\"_D(\\d+)_B(\\d{1,2}[A]?)\\.jp2$\"\n",
+ "\n",
+ "for granule in granule_folders:\n",
+ " img_data_dir = os.path.join(granule, \"IMG_DATA\")\n",
+ "\n",
+ " if not os.path.isdir(img_data_dir):\n",
+ " continue\n",
+ "\n",
+ " for fname in os.listdir(img_data_dir):\n",
+ " match = re.search(pattern, fname)\n",
+ " if match:\n",
+ " detectors.add(match.group(1))\n",
+ " bands.add(f\"B{match.group(2)}\")\n",
+ "\n",
+ "detectors = sorted(detectors)\n",
+ "bands = sorted(bands)\n",
+ "\n",
+ "print(\"Detected detectors: \", detectors)\n",
+ "print(\"Detected bands: \", bands)\n",
+ "# =====================================================\n",
+ "# Write params.json\n",
+ "# =====================================================\n",
+ "\n",
+ "# Validate keep_detectors: must be a non-empty list (user requirement)\n",
+ "if not isinstance(ORTHO_SETTINGS.get(\"keep_detectors\"), list) or len(ORTHO_SETTINGS[\"keep_detectors\"]) == 0:\n",
+ " raise RuntimeError(\"ORTHO_SETTINGS['keep_detectors'] must be a non-empty list of detector ids (e.g. ['01','02']).\")\n",
+ "\n",
+ "# Filter detectors to only include those selected by the user\n",
+ "keep_detectors_list = ORTHO_SETTINGS[\"keep_detectors\"]\n",
+ "selected_detectors = [d for d in keep_detectors_list if d in detectors]\n",
+ "\n",
+ "# Validate that all requested detectors exist\n",
+ "missing_detectors = [d for d in keep_detectors_list if d not in detectors]\n",
+ "if missing_detectors:\n",
+ " raise RuntimeError(f\"Some requested detectors are not available in the product: {missing_detectors}. Available detectors: {detectors}\")\n",
+ "\n",
+ "# Only include selected detectors and bands in params.json\n",
+ "params = {\n",
+ " \"detectors\": sorted(selected_detectors),\n",
+ " \"bands\": sorted(ORTHO_SETTINGS[\"keep_bands\"])\n",
+ "}\n",
+ "\n",
+ "params_path = os.path.join(USERCONF_DIR, \"params.json\")\n",
+ "\n",
+ "with open(params_path, \"w\") as f:\n",
+ " json.dump(params, f, indent=4)\n",
+ "\n",
+ "print(\"params.json written to: \", params_path)\n",
+ "print(f\" - Detectors: {params['detectors']}\")\n",
+ "print(f\" - Bands: {params['bands']}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ec6f67fe",
+ "metadata": {},
+ "source": [
+ "# Sen2VM run"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "572bd72a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# === RUN SEN2VM (Docker: BUILD + RUN + CLEAN) ===\n",
+ "\n",
+ "import os\n",
+ "import subprocess\n",
+ "# Notebook location (NOT relative to CWD)\n",
+ "notebook_dir = os.getcwd()\n",
+ "dockerfile_dir = os.path.abspath(os.path.join(\n",
+ " notebook_dir,\n",
+ " \"..\", \"..\"\n",
+ "))\n",
+ "\n",
+ "config_inside = \"/workspace/UserConf/config.json\"\n",
+ "params_inside = \"/workspace/UserConf/params.json\"\n",
+ "# =====================================================\n",
+ "# 1. BUILD DOCKER IMAGE\n",
+ "# =====================================================\n",
+ "\n",
+ "print(f\"Building Docker image 'sen2vm' from: {dockerfile_dir}\")\n",
+ "cmd_build = [\n",
+ " \"docker\", \"build\",\n",
+ " \"-t\", \"sen2vm\",\n",
+ " dockerfile_dir\n",
+ "]\n",
+ "print(\"Command:\", \" \".join(cmd_build), \"\\n\")\n",
+ "subprocess.run(cmd_build, check=True)\n",
+ "print(\"Docker image built successfully.\\n\")\n",
+ "# =====================================================\n",
+ "# 2. RUN SEN2VM CONTAINER\n",
+ "# =====================================================\n",
+ "\n",
+ "cmd_run = [\n",
+ " \"docker\", \"run\",\n",
+ " \"--rm\",\n",
+ " \"-v\", f\"{PATH_L1B_DATA}:/data/L1B\",\n",
+ " \"-v\", f\"{PATH_DEM}:/data/DEM\",\n",
+ " \"-v\", f\"{PATH_GIPP}:/data/GIPP\",\n",
+ " \"-v\", f\"{PATH_GEOID}:/data/GEOID\",\n",
+ " \"-v\", f\"{PATH_IERS}:/data/IERS\",\n",
+ " \"-v\", f\"{WORKDIR}:/workspace\",\n",
+ " \"sen2vm\",\n",
+ " \"-c\", config_inside,\n",
+ " \"-p\", params_inside\n",
+ "]\n",
+ "print(\"Running Docker container...\\n\")\n",
+ "print(\"Command:\", \" \".join(cmd_run), \"\\n\")\n",
+ "subprocess.run(cmd_run, check=True)\n",
+ "print(\"\\nDocker execution complete.\\n\")\n",
+ "# =====================================================\n",
+ "# 3. REMOVE DOCKER IMAGE\n",
+ "# =====================================================\n",
+ "\n",
+ "if REMOVE_DOCKER_IMAGE:\n",
+ " print(\"Removing Docker image 'sen2vm'...\")\n",
+ " subprocess.run([\"docker\", \"rmi\", \"-f\", \"sen2vm\"], check=True)\n",
+ " print(\"Docker images removed.\\n\")\n",
+ "else:\n",
+ " print(\"Docker images kept.\\n\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "e54044d2",
+ "metadata": {},
+ "source": [
+ "# Generate Orthorectification images"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a091ccaf",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import subprocess\n",
+ "import glob\n",
+ "import json\n",
+ "import re\n",
+ "\n",
+ "# =====================================================\n",
+ "# Import of sen2vm_invloc_from_dir_loc_grid\n",
+ "# =====================================================\n",
+ "import sys\n",
+ "from pathlib import Path\n",
+ "sys.path.append(\n",
+ " str(Path(\"../../assets/scripts\").resolve())\n",
+ ")\n",
+ "\n",
+ "from sen2vm_invloc_from_dir_loc_grid import (\n",
+ " sen2vm_invloc_from_dir_loc_grid\n",
+ ")\n",
+ "print(Path(\"../../assets/scripts\").resolve())\n",
+ "\n",
+ "# =====================================================\n",
+ "# SAFETY CHECKS\n",
+ "# =====================================================\n",
+ "assert os.path.exists(PATH_L1B_DATA), \"L1B path missing\"\n",
+ "os.makedirs(OUTPUT_FOLDER, exist_ok=True)\n",
+ "assert os.path.exists(OUTPUT_FOLDER), \"Output folder missing\"\n",
+ "\n",
+ "# =====================================================\n",
+ "# Locate product name\n",
+ "# =====================================================\n",
+ "product = os.path.basename(os.path.normpath(PATH_L1B_DATA))\n",
+ "\n",
+ "# =====================================================\n",
+ "# Locate XML\n",
+ "# =====================================================\n",
+ "xml_list = glob.glob(\n",
+ " os.path.join(\n",
+ " PATH_L1B_DATA,\n",
+ " \"S2*_MTD_*.xml\"\n",
+ " )\n",
+ ")\n",
+ "if len(xml_list) == 0:\n",
+ " raise RuntimeError(\"No DATASTRIP MTD XML found\")\n",
+ "\n",
+ "xml_path = xml_list[0]\n",
+ "xml_name = os.path.basename(xml_path)\n",
+ "\n",
+ "# XML path inside docker (relative, after cd)\n",
+ "xml_docker = f\"./{xml_name}\"\n",
+ "\n",
+ "print(\"Using XML: \", xml_path)\n",
+ "\n",
+ "# =====================================================\n",
+ "# Read params.json to get selected bands and detectors\n",
+ "# =====================================================\n",
+ "params_path = os.path.join(WORKDIR, \"UserConf\", \"params.json\")\n",
+ "if not os.path.exists(params_path):\n",
+ " raise RuntimeError(f\"params.json not found at {params_path}\")\n",
+ "\n",
+ "with open(params_path, \"r\") as f:\n",
+ " params = json.load(f)\n",
+ "\n",
+ "selected_bands = params.get(\"bands\", [])\n",
+ "selected_detectors = params.get(\"detectors\", [])\n",
+ "\n",
+ "print(f\"Bands from params.json: {selected_bands}\")\n",
+ "print(f\"Detectors from params.json: {selected_detectors}\")\n",
+ "# =====================================================\n",
+ "# Create output folders\n",
+ "# =====================================================\n",
+ "\n",
+ "RAW_DIR = os.path.join(OUTPUT_FOLDER, \"raw\")\n",
+ "OTB_DIR = os.path.join(OUTPUT_FOLDER, \"otb_no_georef\")\n",
+ "GEOREF_DIR = os.path.join(OUTPUT_FOLDER, \"output_georef\")\n",
+ "MOSAIC_DIR = os.path.join(OUTPUT_FOLDER, \"mosaic\")\n",
+ "os.makedirs(os.path.join(OUTPUT_FOLDER, \"INVERSE_GRID\"), exist_ok=True)\n",
+ "os.makedirs(RAW_DIR, exist_ok=True)\n",
+ "os.makedirs(OTB_DIR, exist_ok=True)\n",
+ "os.makedirs(GEOREF_DIR, exist_ok=True)\n",
+ "os.makedirs(MOSAIC_DIR, exist_ok=True)\n",
+ "\n",
+ "# =====================================================\n",
+ "# Locate and filter direct grids\n",
+ "# =====================================================\n",
+ "\n",
+ "grid_pattern = r\"_D(\\d+)_B(\\d{1,2}[A]?)\"\n",
+ "direct_grids = []\n",
+ "all_grids = glob.glob(\n",
+ " os.path.join(\n",
+ " PATH_L1B_DATA,\n",
+ " \"DATASTRIP\",\n",
+ " \"S2*\",\n",
+ " \"GEO_DATA\",\n",
+ " \"*.vrt\"\n",
+ " )\n",
+ ")\n",
+ "print(f\"Total direct grids found: {len(all_grids)}\")\n",
+ "\n",
+ "for grid in all_grids:\n",
+ " grid_name = Path(grid).stem\n",
+ " match = re.search(grid_pattern, grid_name)\n",
+ " if not match:\n",
+ " continue\n",
+ " detector = match.group(1)\n",
+ " band = f\"B{match.group(2)}\"\n",
+ " if detector in selected_detectors and band in selected_bands:\n",
+ " direct_grids.append(grid)\n",
+ " print(\n",
+ " f\"✓ Keep {grid_name} \"\n",
+ " f\"(Detector={detector}, Band={band})\"\n",
+ " )\n",
+ "print(f\"\\nFiltered direct grids: {len(direct_grids)}\")\n",
+ "# =====================================================\n",
+ "# Create inverse grids\n",
+ "# =====================================================\n",
+ "\n",
+ "inverse_grid_list = []\n",
+ "\n",
+ "for direct_grid in direct_grids:\n",
+ " grid_name = Path(direct_grid).stem\n",
+ " match = re.search(grid_pattern, grid_name)\n",
+ " detector = match.group(1)\n",
+ " band = f\"B{match.group(2)}\"\n",
+ " \n",
+ " if band in [\"B02\", \"B03\", \"B04\", \"B08\"]:\n",
+ " resolution = 10\n",
+ " elif band in [\"B05\", \"B06\", \"B07\", \"B8A\", \"B11\", \"B12\"]:\n",
+ " resolution = 20\n",
+ " elif band in [\"B01\", \"B09\", \"B10\"]:\n",
+ " resolution = 60\n",
+ " else:\n",
+ " resolution = 10\n",
+ " inverse_grid = os.path.join(\n",
+ " OUTPUT_FOLDER,\n",
+ " \"INVERSE_GRID\",\n",
+ " f\"{grid_name}.tif\"\n",
+ " )\n",
+ " print(\"\\n========================================\")\n",
+ " print(\"Generating inverse grid\")\n",
+ " print(\"Direct: \", direct_grid)\n",
+ " print(\"Inverse: \", inverse_grid)\n",
+ " print(\"========================================\")\n",
+ "\n",
+ " sen2vm_invloc_from_dir_loc_grid(\n",
+ " grid=direct_grid,\n",
+ " resolution=resolution,\n",
+ " out_file=inverse_grid,\n",
+ " epsg=UTM_EPSG\n",
+ " )\n",
+ " inverse_grid_list.append(inverse_grid)\n",
+ "\n",
+ "# =====================================================\n",
+ "# Process all inverse grids\n",
+ "# =====================================================\n",
+ "for inv_grid_path in inverse_grid_list:\n",
+ " test_grid = os.path.basename(inv_grid_path)\n",
+ " print(\"\\n=================================================\")\n",
+ " print(\"Processing: \", test_grid)\n",
+ " print(\"=================================================\")\n",
+ "\n",
+ " # =================================================\n",
+ " # Parse detector / band\n",
+ " # =================================================\n",
+ " match = re.search(r\"_D(\\d+)_B(\\d{1,2}[A]?)\", test_grid)\n",
+ " if not match:\n",
+ " print(\"Could not parse detector/band\")\n",
+ " continue\n",
+ "\n",
+ " detector = match.group(1)\n",
+ " band = f\"B{match.group(2)}\"\n",
+ "\n",
+ " print(\"Detector: \", detector)\n",
+ " print(\"Band: \", band)\n",
+ "\n",
+ " # =================================================\n",
+ " # Build subdataset dynamically\n",
+ " # =================================================\n",
+ " subdataset = test_grid.replace(\".tif\", \"\")\n",
+ " print(\"Subdataset: \", subdataset)\n",
+ " \n",
+ " # =================================================\n",
+ " # Determine resolution\n",
+ " # =================================================\n",
+ " if band in [\"B02\", \"B03\", \"B04\", \"B08\"]:\n",
+ " resolution = 10\n",
+ " elif band in [\"B05\", \"B06\", \"B07\", \"B8A\", \"B11\", \"B12\"]:\n",
+ " resolution = 20\n",
+ " elif band in [\"B01\", \"B09\", \"B10\"]:\n",
+ " resolution = 60\n",
+ " else:\n",
+ " resolution = 10\n",
+ " \n",
+ " print(\"Resolution: \", resolution)\n",
+ " \n",
+ " # =================================================\n",
+ " # Compute output size\n",
+ " # =================================================\n",
+ " width = int(\n",
+ " (LOCATION[\"lr_x\"] - LOCATION[\"ul_x\"]) / resolution\n",
+ " )\n",
+ " height = int(\n",
+ " (LOCATION[\"ul_y\"] - LOCATION[\"lr_y\"]) / resolution\n",
+ " )\n",
+ " print(\"Width: \", width)\n",
+ " print(\"Height: \", height)\n",
+ "\n",
+ " # =================================================\n",
+ " # File paths\n",
+ " # =================================================\n",
+ " raw_name = f\"raw_D{detector}_{band}.tif\"\n",
+ " ortho_name = f\"ortho_D{detector}_{band}.tif\"\n",
+ " georef_name = f\"ortho_D{detector}_{band}_georef.tif\"\n",
+ " \n",
+ " # =================================================\n",
+ " # Extract raw detector image\n",
+ " # =================================================\n",
+ " cmd_extract = [\n",
+ " \"docker\", \"run\",\n",
+ " \"--rm\",\n",
+ " \"-v\", f\"{PATH_L1B_DATA}:/data/L1B\",\n",
+ " \"-v\", f\"{OUTPUT_FOLDER}:/output\",\n",
+ " \"gdal-latest\",\n",
+ " \"-c\",\n",
+ " f'''\n",
+ " gdal_translate \\\n",
+ " \"SENTINEL2_L1B_WITH_GEOLOC:/data/L1B/{xml_name}:{subdataset}\" \\\n",
+ " \"/output/raw/{raw_name}\"\n",
+ " '''\n",
+ " ]\n",
+ " print(\"\\n=== EXTRACTION ===\\n\")\n",
+ " print(\" \".join(cmd_extract))\n",
+ "\n",
+ " subprocess.run(cmd_extract, check=True)\n",
+ " \n",
+ " # Half-pixel correction (OTB bug fix)\n",
+ " corrected_ulx = LOCATION[\"ul_x\"] + resolution / 2\n",
+ " corrected_uly = LOCATION[\"ul_y\"] - resolution / 2\n",
+ "\n",
+ " # =================================================\n",
+ " # OTB resampling\n",
+ " # =================================================\n",
+ " cmd_otb = [\n",
+ " \"docker\", \"run\",\n",
+ " \"--rm\",\n",
+ " \"-v\", f\"{OUTPUT_FOLDER}:/output\",\n",
+ " \"orfeotoolbox/otb:9.0.0\",\n",
+ " \"otbcli_GridBasedImageResampling\",\n",
+ " \"-io.in\",\n",
+ " f\"/output/raw/{raw_name}\",\n",
+ " \"-io.out\",\n",
+ " f\"/output/otb_no_georef/{ortho_name}\",\n",
+ " \"-grid.in\",\n",
+ " f\"/output/INVERSE_GRID/{test_grid}\",\n",
+ " \"-grid.type\",\n",
+ " \"loc\",\n",
+ " \"-out.ulx\",\n",
+ " str(corrected_ulx),\n",
+ " \"-out.uly\",\n",
+ " str(corrected_uly),\n",
+ " \"-out.spacingx\",\n",
+ " str(resolution),\n",
+ " \"-out.spacingy\",\n",
+ " str(-resolution),\n",
+ " \"-out.sizex\",\n",
+ " str(width),\n",
+ " \"-out.sizey\",\n",
+ " str(height)\n",
+ " ]\n",
+ "\n",
+ " print(\"\\n=== OTB RESAMPLING ===\\n\")\n",
+ " print(\" \".join(cmd_otb))\n",
+ "\n",
+ " try:\n",
+ " subprocess.run(cmd_otb, check=True)\n",
+ " print(\"OTB success\")\n",
+ " \n",
+ " except subprocess.CalledProcessError:\n",
+ " print(\"OTB failed (outside grid) → skipping\")\n",
+ " continue\n",
+ "\n",
+ " # ==================a===============================\n",
+ " # Add georeferencing to correct otb missing one\n",
+ " # =================================================\n",
+ " cmd_georef = [\n",
+ " \"docker\", \"run\",\n",
+ " \"--rm\",\n",
+ " \"-v\", f\"{OUTPUT_FOLDER}:/output\",\n",
+ " \"gdal-latest\",\n",
+ " \"-c\",\n",
+ " f'''\n",
+ " gdal_translate \\\n",
+ " -a_srs EPSG:{UTM_EPSG} \\\n",
+ " /output/otb_no_georef/{ortho_name} \\\n",
+ " /output/output_georef/{georef_name}\n",
+ " '''\n",
+ " ]\n",
+ "\n",
+ " print(\"\\n=== ADD GEOREF ===\\n\")\n",
+ " print(\" \".join(cmd_georef))\n",
+ " subprocess.run(cmd_georef, check=True)\n",
+ " print(\"\\nFinished: \", georef_name)\n",
+ "\n",
+ "# =====================================================\n",
+ "# Mosaic\n",
+ "# =====================================================\n",
+ "print(\"\\n=== MOSAIC CREATION ===\\n\")\n",
+ "bands = set()\n",
+ "for f in glob.glob(os.path.join(GEOREF_DIR, \"*.tif\")):\n",
+ " match = re.search(r\"_B(\\d{2}[A]?)\", f)\n",
+ " if match:\n",
+ " bands.add(f\"B{match.group(1)}\")\n",
+ "print(\"Bands detected: \", bands)\n",
+ "\n",
+ "for band in bands:\n",
+ "\n",
+ " print(\"\\n----------------------------------------\")\n",
+ " print(\"Creating mosaic for band: \", band)\n",
+ " print(\"----------------------------------------\")\n",
+ " # fichiers correspondant à la bande\n",
+ " input_files = glob.glob(\n",
+ " os.path.join(GEOREF_DIR, f\"*_{band}_georef.tif\")\n",
+ " )\n",
+ " if len(input_files) == 0:\n",
+ " print(\"No ortho images found for band\", band)\n",
+ " continue\n",
+ " input_files = sorted(input_files)\n",
+ " \n",
+ " output_path = os.path.join(\n",
+ " MOSAIC_DIR,\n",
+ " f\"ORTHO_mosaic_{band}.tif\"\n",
+ " )\n",
+ " print(\"Found\", len(input_files), \"files\")\n",
+ " for f in input_files:\n",
+ " print(\" \", os.path.basename(f))\n",
+ " \n",
+ " gdal_cmd = \"gdal_merge.py \" \\\n",
+ " f\"-o /output/mosaic/ORTHO_mosaic_{band}.tif \" \\\n",
+ " \"-of GTiff \" \\\n",
+ " \"-co COMPRESS=LZW \" \\\n",
+ " \"-co TILED=YES \" \\\n",
+ " \"-ot UInt16 \" \\\n",
+ " \"-n 0 -a_nodata 0 \"\n",
+ "\n",
+ " for f in input_files:\n",
+ " fname = os.path.basename(f)\n",
+ " gdal_cmd += f\"/output/output_georef/{fname} \"\n",
+ "\n",
+ " cmd_mosaic = [\n",
+ " \"docker\", \"run\",\n",
+ " \"--rm\",\n",
+ " \"-v\", f\"{OUTPUT_FOLDER}:/output\",\n",
+ " \"gdal-latest\",\n",
+ " \"-c\",\n",
+ " gdal_cmd\n",
+ " ]\n",
+ "\n",
+ " print(\"\\nRunning mosaic:\\n\", gdal_cmd)\n",
+ " subprocess.run(cmd_mosaic, check=True)\n",
+ " print(\"Mosaic written →\", output_path)\n",
+ "\n",
+ "# =====================================================\n",
+ "# Cleanup docker image\n",
+ "# =====================================================\n",
+ "if REMOVE_DOCKER_IMAGE:\n",
+ " print(\"Removing gdal-latest image...\\n\")\n",
+ " subprocess.run([\"docker\", \"rmi\", \"-f\", \"gdal-latest\"], check=True)\n",
+ " print(\"GDAL image removed.\\n\")\n",
+ "else:\n",
+ " print(\"Docker images kept (faster next run, ~4 GB disk usage).\\n\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f3f9c566-f0ba-42fa-872e-333f9f41b851",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.13.11"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/sen2vm-notebook/src/notebook.ipynb b/sen2vm-notebook/src/notebook-direct-grid.ipynb
similarity index 74%
rename from sen2vm-notebook/src/notebook.ipynb
rename to sen2vm-notebook/src/notebook-direct-grid.ipynb
index 8cd94e73..14431f3f 100644
--- a/sen2vm-notebook/src/notebook.ipynb
+++ b/sen2vm-notebook/src/notebook-direct-grid.ipynb
@@ -2,16 +2,16 @@
"cells": [
{
"cell_type": "markdown",
- "id": "e8bbfd73",
+ "id": "8e5ed8d8-c718-4f80-a220-464bbc67c748",
"metadata": {},
"source": [
- "# === USER CONFIGURATION ===\n"
+ "# === USER CONFIGURATION ==="
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "3be1da11",
+ "id": "2b816cd9-5254-452b-bced-2943eb6b7aec",
"metadata": {},
"outputs": [],
"source": [
@@ -21,103 +21,75 @@
"# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
"\n",
"WORKDIR = \"PATH/TO/WORKDIR\" # Working directory for sen2vm processing\n",
- "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
- "# /!\\/!\\/!\\ CAREFUL, the structure of the WORKDIR shall respect the one described in README_Notebooks.md\n",
- "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
"\n",
"# Path to downloaded product\n",
- "PATH_L1B_DATA = \"PATH/TO/L1B/PRODUCT\"\n",
+ "PATH_L1B_DATA = \"PATH/TO/L1B_PRODUCT\"\n",
+ "\n",
+ "# Path to GIPP directory\n",
+ "PATH_GIPP = \"PATH/TO/GIPP\"\n",
"# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
- "# /!\\/!\\/!\\ CAREFUL, for now PATH_L1B_DATA shall be inside WORKID/DATA and respect the one described in README_Notebooks.md\n",
+ "# /!\\/!\\/!\\ If you do not have your own GIPP folder, you may use the inputs-download-notebook to download it, then you may indicate the path were you \n",
+ "# /!\\/!\\/!\\ downloaded it here in PATH_GIPP\n",
+ "# /!\\/!\\/!\\ If you have your own GIPP folder, please note that this current notebook will search for a subfolder with mission S2[A/B/C] inside the GIPP folder\n",
"# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
"\n",
- "# Output folder chosen by the user\n",
- "INVERSE_OUTPUT_FOLDER = \"PATH/TO/OUTPUT/FOLDER\" # Used only if GRID_MODE = \"inverse\"\n",
- "\n",
+ "# Path to DEM directory \n",
+ "PATH_DEM = \"PATH/TO/DEM\"\n",
"\n",
- "# === SEN2VM OPTIONS ===\n",
+ "# Path to GEOID directory\n",
+ "PATH_GEOID = \"PATH/TO/GEOID\"\n",
+ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
+ "# /!\\/!\\/!\\ If you do not have your own GEOID folder, you may use the inputs-download-notebook to download it, then you may indicate the path were you\n",
+ "# /!\\/!\\/!\\ downloaded it here in \n",
+ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
"\n",
- "GRID_MODE = \"direct\" # direct or inverse\n",
+ "# Path to IERS file\n",
+ "PATH_IERS = \"PATH/TO/IERS_FILE\"\n",
"# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
- "# /!\\/!\\/!\\ CAREFUL, for now only direct shall be used with a full product (no missing granules)\n",
+ "# /!\\/!\\/!\\ If you do not have your own IERS, you may let this path empy (i.e. \"\") and execute the cell number 2 named #IERS Download\n",
+ "# /!\\/!\\/!\\ In this case, the IERS will be downloaded in the WORKDIR\n",
"# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
"\n",
- "UTM_EPSG = 32636 # UTM zone EPSG code for the ROI\n",
+ "# The output folder is put in the WORKDIR\n",
+ "OUTPUT_FOLDER = WORKDIR + \"/output/DIRECT_gdal\"\n",
+ "\n",
+ "# === SEN2VM OPTIONS ===\n",
+ "UTM_EPSG = 32628 # UTM zone EPSG code for the ROI\n",
"LOCATION = {\n",
- " \"ul_x\": 210000, \n",
- " \"ul_y\": 3504541, \n",
- " \"lr_x\": 436397, \n",
- " \"lr_y\": 3296817 \n",
+ " \"ul_x\": 283910,\n",
+ " \"ul_y\": 3641660,\n",
+ " \"lr_x\": 354280,\n",
+ " \"lr_y\": 3608316.0\n",
"}\n",
"\n",
"# Grid step in pixels for sen2vm grid generation \n",
- "# These values represent the grid step in pixels, typically DEM resolution / 2\n",
- "# For S2 with DEM90, this gives approximately: 45m/10m=4.5, 45m/20m=2.25, 45m/60m≈1\n",
+ "# These values represent the grid step in pixels\n",
+ "# In operation, this step is linked to DEM step which is not constant over the world, pending the Latitude\n",
"STEPS = {\n",
- " \"10m_bands\": 4.5, # Grid step: ~DEM90/2 = 45m/10m = 4.5 pixels (one point every 4.5 pixels at 10m)\n",
- " \"20m_bands\": 2.25, # Grid step: ~DEM90/2 = 45m/20m = 2.25 pixels (one point every 2.25 pixels at 20m)\n",
- " \"60m_bands\": 1 # Grid step: ~DEM90/2 = 45m/60m ≈ 1 pixel (one point every 1 pixel at 60m, or 2 is also fine)\n",
+ " \"10m_bands\": 4.5, # Around 45m in operation (pending laittude) with 30m and also 90m DEM\n",
+ " \"20m_bands\": 4.5, # Around 90m in operation (pending laittude) with 30m and also 90m DEM\n",
+ " \"60m_bands\": 3.0 # Around 180m in operation (pending laittude) with 30m and also 90m DEM\n",
"}\n",
"\n",
- "\n",
"# === GDAL ORTHO OPTIONS ===\n",
"\n",
"ORTHO_SETTINGS = {\n",
- " \"keep_bands\": [\"B03\"], # list of bands to keep for orthorectification\n",
- " \"keep_detectors\": [\"11\",\"10\"] # list of the detectors to keep \n",
+ " \"keep_bands\": [\"B01\", \"B02\", \"B03\", \"B04\", \"B05\", \"B06\", \"B07\", \"B08\", \"B8A\", \"B09\", \"B10\", \"B11\", \"B12\"], # list of bands to keep for orthorectification\n",
+ " \"keep_detectors\": [\"07\",\"08\",\"09\",\"10\",\"11\"] # list of the detectors to keep \n",
"}\n",
- " \n"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "c9e95184",
- "metadata": {},
- "source": [
- "# GIPP database download (Optional)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "a4e410c1",
- "metadata": {},
- "outputs": [],
- "source": [
- "# === GIPPs ===\n",
- "# This step is optional if the Database was already downloaded or if you want to use your own GIPP\n",
- "# Please note that this current notebook will search for a subfolder with mission S2[A/B/C] inside the GIPP folder\n",
"\n",
- "import os\n",
- "import shutil\n",
- "import tarfile\n",
- "import re\n",
- "from datetime import datetime\n",
- "\n",
- "\n",
- "gipp_dir = os.path.join(WORKDIR, \"DATA\") \n",
- "os.makedirs(gipp_dir, exist_ok=True) \n",
- "\n",
- "# =====================================================================\n",
- "# 1) CLONE sen2vm-gipp-database\n",
- "# =====================================================================\n",
- "\n",
- "gipp_repo_name = \"sen2vm-gipp-database\"\n",
- "gipp_repo_path = os.path.join(gipp_dir, gipp_repo_name)\n",
- "\n",
- "# Delete repository if exists\n",
- "if os.path.exists(gipp_repo_path):\n",
- " print(f\"Removing existing repository: {gipp_repo_path}\")\n",
- " shutil.rmtree(gipp_repo_path)\n",
- "\n",
- "# Clone repository fresh\n",
- "print(\"Cloning sen2vm-gipp-database...\")\n",
- "!git clone https://github.com/sen2vm/sen2vm-gipp-database.git {gipp_repo_path}\n",
- "print(\"Clone complete.\\n\")\n",
- "\n",
- "\n",
- "\n",
- "print(\"GIPP processing finished successfully.\")\n"
+ "# === Docker Options ===\n",
+ "# Building Docker images may take several minutes.\n",
+ "# By setting REMOVE_DOCKER_IMAGE = False, the images are kept,\n",
+ "# which significantly speeds up subsequent executions.\n",
+ "#\n",
+ "# However, Docker images use disk space (~4 GB in this case).\n",
+ "#\n",
+ "# Docker images can be manually removed using:\n",
+ "# docker images\n",
+ "# docker rmi \n",
+ "#\n",
+ "REMOVE_DOCKER_IMAGE = False # True = remove Docker images after execution"
]
},
{
@@ -125,7 +97,7 @@
"id": "63f616a3",
"metadata": {},
"source": [
- "# IERS Download (optionnal)"
+ "# IERS Download"
]
},
{
@@ -136,36 +108,38 @@
"outputs": [],
"source": [
"# === DOWNLOAD IERS ===\n",
- "\n",
"import os\n",
"import re\n",
"import requests\n",
+ "import sys\n",
"from datetime import datetime\n",
"\n",
- "DATA_DIR = os.path.join(WORKDIR, \"DATA\")\n",
+ "if len(PATH_IERS) != 0: # Safety in case the PATH_IERS isn't empty\n",
+ " print(\"Stopping because PATH_IERS is not empty\")\n",
+ " sys.exit(0)\n",
"\n",
- "if not os.path.exists(DATA_DIR):\n",
- " raise RuntimeError(f\"DATA directory not found: {DATA_DIR}\")\n",
+ "if not os.path.exists(WORKDIR):\n",
+ " raise RuntimeError(f\"WORKDIR directory not found: {WORKDIR}\")\n",
"\n",
- "print(\"Bulletin output directory:\", DATA_DIR)\n",
+ "print(\"Bulletin output directory: \", WORKDIR)\n",
"\n",
+ "PATH_IERS = WORKDIR # then our file will be there\n",
"# =====================================================================\n",
- "# 0. REMOVE EXISTING IERS BULLETINS\n",
+ "# REMOVE EXISTING IERS BULLETINS\n",
"# =====================================================================\n",
"\n",
- "for f in os.listdir(DATA_DIR):\n",
+ "for f in os.listdir(WORKDIR):\n",
" if f.startswith(\"bulletina-\") and f.endswith(\".txt\"):\n",
- " os.remove(os.path.join(DATA_DIR, f))\n",
- " print(\"Removed old bulletin A:\", f)\n",
+ " os.remove(os.path.join(WORKDIR, f))\n",
+ " print(\"Removed old bulletin A: \", f)\n",
"\n",
" if f.startswith(\"bulletinb-\") and f.endswith(\".txt\"):\n",
- " os.remove(os.path.join(DATA_DIR, f))\n",
- " print(\"Removed old bulletin B:\", f)\n",
+ " os.remove(os.path.join(WORKDIR, f))\n",
+ " print(\"Removed old bulletin B: \", f)\n",
"\n",
"print(\"Cleanup of old bulletins complete.\\n\")\n",
- "\n",
"# =====================================================================\n",
- "# 2. EXTRACT PRODUCT DATE (FROM DATASTRIP)\n",
+ "# EXTRACT PRODUCT DATE (FROM DATASTRIP)\n",
"# =====================================================================\n",
"\n",
"datastrip_dir = os.path.join(PATH_L1B_DATA, \"DATASTRIP\")\n",
@@ -192,8 +166,7 @@
"\n",
"product_date = datetime(year, month, day)\n",
"\n",
- "print(\"Product date extracted from DATASTRIP:\", product_date.date(), \"\\n\")\n",
- "\n",
+ "print(\"Product date extracted from DATASTRIP: \", product_date.date(), \"\\n\")\n",
"# =====================================================================\n",
"# Roman conversion \n",
"# =====================================================================\n",
@@ -210,13 +183,10 @@
" res += s\n",
" n -= v\n",
" return res\n",
- "\n",
- "\n",
"# =====================================================================\n",
"# Bulletin A\n",
"# =====================================================================\n",
"\n",
- "\n",
"roman_year = int_to_roman(year - 1987)\n",
"doy = product_date.timetuple().tm_yday\n",
"index = (doy - 1) // 7 + 1\n",
@@ -229,13 +199,13 @@
"while index > 0:\n",
" index_str = f\"{index:03d}\"\n",
" url = f\"https://datacenter.iers.org/data/6/bulletina-{roman_year}-{index_str}.txt\"\n",
- " print(\"Trying bulletin:\", url)\n",
+ " print(\"Trying bulletin: \", url)\n",
"\n",
" response = requests.get(url)\n",
"\n",
" if response.status_code == 200:\n",
- " print(\"Bulletin found:\", index_str)\n",
- " dest_file = os.path.join(DATA_DIR, f\"bulletina-{roman_year}-{index_str}.txt\")\n",
+ " print(\"Bulletin found: \", index_str)\n",
+ " dest_file = os.path.join(WORKDIR, f\"bulletina-{roman_year}-{index_str}.txt\")\n",
" found = True\n",
" break\n",
"\n",
@@ -247,7 +217,7 @@
"with open(dest_file, \"wb\") as f:\n",
" f.write(response.content)\n",
"\n",
- "print(\"Downloaded:\", dest_file)\n"
+ "print(\"Downloaded: \", dest_file)"
]
},
{
@@ -273,16 +243,11 @@
"import shutil\n",
"from numpy import double\n",
"\n",
- "\n",
"USERCONF_DIR = os.path.join(WORKDIR, \"UserConf\")\n",
"os.makedirs(USERCONF_DIR, exist_ok=True)\n",
"\n",
- "print(\"UserConf directory:\", USERCONF_DIR)\n",
- "\n",
- "GEOID_DIR = os.path.join(WORKDIR, \"DATA\", \"GEOID\")\n",
- "os.makedirs(GEOID_DIR, exist_ok=True)\n",
- "\n",
- "print(\"Geoid directory:\", GEOID_DIR)\n",
+ "print(\"UserConf directory: \", USERCONF_DIR)\n",
+ "print(\"Geoid directory: \", PATH_GEOID)\n",
"\n",
"# =====================================================\n",
"# 1. Extract mission from DATASTRIP\n",
@@ -304,63 +269,43 @@
" raise RuntimeError(\"Cannot extract mission (S2A/S2B/S2C) from DATASTRIP name.\")\n",
"\n",
"mission = match.group(1)\n",
- "\n",
- "\n",
"# =====================================================\n",
"# 2. Docker paths inside /workspace\n",
"# =====================================================\n",
"\n",
- "safe_name = os.path.basename(PATH_L1B_DATA)\n",
- "\n",
- "docker_l1b = f\"/workspace/DATA/{safe_name}\"\n",
- "docker_dem = \"/workspace/DATA/DEM\"\n",
- "docker_gipp = f\"/workspace/DATA/sen2vm-gipp-database/{mission}\"\n",
- "\n",
+ "docker_l1b = \"/data/L1B\"\n",
+ "docker_dem = \"/data/DEM\"\n",
+ "docker_gipp = f\"/data/GIPP/{mission}\"\n",
"# =====================================================\n",
"# 3. Geoid management\n",
"# =====================================================\n",
"\n",
- "# Notebook location (NOT relative to CWD)\n",
- "notebook_dir = os.getcwd()\n",
- "\n",
- "# Relative path to DEM_GEOID from notebook\n",
- "internal_geoid_dir = os.path.abspath(os.path.join(\n",
- " notebook_dir,\n",
- " \"..\", \"..\", \"src\", \"test\", \"resources\", \"DEM_GEOID\"\n",
- "))\n",
- "\n",
- "# If WORKDIR/DATA/GEOID is empty -> copy internal files\n",
- "if len(os.listdir(GEOID_DIR)) == 0:\n",
- " print(\"GEOID directory is empty -> copying default geoid files...\")\n",
- " for f in os.listdir(internal_geoid_dir):\n",
- " src = os.path.join(internal_geoid_dir, f)\n",
- " dst = os.path.join(GEOID_DIR, f)\n",
- " shutil.copy(src, dst)\n",
- "\n",
- "# Detect .gtx inside GEOID_DIR\n",
- "geoid_files = [f for f in os.listdir(GEOID_DIR) if f.lower().endswith(\".gtx\")]\n",
+ "# Detect .gtx inside PATH_GEOID\n",
+ "geoid_files = [f for f in os.listdir(PATH_GEOID) if f.lower().endswith(\".gtx\")]\n",
"if len(geoid_files) == 0:\n",
- " raise RuntimeError(\"No .gtx geoid file found in WORKDIR/DATA/GEOID.\")\n",
- "\n",
- "docker_geoid = f\"/workspace/DATA/GEOID/{geoid_files[0]}\"\n",
+ " raise RuntimeError(\"No .gtx geoid file found in PATH_GEOID.\")\n",
"\n",
+ "docker_geoid = f\"/data/GEOID/{geoid_files[0]}\"\n",
"# =====================================================\n",
"# 4. Locate IERS bulletin on host\n",
"# =====================================================\n",
"\n",
- "DATA_DIR = os.path.join(WORKDIR, \"DATA\")\n",
"iers_host = None\n",
"\n",
- "for f in os.listdir(DATA_DIR):\n",
- " if f.startswith(\"bulletin\"):\n",
- " iers_host = os.path.join(DATA_DIR, f)\n",
- " break\n",
+ "if os.path.isfile(PATH_IERS):\n",
+ " iers_host = PATH_IERS\n",
+ " PATH_IERS = os.path.dirname(PATH_IERS)\n",
+ "elif os.path.isdir(PATH_IERS):\n",
+ " for f in os.listdir(PATH_IERS):\n",
+ " if f.startswith(\"bulletin\"):\n",
+ " iers_host = os.path.join(PATH_IERS, f)\n",
+ " break\n",
"\n",
"if iers_host is None:\n",
- " raise RuntimeError(\"IERS bulletin not found inside WORKDIR/DATA directory.\")\n",
- "\n",
- "docker_iers = \"/workspace/DATA/\" + os.path.basename(iers_host)\n",
+ " raise RuntimeError(\"IERS bulletin not found inside PATH_IERS directory.\")\n",
"\n",
+ "docker_iers = f\"/data/IERS/{os.path.basename(iers_host)}\"\n",
+ "print(\"IERS: \", docker_iers)\n",
"# =====================================================\n",
"# 5. Build config dictionary\n",
"# =====================================================\n",
@@ -373,7 +318,7 @@
" \"dem\": docker_dem,\n",
" \"geoid\": docker_geoid,\n",
" \"iers\": docker_iers,\n",
- " \"operation\": GRID_MODE,\n",
+ " \"operation\": \"direct\",\n",
" \"deactivate_available_refining\": False,\n",
" \"steps\": {\n",
" \"10m_bands\": STEPS[\"10m_bands\"],\n",
@@ -382,18 +327,6 @@
" },\n",
" \"export_alt\": True\n",
"}\n",
- "\n",
- "# Add inverse block only if needed\n",
- "if GRID_MODE == \"inverse\":\n",
- " config[\"inverse_location_additional_info\"] = {\n",
- " \"ul_x\": double(LOCATION[\"ul_x\"]),\n",
- " \"ul_y\": double(LOCATION[\"ul_y\"]),\n",
- " \"lr_x\": double(LOCATION[\"lr_x\"]),\n",
- " \"lr_y\": double(LOCATION[\"lr_y\"]),\n",
- " \"referential\": f\"EPSG:{UTM_EPSG}\",\n",
- " \"output_folder\": \"/workspace/DATA/Output\"\n",
- " }\n",
- "\n",
"# =====================================================\n",
"# Save config.json\n",
"# =====================================================\n",
@@ -403,8 +336,8 @@
"with open(config_path, \"w\") as f:\n",
" json.dump(config, f, indent=4)\n",
"\n",
- "print(\"Configuration file generated:\")\n",
- "print(config_path)\n"
+ "print(\"Configuration file generated: \")\n",
+ "print(config_path)"
]
},
{
@@ -423,11 +356,11 @@
"USERCONF_DIR = os.path.join(WORKDIR, \"UserConf\")\n",
"os.makedirs(USERCONF_DIR, exist_ok=True)\n",
"\n",
- "print(\"UserConf directory:\", USERCONF_DIR)\n",
- "\n",
+ "print(\"UserConf directory: \", USERCONF_DIR)\n",
"# =====================================================\n",
"# Locate GRANULE folders\n",
"# =====================================================\n",
+ "\n",
"GR_TARGET_DIR = os.path.join(PATH_L1B_DATA, \"GRANULE\")\n",
"\n",
"if not os.path.exists(GR_TARGET_DIR):\n",
@@ -439,11 +372,11 @@
" if os.path.isdir(os.path.join(GR_TARGET_DIR, d))\n",
"]\n",
"\n",
- "print(\"Found\", len(granule_folders), \"granule folders.\")\n",
- "\n",
+ "print(\"Found \", len(granule_folders), \" granule folders.\")\n",
"# =====================================================\n",
"# Extract detectors and bands from JP2\n",
"# =====================================================\n",
+ "\n",
"detectors = set()\n",
"bands = set()\n",
"\n",
@@ -464,9 +397,8 @@
"detectors = sorted(detectors)\n",
"bands = sorted(bands)\n",
"\n",
- "print(\"Detected detectors:\", detectors)\n",
- "print(\"Detected bands:\", bands)\n",
- "\n",
+ "print(\"Detected detectors: \", detectors)\n",
+ "print(\"Detected bands: \", bands)\n",
"# =====================================================\n",
"# Write params.json\n",
"# =====================================================\n",
@@ -495,9 +427,9 @@
"with open(params_path, \"w\") as f:\n",
" json.dump(params, f, indent=4)\n",
"\n",
- "print(\"params.json written to:\", params_path)\n",
+ "print(\"params.json written to: \", params_path)\n",
"print(f\" - Detectors: {params['detectors']}\")\n",
- "print(f\" - Bands: {params['bands']}\")\n"
+ "print(f\" - Bands: {params['bands']}\")"
]
},
{
@@ -520,6 +452,8 @@
"import os\n",
"import subprocess\n",
"\n",
+ "# Notebook location (NOT relative to CWD)\n",
+ "notebook_dir = os.getcwd()\n",
"\n",
"dockerfile_dir = os.path.abspath(os.path.join(\n",
" notebook_dir,\n",
@@ -528,13 +462,11 @@
"\n",
"config_inside = \"/workspace/UserConf/config.json\"\n",
"params_inside = \"/workspace/UserConf/params.json\"\n",
- "\n",
"# =====================================================\n",
"# 1. BUILD DOCKER IMAGE\n",
"# =====================================================\n",
"\n",
"print(f\"Building Docker image 'sen2vm' from: {dockerfile_dir}\")\n",
- "\n",
"cmd_build = [\n",
" \"docker\", \"build\",\n",
" \"-t\", \"sen2vm\",\n",
@@ -544,7 +476,6 @@
"print(\"Command:\", \" \".join(cmd_build), \"\\n\")\n",
"subprocess.run(cmd_build, check=True)\n",
"print(\"Docker image built successfully.\\n\")\n",
- "\n",
"# =====================================================\n",
"# 2. RUN SEN2VM CONTAINER\n",
"# =====================================================\n",
@@ -552,7 +483,12 @@
"cmd_run = [\n",
" \"docker\", \"run\",\n",
" \"--rm\",\n",
- " \"-v\", f\"{WORKDIR}:/workspace\", \n",
+ " \"-v\", f\"{PATH_L1B_DATA}:/data/L1B\",\n",
+ " \"-v\", f\"{PATH_DEM}:/data/DEM\",\n",
+ " \"-v\", f\"{PATH_GIPP}:/data/GIPP\",\n",
+ " \"-v\", f\"{PATH_GEOID}:/data/GEOID\",\n",
+ " \"-v\", f\"{PATH_IERS}:/data/IERS\",\n",
+ " \"-v\", f\"{WORKDIR}:/workspace\",\n",
" \"sen2vm\",\n",
" \"-c\", config_inside,\n",
" \"-p\", params_inside\n",
@@ -560,20 +496,18 @@
"\n",
"print(\"Running Docker container...\\n\")\n",
"print(\"Command:\", \" \".join(cmd_run), \"\\n\")\n",
- "\n",
"subprocess.run(cmd_run, check=True)\n",
- "\n",
"print(\"\\nDocker execution complete.\\n\")\n",
- "\n",
"# =====================================================\n",
"# 3. REMOVE DOCKER IMAGE\n",
"# =====================================================\n",
"\n",
- "print(\"Removing Docker image 'sen2vm'...\")\n",
- "\n",
- "subprocess.run([\"docker\", \"rmi\", \"-f\", \"sen2vm\"], check=True)\n",
- "\n",
- "print(\"Docker image removed.\\n\")"
+ "if REMOVE_DOCKER_IMAGE:\n",
+ " print(\"Removing Docker image 'sen2vm'...\")\n",
+ " subprocess.run([\"docker\", \"rmi\", \"-f\", \"sen2vm\"], check=True)\n",
+ " print(\"Docker images removed.\\n\")\n",
+ "else:\n",
+ " print(\"Docker images kept.\\n\")"
]
},
{
@@ -598,6 +532,13 @@
"import re\n",
"\n",
"# =====================================================\n",
+ "# SAFETY CHECKS\n",
+ "# =====================================================\n",
+ "assert os.path.exists(PATH_L1B_DATA), \"L1B path missing\"\n",
+ "os.makedirs(OUTPUT_FOLDER, exist_ok=True)\n",
+ "assert os.path.exists(OUTPUT_FOLDER), \"Output folder missing\"\n",
+ "\n",
+ "# =====================================================\n",
"# Locate product name\n",
"# =====================================================\n",
"product = os.path.basename(os.path.normpath(PATH_L1B_DATA))\n",
@@ -621,7 +562,7 @@
"# XML path inside docker (relative, after cd)\n",
"xml_docker = f\"./{xml_name}\"\n",
"\n",
- "print(\"Using XML:\", xml_path)\n",
+ "print(\"Using XML: \", xml_path)\n",
"\n",
"# =====================================================\n",
"# Read params.json to get selected bands and detectors\n",
@@ -678,10 +619,10 @@
"# =====================================================\n",
"# Output directories\n",
"# =====================================================\n",
- "OUTDIR = os.path.join(WORKDIR, \"DATA\", \"GDAL_OUTPUT_ORTHO\")\n",
- "os.makedirs(OUTDIR, exist_ok=True)\n",
+ "os.makedirs(os.path.join(OUTPUT_FOLDER, \"GDAL_OUTPUT_ORTHO\"), exist_ok=True)\n",
+ "os.makedirs(os.path.join(OUTPUT_FOLDER, \"GDAL_OUTPUT_MOSAIC\"), exist_ok=True)\n",
"\n",
- "OUTDIR_DOCKER = \"/workspace/DATA/GDAL_OUTPUT_ORTHO\"\n",
+ "print(\"Output folder: \", OUTPUT_FOLDER)\n",
"\n",
"# =====================================================\n",
"# Build GDAL docker\n",
@@ -733,10 +674,10 @@
" f.write(f\"\"\"#!/bin/bash\n",
"set +e\n",
"\n",
- "cd /workspace/DATA/{product}\n",
+ "cd /data/L1B\n",
"\n",
- "OUT_ORTHO=\"/workspace/DATA/GDAL_OUTPUT_ORTHO\"\n",
- "OUT_MOSAIC=\"/workspace/DATA/GDAL_OUTPUT_MOSAIC\"\n",
+ "OUT_ORTHO=\"/output/GDAL_OUTPUT_ORTHO\"\n",
+ "OUT_MOSAIC=\"/output/GDAL_OUTPUT_MOSAIC\"\n",
"\n",
"mkdir -p \"$OUT_ORTHO\"\n",
"mkdir -p \"$OUT_MOSAIC\"\n",
@@ -771,15 +712,14 @@
"echo \"=== ORTHORECTIFICATION ===\"\n",
"\n",
"for VRT in {vrt_array}; do\n",
- " BASENAME=\"$VRT\"\n",
- " OUT=\"$OUT_ORTHO/${{BASENAME}}_ortho.tif\"\n",
- "\n",
+ " OUT=\"$OUT_ORTHO/${{VRT}}_ortho.tif\"\n",
+ " \n",
" echo \"----------------------------------------\"\n",
" echo \"Processing VRT: $VRT\"\n",
" echo \"----------------------------------------\"\n",
"\n",
" rm -f \"$OUT\"\n",
- "\n",
+ " \n",
" # Get resolution for this band (Issue #61)\n",
" RESOLUTION=$(get_band_resolution \"$VRT\")\n",
" echo \"Band resolution: $RESOLUTION m\"\n",
@@ -796,8 +736,6 @@
" -co COMPRESS=LZW \\\\\n",
" -co TILED=YES \\\\\n",
" -overwrite\n",
- "\n",
- " echo \"\"\n",
"done\n",
"\n",
"echo \"\"\n",
@@ -808,16 +746,15 @@
" echo \"----------------------------------------\"\n",
" echo \"Creating mosaic for band: $BAND\"\n",
" echo \"----------------------------------------\"\n",
- "\n",
- " INPUT_FILES=($(ls $OUT_ORTHO/*_${{BAND}}_ortho.tif 2>/dev/null))\n",
+ " \n",
+ " INPUT_FILES=($OUT_ORTHO/*_${{BAND}}_ortho.tif)\n",
"\n",
" if [ ${{#INPUT_FILES[@]}} -eq 0 ]; then\n",
" echo \"No ortho images found for band $BAND\"\n",
" continue\n",
" fi\n",
- "\n",
" OUTPUT=\"$OUT_MOSAIC/ORTHO_mosaic_${{BAND}}.tif\"\n",
- "\n",
+ " \n",
" # Use gdal_merge.py instead of gdalwarp to avoid double resampling \n",
" # All ortho images should have the same resolution and geometry\n",
" # No resampling needed, just assembly\n",
@@ -839,7 +776,6 @@
"\"\"\")\n",
"\n",
"os.chmod(gdal_script_path, 0o755)\n",
- "print(\"Generated:\", gdal_script_path)\n",
"\n",
"# =====================================================\n",
"# Run GDAL docker\n",
@@ -849,7 +785,9 @@
"cmd_run = [\n",
" \"docker\", \"run\",\n",
" \"--rm\",\n",
+ " \"-v\", f\"{PATH_L1B_DATA}:/data/L1B\",\n",
" \"-v\", f\"{WORKDIR}:/workspace\",\n",
+ " \"-v\", f\"{OUTPUT_FOLDER}:/output\",\n",
" \"gdal-latest\",\n",
" \"/workspace/src/gdal_ortho.sh\"\n",
"]\n",
@@ -857,14 +795,25 @@
"print(\"Command:\", \" \".join(cmd_run), \"\\n\")\n",
"subprocess.run(cmd_run, check=True)\n",
"print(\"\\nGDAL ortho + mosaic complete.\\n\")\n",
- "\n",
"# =====================================================\n",
"# Cleanup docker image\n",
"# =====================================================\n",
- "print(\"Removing gdal-latest image...\\n\")\n",
- "subprocess.run([\"docker\", \"rmi\", \"-f\", \"gdal-latest\"], check=True)\n",
- "print(\"GDAL image removed.\\n\")\n"
+ "\n",
+ "if REMOVE_DOCKER_IMAGE:\n",
+ " print(\"Removing gdal-latest image...\\n\")\n",
+ " subprocess.run([\"docker\", \"rmi\", \"-f\", \"gdal-latest\"], check=True)\n",
+ " print(\"GDAL image removed.\\n\")\n",
+ "else:\n",
+ " print(\"Docker images kept (faster next run, ~4 GB disk usage).\\n\")"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "838b73c6-4666-4492-8882-be899a53b2da",
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {
@@ -883,7 +832,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.11.11"
+ "version": "3.13.11"
}
},
"nbformat": 4,
diff --git a/sen2vm-notebook/src/notebook-inverse-grid.ipynb b/sen2vm-notebook/src/notebook-inverse-grid.ipynb
new file mode 100644
index 00000000..de748503
--- /dev/null
+++ b/sen2vm-notebook/src/notebook-inverse-grid.ipynb
@@ -0,0 +1,900 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "8e5ed8d8-c718-4f80-a220-464bbc67c748",
+ "metadata": {},
+ "source": [
+ "# === USER CONFIGURATION ==="
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2b816cd9-5254-452b-bced-2943eb6b7aec",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# # === USER CONFIGURATION ===\n",
+ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
+ "# /!\\/!\\/!\\ CAREFUL, please read the README_Notebooks.md file before\n",
+ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
+ "\n",
+ "WORKDIR = \"PATH/TO/WORKDIR\" # Working directory for sen2vm processing\n",
+ "\n",
+ "# Path to downloaded product\n",
+ "PATH_L1B_DATA = \"PATH/TO/L1B_PRODUCT\"\n",
+ "\n",
+ "# Path to GIPP directory\n",
+ "PATH_GIPP = \"PATH/TO/GIPP\"\n",
+ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
+ "# /!\\/!\\/!\\ If you do not have your own GIPP folder, you may use the inputs-download-notebook to download it, then you may indicate the path were you \n",
+ "# /!\\/!\\/!\\ downloaded it here in PATH_GIPP\n",
+ "# /!\\/!\\/!\\ If you have your own GIPP folder, please note that this current notebook will search for a subfolder with mission S2[A/B/C] inside the GIPP folder\n",
+ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
+ "\n",
+ "# Path to DEM directory \n",
+ "PATH_DEM = \"PATH/TO/DEM\"\n",
+ "\n",
+ "# Path to GEOID directory\n",
+ "PATH_GEOID = \"PATH/TO/GEOID\"\n",
+ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
+ "# /!\\/!\\/!\\ If you do not have your own GEOID folder, you may use the inputs-download-notebook to download it, then you may indicate the path were you\n",
+ "# /!\\/!\\/!\\ downloaded it here in \n",
+ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
+ "\n",
+ "# Path to IERS file\n",
+ "PATH_IERS = \"PATH/TO/IERS_FILE\"\n",
+ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
+ "# /!\\/!\\/!\\ If you do not have your own IERS, you may let this path empy (i.e. \"\") and execute the cell number 2 named #IERS Download\n",
+ "# /!\\/!\\/!\\ In this case, the IERS will be downloaded in the WORKDIR\n",
+ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n",
+ "\n",
+ "# The output folder is put in the WORKDIR\n",
+ "OUTPUT_FOLDER = WORKDIR + \"/output/INVERSE\"\n",
+ "\n",
+ "# === SEN2VM OPTIONS ===\n",
+ "UTM_EPSG = 32628 # UTM zone EPSG code for the ROI\n",
+ "LOCATION = {\n",
+ " \"ul_x\": 283910,\n",
+ " \"ul_y\": 3641660,\n",
+ " \"lr_x\": 354280,\n",
+ " \"lr_y\": 3608316.0\n",
+ "}\n",
+ "\n",
+ "# Grid step in pixels for sen2vm grid generation \n",
+ "# These values represent the grid step in pixels, it's the pixel in the detector geometry\n",
+ "# In operation, this step is linked to DEM step which is not constant over the world, pending the Latitude\n",
+ "STEPS = {\n",
+ " \"10m_bands\": 45, # Around 45m in operation (pending laittude) with 30m and also 90m DEM\n",
+ " \"20m_bands\": 90, # Around 90m in operation (pending laittude) with 30m and also 90m DEM\n",
+ " \"60m_bands\": 180 # Around 180m in operation (pending laittude) with 30m and also 90m DEM\n",
+ "}\n",
+ "\n",
+ "# === GDAL ORTHO OPTIONS ===\n",
+ "\n",
+ "ORTHO_SETTINGS = {\n",
+ " \"keep_bands\": [\"B01\", \"B02\", \"B03\", \"B04\", \"B05\", \"B06\", \"B07\", \"B08\", \"B8A\", \"B09\", \"B10\", \"B11\", \"B12\"], # list of bands to keep for orthorectification\n",
+ " \"keep_detectors\": [\"01\",\"02\",\"03\",\"04\",\"05\",\"06\",\"07\",\"08\",\"09\",\"10\",\"11\",\"12\"] # list of the detectors to keep \n",
+ "}\n",
+ "\n",
+ "# === Docker Options ===\n",
+ "# By setting REMOVE_DOCKER_IMAGE = False, the images are kept,\n",
+ "# which significantly speeds up subsequent executions.\n",
+ "#\n",
+ "# However, Docker images use disk space (~4 GB in this case).\n",
+ "#\n",
+ "# Docker images can be manually removed using:\n",
+ "# docker images\n",
+ "# docker rmi \n",
+ "#\n",
+ "REMOVE_DOCKER_IMAGE = False # True = remove Docker images after execution"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "63f616a3",
+ "metadata": {},
+ "source": [
+ "# IERS Download"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8ec4d6f6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# === DOWNLOAD IERS ===\n",
+ "import os\n",
+ "import re\n",
+ "import requests\n",
+ "import sys\n",
+ "from datetime import datetime\n",
+ "\n",
+ "if len(PATH_IERS) != 0: # Safety in case the PATH_IERS isn't empty\n",
+ " print(\"Stopping because PATH_IERS is not empty\")\n",
+ " sys.exit(0)\n",
+ "\n",
+ "if not os.path.exists(WORKDIR):\n",
+ " raise RuntimeError(f\"WORKDIR directory not found: {WORKDIR}\")\n",
+ "\n",
+ "print(\"Bulletin output directory: \", WORKDIR)\n",
+ "\n",
+ "PATH_IERS = WORKDIR # then our file will be there\n",
+ "# =====================================================================\n",
+ "# REMOVE EXISTING IERS BULLETINS\n",
+ "# =====================================================================\n",
+ "\n",
+ "for f in os.listdir(WORKDIR):\n",
+ " if f.startswith(\"bulletina-\") and f.endswith(\".txt\"):\n",
+ " os.remove(os.path.join(WORKDIR, f))\n",
+ " print(\"Removed old bulletin A: \", f)\n",
+ "\n",
+ " if f.startswith(\"bulletinb-\") and f.endswith(\".txt\"):\n",
+ " os.remove(os.path.join(WORKDIR, f))\n",
+ " print(\"Removed old bulletin B: \", f)\n",
+ "\n",
+ "print(\"Cleanup of old bulletins complete.\\n\")\n",
+ "# =====================================================================\n",
+ "# EXTRACT PRODUCT DATE (FROM DATASTRIP)\n",
+ "# =====================================================================\n",
+ "\n",
+ "datastrip_dir = os.path.join(PATH_L1B_DATA, \"DATASTRIP\")\n",
+ "\n",
+ "if not os.path.isdir(datastrip_dir):\n",
+ " raise RuntimeError(f\"DATASTRIP directory not found: {datastrip_dir}\")\n",
+ "\n",
+ "datastrip_entries = os.listdir(datastrip_dir)\n",
+ "if not datastrip_entries:\n",
+ " raise RuntimeError(f\"No DATASTRIP found in: {datastrip_dir}\")\n",
+ "\n",
+ "# Take the first DATASTRIP product\n",
+ "datastrip_name = datastrip_entries[0]\n",
+ "\n",
+ "match = re.search(r\"_S(\\d{8})T\\d{6}_\", datastrip_name)\n",
+ "if not match:\n",
+ " raise RuntimeError(\"Could not extract product date from DATASTRIP name.\")\n",
+ "\n",
+ "product_date_str = match.group(1)\n",
+ "\n",
+ "year = int(product_date_str[:4])\n",
+ "month = int(product_date_str[4:6])\n",
+ "day = int(product_date_str[6:8])\n",
+ "\n",
+ "product_date = datetime(year, month, day)\n",
+ "\n",
+ "print(\"Product date extracted from DATASTRIP: \", product_date.date(), \"\\n\")\n",
+ "# =====================================================================\n",
+ "# Roman conversion \n",
+ "# =====================================================================\n",
+ "\n",
+ "def int_to_roman(n):\n",
+ " vals = [\n",
+ " (1000, 'm'), (900, 'cm'), (500, 'd'), (400, 'cd'),\n",
+ " (100, 'c'), (90, 'xc'), (50, 'l'), (40, 'xl'),\n",
+ " (10, 'x'), (9, 'ix'), (5, 'v'), (4, 'iv'), (1, 'i')\n",
+ " ]\n",
+ " res = \"\"\n",
+ " for v, s in vals:\n",
+ " while n >= v:\n",
+ " res += s\n",
+ " n -= v\n",
+ " return res\n",
+ "# =====================================================================\n",
+ "# Bulletin A\n",
+ "# =====================================================================\n",
+ "\n",
+ "roman_year = int_to_roman(year - 1987)\n",
+ "doy = product_date.timetuple().tm_yday\n",
+ "index = (doy - 1) // 7 + 1\n",
+ "\n",
+ "print(f\"Bulletin A Roman year: {roman_year}\")\n",
+ "print(f\"Initial weekly index: {index}\\n\")\n",
+ "\n",
+ "found = False\n",
+ "\n",
+ "while index > 0:\n",
+ " index_str = f\"{index:03d}\"\n",
+ " url = f\"https://datacenter.iers.org/data/6/bulletina-{roman_year}-{index_str}.txt\"\n",
+ " print(\"Trying bulletin: \", url)\n",
+ "\n",
+ " response = requests.get(url)\n",
+ "\n",
+ " if response.status_code == 200:\n",
+ " print(\"Bulletin found: \", index_str)\n",
+ " dest_file = os.path.join(WORKDIR, f\"bulletina-{roman_year}-{index_str}.txt\")\n",
+ " found = True\n",
+ " break\n",
+ "\n",
+ " index -= 1\n",
+ "\n",
+ "if not found:\n",
+ " raise RuntimeError(\"No Bulletin A available for current or previous weeks.\")\n",
+ "\n",
+ "with open(dest_file, \"wb\") as f:\n",
+ " f.write(response.content)\n",
+ "\n",
+ "print(\"Downloaded: \", dest_file)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9e85608d",
+ "metadata": {},
+ "source": [
+ "# Sen2VM configuration"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "276592b5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# === GENERATE CONFIG.JSON ===\n",
+ "\n",
+ "import os\n",
+ "import json\n",
+ "import re\n",
+ "import shutil\n",
+ "from numpy import double\n",
+ "\n",
+ "USERCONF_DIR = os.path.join(WORKDIR, \"UserConf\")\n",
+ "os.makedirs(USERCONF_DIR, exist_ok=True)\n",
+ "\n",
+ "print(\"UserConf directory: \", USERCONF_DIR)\n",
+ "\n",
+ "print(\"Geoid directory: \", PATH_GEOID)\n",
+ "# =====================================================\n",
+ "# 1. Extract mission from DATASTRIP\n",
+ "# =====================================================\n",
+ "\n",
+ "datastrip_dir = os.path.join(PATH_L1B_DATA, \"DATASTRIP\")\n",
+ "\n",
+ "if not os.path.isdir(datastrip_dir):\n",
+ " raise RuntimeError(f\"DATASTRIP directory not found: {datastrip_dir}\")\n",
+ "\n",
+ "datastrip_entries = os.listdir(datastrip_dir)\n",
+ "if not datastrip_entries:\n",
+ " raise RuntimeError(f\"No DATASTRIP found in: {datastrip_dir}\")\n",
+ "\n",
+ "datastrip_name = datastrip_entries[0]\n",
+ "\n",
+ "match = re.match(r\"(S2[A-C])_OPER_\", datastrip_name)\n",
+ "if not match:\n",
+ " raise RuntimeError(\"Cannot extract mission (S2A/S2B/S2C) from DATASTRIP name.\")\n",
+ "\n",
+ "mission = match.group(1)\n",
+ "# =====================================================\n",
+ "# 2. Docker paths inside /workspace\n",
+ "# =====================================================\n",
+ "\n",
+ "docker_l1b = \"/data/L1B\"\n",
+ "docker_dem = \"/data/DEM\"\n",
+ "docker_gipp = f\"/data/GIPP/{mission}\"\n",
+ "# =====================================================\n",
+ "# 3. Geoid management\n",
+ "# =====================================================\n",
+ "\n",
+ "# Detect .gtx inside PATH_GEOID\n",
+ "geoid_files = [f for f in os.listdir(PATH_GEOID) if f.lower().endswith(\".gtx\")]\n",
+ "if len(geoid_files) == 0:\n",
+ " raise RuntimeError(\"No .gtx geoid file found in PATH_GEOID.\")\n",
+ "\n",
+ "docker_geoid = f\"/data/GEOID/{geoid_files[0]}\"\n",
+ "# =====================================================\n",
+ "# 4. Locate IERS bulletin on host\n",
+ "# =====================================================\n",
+ "\n",
+ "iers_host = None\n",
+ "\n",
+ "if os.path.isfile(PATH_IERS):\n",
+ " iers_host = PATH_IERS\n",
+ " PATH_IERS = os.path.dirname(PATH_IERS)\n",
+ "elif os.path.isdir(PATH_IERS):\n",
+ " for f in os.listdir(PATH_IERS):\n",
+ " if f.startswith(\"bulletin\"):\n",
+ " iers_host = os.path.join(PATH_IERS, f)\n",
+ " break\n",
+ "\n",
+ "if iers_host is None:\n",
+ " raise RuntimeError(\"IERS bulletin not found inside PATH_IERS directory.\")\n",
+ "\n",
+ "docker_iers = f\"/data/IERS/{os.path.basename(iers_host)}\"\n",
+ "print(\"IERS: \", docker_iers)\n",
+ "# =====================================================\n",
+ "# 5. Build config dictionary\n",
+ "# =====================================================\n",
+ "\n",
+ "config = {\n",
+ " \"l1b_product\": docker_l1b,\n",
+ " \"gipp_folder\": docker_gipp,\n",
+ " \"auto_gipp_selection\": True,\n",
+ " \"grids_overwriting\": True,\n",
+ " \"dem\": docker_dem,\n",
+ " \"geoid\": docker_geoid,\n",
+ " \"iers\": docker_iers,\n",
+ " \"operation\": \"inverse\",\n",
+ " \"deactivate_available_refining\": False,\n",
+ " \"steps\": {\n",
+ " \"10m_bands\": STEPS[\"10m_bands\"],\n",
+ " \"20m_bands\": STEPS[\"20m_bands\"],\n",
+ " \"60m_bands\": STEPS[\"60m_bands\"]\n",
+ " },\n",
+ " \"export_alt\": True\n",
+ "}\n",
+ "\n",
+ "config[\"inverse_location_additional_info\"] = {\n",
+ " \"ul_x\": double(LOCATION[\"ul_x\"]),\n",
+ " \"ul_y\": double(LOCATION[\"ul_y\"]),\n",
+ " \"lr_x\": double(LOCATION[\"lr_x\"]),\n",
+ " \"lr_y\": double(LOCATION[\"lr_y\"]),\n",
+ " \"referential\": f\"EPSG:{UTM_EPSG}\",\n",
+ " \"output_folder\": \"/output/INVERSE_GRID\"\n",
+ "}\n",
+ "# =====================================================\n",
+ "# Save config.json\n",
+ "# =====================================================\n",
+ "\n",
+ "config_path = os.path.join(USERCONF_DIR, \"config.json\")\n",
+ "\n",
+ "with open(config_path, \"w\") as f:\n",
+ " json.dump(config, f, indent=4)\n",
+ "\n",
+ "print(\"Configuration file generated: \")\n",
+ "print(config_path)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "23b1da25",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# === GENERATE PARAMS.JSON ===\n",
+ "\n",
+ "import os\n",
+ "import json\n",
+ "import re\n",
+ "\n",
+ "USERCONF_DIR = os.path.join(WORKDIR, \"UserConf\")\n",
+ "os.makedirs(USERCONF_DIR, exist_ok=True)\n",
+ "\n",
+ "print(\"UserConf directory: \", USERCONF_DIR)\n",
+ "\n",
+ "# =====================================================\n",
+ "# Locate GRANULE folders\n",
+ "# =====================================================\n",
+ "\n",
+ "GR_TARGET_DIR = os.path.join(PATH_L1B_DATA, \"GRANULE\")\n",
+ "\n",
+ "if not os.path.exists(GR_TARGET_DIR):\n",
+ " raise RuntimeError(\"GRANULE directory not found inside L1B SAFE.\")\n",
+ "\n",
+ "granule_folders = [\n",
+ " os.path.join(GR_TARGET_DIR, d)\n",
+ " for d in os.listdir(GR_TARGET_DIR)\n",
+ " if os.path.isdir(os.path.join(GR_TARGET_DIR, d))\n",
+ "]\n",
+ "\n",
+ "print(\"Found\", len(granule_folders), \"granule folders.\")\n",
+ "# =====================================================\n",
+ "# Extract detectors and bands from JP2\n",
+ "# =====================================================\n",
+ "\n",
+ "detectors = set()\n",
+ "bands = set()\n",
+ "\n",
+ "pattern = r\"_D(\\d+)_B(\\d{1,2}[A]?)\\.jp2$\"\n",
+ "\n",
+ "for granule in granule_folders:\n",
+ " img_data_dir = os.path.join(granule, \"IMG_DATA\")\n",
+ "\n",
+ " if not os.path.isdir(img_data_dir):\n",
+ " continue\n",
+ "\n",
+ " for fname in os.listdir(img_data_dir):\n",
+ " match = re.search(pattern, fname)\n",
+ " if match:\n",
+ " detectors.add(match.group(1))\n",
+ " bands.add(f\"B{match.group(2)}\")\n",
+ "\n",
+ "detectors = sorted(detectors)\n",
+ "bands = sorted(bands)\n",
+ "\n",
+ "print(\"Detected detectors: \", detectors)\n",
+ "print(\"Detected bands: \", bands)\n",
+ "# =====================================================\n",
+ "# Write params.json\n",
+ "# =====================================================\n",
+ "\n",
+ "# Validate keep_detectors: must be a non-empty list (user requirement)\n",
+ "if not isinstance(ORTHO_SETTINGS.get(\"keep_detectors\"), list) or len(ORTHO_SETTINGS[\"keep_detectors\"]) == 0:\n",
+ " raise RuntimeError(\"ORTHO_SETTINGS['keep_detectors'] must be a non-empty list of detector ids (e.g. ['01','02']).\")\n",
+ "\n",
+ "# Filter detectors to only include those selected by the user\n",
+ "keep_detectors_list = ORTHO_SETTINGS[\"keep_detectors\"]\n",
+ "selected_detectors = [d for d in keep_detectors_list if d in detectors]\n",
+ "\n",
+ "# Validate that all requested detectors exist\n",
+ "missing_detectors = [d for d in keep_detectors_list if d not in detectors]\n",
+ "if missing_detectors:\n",
+ " raise RuntimeError(f\"Some requested detectors are not available in the product: {missing_detectors}. Available detectors: {detectors}\")\n",
+ "\n",
+ "# Only include selected detectors and bands in params.json\n",
+ "params = {\n",
+ " \"detectors\": sorted(selected_detectors),\n",
+ " \"bands\": sorted(ORTHO_SETTINGS[\"keep_bands\"])\n",
+ "}\n",
+ "\n",
+ "params_path = os.path.join(USERCONF_DIR, \"params.json\")\n",
+ "\n",
+ "with open(params_path, \"w\") as f:\n",
+ " json.dump(params, f, indent=4)\n",
+ "\n",
+ "print(\"params.json written to: \", params_path)\n",
+ "print(f\" - Detectors: {params['detectors']}\")\n",
+ "print(f\" - Bands: {params['bands']}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ec6f67fe",
+ "metadata": {},
+ "source": [
+ "# Sen2VM run"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "572bd72a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# === RUN SEN2VM (Docker: BUILD + RUN + CLEAN) ===\n",
+ "\n",
+ "import os\n",
+ "import subprocess\n",
+ "\n",
+ "# Notebook location (NOT relative to CWD)\n",
+ "notebook_dir = os.getcwd()\n",
+ "\n",
+ "dockerfile_dir = os.path.abspath(os.path.join(\n",
+ " notebook_dir,\n",
+ " \"..\", \"..\"\n",
+ "))\n",
+ "\n",
+ "config_inside = \"/workspace/UserConf/config.json\"\n",
+ "params_inside = \"/workspace/UserConf/params.json\"\n",
+ "\n",
+ "os.makedirs(\n",
+ " os.path.join(OUTPUT_FOLDER, \"INVERSE_GRID\"),\n",
+ " exist_ok=True\n",
+ ")\n",
+ "# =====================================================\n",
+ "# 1. BUILD DOCKER IMAGE\n",
+ "# =====================================================\n",
+ "\n",
+ "print(f\"Building Docker image 'sen2vm' from: {dockerfile_dir}\")\n",
+ "cmd_build = [\n",
+ " \"docker\", \"build\",\n",
+ " \"-t\", \"sen2vm\",\n",
+ " dockerfile_dir\n",
+ "]\n",
+ "\n",
+ "print(\"Command:\", \" \".join(cmd_build), \"\\n\")\n",
+ "subprocess.run(cmd_build, check=True)\n",
+ "print(\"Docker image built successfully.\\n\")\n",
+ "# =====================================================\n",
+ "# 2. RUN SEN2VM CONTAINER\n",
+ "# =====================================================\n",
+ "\n",
+ "cmd_run = [\n",
+ " \"docker\", \"run\",\n",
+ " \"--rm\",\n",
+ " \"-v\", f\"{PATH_L1B_DATA}:/data/L1B\",\n",
+ " \"-v\", f\"{PATH_DEM}:/data/DEM\",\n",
+ " \"-v\", f\"{PATH_GIPP}:/data/GIPP\",\n",
+ " \"-v\", f\"{PATH_GEOID}:/data/GEOID\",\n",
+ " \"-v\", f\"{PATH_IERS}:/data/IERS\",\n",
+ " \"-v\", f\"{WORKDIR}:/workspace\",\n",
+ " \"-v\", f\"{OUTPUT_FOLDER}:/output\",\n",
+ " \"sen2vm\",\n",
+ " \"-c\", config_inside,\n",
+ " \"-p\", params_inside\n",
+ "]\n",
+ "\n",
+ "print(\"Running Docker container...\\n\")\n",
+ "print(\"Command:\", \" \".join(cmd_run), \"\\n\")\n",
+ "subprocess.run(cmd_run, check=True)\n",
+ "print(\"\\nDocker execution complete.\\n\")\n",
+ "# =====================================================\n",
+ "# 3. REMOVE DOCKER IMAGE\n",
+ "# =====================================================\n",
+ "if REMOVE_DOCKER_IMAGE:\n",
+ " print(\"Removing Docker image 'sen2vm'...\")\n",
+ " subprocess.run([\"docker\", \"rmi\", \"-f\", \"sen2vm\"], check=True)\n",
+ " print(\"Docker images removed.\\n\")\n",
+ "else:\n",
+ " print(\"Docker images kept.\\n\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "e54044d2",
+ "metadata": {},
+ "source": [
+ "# Generate Orthorectification images"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "61ebba54-43f0-4d39-9589-40c87609f70e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# =====================================================\n",
+ "# INVERSE\n",
+ "# =====================================================\n",
+ "import os\n",
+ "import subprocess\n",
+ "import glob\n",
+ "import json\n",
+ "import re\n",
+ "\n",
+ "# =====================================================\n",
+ "# SAFETY CHECKS\n",
+ "# =====================================================\n",
+ "assert os.path.exists(PATH_L1B_DATA), \"L1B path missing\"\n",
+ "os.makedirs(OUTPUT_FOLDER, exist_ok=True)\n",
+ "assert os.path.exists(OUTPUT_FOLDER), \"Output folder missing\"\n",
+ "\n",
+ "# =====================================================\n",
+ "# OUTPUT FOLDERS\n",
+ "# =====================================================\n",
+ "RAW_DIR = os.path.join(OUTPUT_FOLDER, \"raw\")\n",
+ "OTB_DIR = os.path.join(OUTPUT_FOLDER, \"otb_no_georef\")\n",
+ "GEOREF_DIR = os.path.join(OUTPUT_FOLDER, \"output_georef\")\n",
+ "MOSAIC_DIR = os.path.join(OUTPUT_FOLDER, \"mosaic\")\n",
+ "os.makedirs(RAW_DIR, exist_ok=True)\n",
+ "os.makedirs(OTB_DIR, exist_ok=True)\n",
+ "os.makedirs(GEOREF_DIR, exist_ok=True)\n",
+ "os.makedirs(MOSAIC_DIR, exist_ok=True)\n",
+ "\n",
+ "# =====================================================\n",
+ "# Locate product name\n",
+ "# =====================================================\n",
+ "product = os.path.basename(os.path.normpath(PATH_L1B_DATA))\n",
+ "print(\"Product: \", product)\n",
+ "\n",
+ "# =====================================================\n",
+ "# Locate XML\n",
+ "# =====================================================\n",
+ "xml_list = glob.glob(\n",
+ " os.path.join(\n",
+ " PATH_L1B_DATA,\n",
+ " \"S2*_MTD_*.xml\"\n",
+ " )\n",
+ ")\n",
+ "\n",
+ "if len(xml_list) == 0:\n",
+ " raise RuntimeError(\"No DATASTRIP MTD XML found\")\n",
+ "\n",
+ "xml_path = xml_list[0]\n",
+ "xml_name = os.path.basename(xml_path)\n",
+ "print(\"Using XML: \", xml_path)\n",
+ "\n",
+ "# =====================================================\n",
+ "# Read params.json to get selected bands and detectors\n",
+ "# =====================================================\n",
+ "params_path = os.path.join(WORKDIR, \"UserConf\", \"params.json\")\n",
+ "if not os.path.exists(params_path):\n",
+ " raise RuntimeError(f\"params.json not found at {params_path}\")\n",
+ "\n",
+ "with open(params_path, \"r\") as f:\n",
+ " params = json.load(f)\n",
+ "\n",
+ "selected_bands = params.get(\"bands\", [])\n",
+ "selected_detectors = params.get(\"detectors\", [])\n",
+ "\n",
+ "print(f\"Bands from params.json: {selected_bands}\")\n",
+ "print(f\"Detectors from params.json: {selected_detectors}\")\n",
+ "\n",
+ "# =====================================================\n",
+ "# Build GDAL Docker\n",
+ "# =====================================================\n",
+ "notebook_dir = os.path.dirname(os.getcwd())\n",
+ "dockerfile_dir = os.path.abspath(os.path.join(\n",
+ " notebook_dir,\n",
+ " \"src\",\n",
+ " \"gdal-latest\"\n",
+ "))\n",
+ "print(\"\\n=== BUILDING GDAL CONTAINER ===\\n\")\n",
+ "\n",
+ "cmd_build = [\n",
+ " \"docker\", \"build\",\n",
+ " \"--platform=linux/amd64\",\n",
+ " \"-t\", \"gdal-latest\",\n",
+ " dockerfile_dir\n",
+ "]\n",
+ "\n",
+ "print(\"Command:\")\n",
+ "print(\" \".join(cmd_build))\n",
+ "subprocess.run(cmd_build, check=True)\n",
+ "print(\"\\nGDAL image built successfully.\\n\")\n",
+ "\n",
+ "# =====================================================\n",
+ "# Locate inverse grids\n",
+ "# =====================================================\n",
+ "INV_FOLDER = os.path.join(OUTPUT_FOLDER, \"INVERSE_GRID\")\n",
+ "if not os.path.exists(INV_FOLDER):\n",
+ " raise RuntimeError(f\"Inverse folder not found: {INV_FOLDER}\")\n",
+ " \n",
+ "inv_grid_list = glob.glob(\n",
+ " os.path.join(INV_FOLDER, \"*.tif\")\n",
+ ")\n",
+ "if len(inv_grid_list) == 0:\n",
+ " raise RuntimeError(\"No inverse grids found\")\n",
+ "print(f\"\\nFound {len(inv_grid_list)} inverse grids:\\n\")\n",
+ "\n",
+ "for g in inv_grid_list:\n",
+ " print(\" \", os.path.basename(g))\n",
+ "\n",
+ "# =====================================================\n",
+ "# Process all inverse grids\n",
+ "# =====================================================\n",
+ "for inv_grid_path in inv_grid_list:\n",
+ " test_grid = os.path.basename(inv_grid_path)\n",
+ " print(\"\\n=================================================\")\n",
+ " print(\"Processing: \", test_grid)\n",
+ " print(\"=================================================\")\n",
+ "\n",
+ " # =================================================\n",
+ " # Parse detector / band\n",
+ " # =================================================\n",
+ " match = re.search(r\"_D(\\d+)_B(\\d{1,2}[A]?)\", test_grid)\n",
+ " if not match:\n",
+ " print(\"Could not parse detector/band\")\n",
+ " continue\n",
+ "\n",
+ " detector = match.group(1)\n",
+ " band = f\"B{match.group(2)}\"\n",
+ "\n",
+ " print(\"Detector: \", detector)\n",
+ " print(\"Band: \", band)\n",
+ "\n",
+ " # =================================================\n",
+ " # Build subdataset dynamically\n",
+ " # =================================================\n",
+ " subdataset = test_grid.replace(\"INV_L1B\", \"GEO_L1B\")\n",
+ " subdataset = subdataset.replace(\".tif\", \"\")\n",
+ " print(\"Subdataset: \", subdataset)\n",
+ " \n",
+ " # =================================================\n",
+ " # Determine resolution\n",
+ " # =================================================\n",
+ " if band in [\"B02\", \"B03\", \"B04\", \"B08\"]:\n",
+ " resolution = 10\n",
+ " elif band in [\"B05\", \"B06\", \"B07\", \"B8A\", \"B11\", \"B12\"]:\n",
+ " resolution = 20\n",
+ " elif band in [\"B01\", \"B09\", \"B10\"]:\n",
+ " resolution = 60\n",
+ " else:\n",
+ " resolution = 10\n",
+ " \n",
+ " print(\"Resolution: \", resolution)\n",
+ " \n",
+ " # =================================================\n",
+ " # Compute output size\n",
+ " # =================================================\n",
+ " width = int(\n",
+ " (LOCATION[\"lr_x\"] - LOCATION[\"ul_x\"]) / resolution\n",
+ " )\n",
+ " height = int(\n",
+ " (LOCATION[\"ul_y\"] - LOCATION[\"lr_y\"]) / resolution\n",
+ " )\n",
+ " print(\"Width : \", width)\n",
+ " print(\"Height: \", height)\n",
+ "\n",
+ " # =================================================\n",
+ " # File paths\n",
+ " # =================================================\n",
+ " raw_name = f\"raw_D{detector}_{band}.tif\"\n",
+ " ortho_name = f\"ortho_D{detector}_{band}.tif\"\n",
+ " georef_name = f\"ortho_D{detector}_{band}_georef.tif\"\n",
+ " \n",
+ " # =================================================\n",
+ " # Extract raw detector image\n",
+ " # =================================================\n",
+ " cmd_extract = [\n",
+ " \"docker\", \"run\",\n",
+ " \"--rm\",\n",
+ " \"-v\", f\"{PATH_L1B_DATA}:/data/L1B\",\n",
+ " \"-v\", f\"{OUTPUT_FOLDER}:/output\",\n",
+ " \"gdal-latest\",\n",
+ " \"-c\",\n",
+ " f'''\n",
+ " gdal_translate \\\n",
+ " \"SENTINEL2_L1B_WITH_GEOLOC:/data/L1B/{xml_name}:{subdataset}\" \\\n",
+ " \"/output/raw/{raw_name}\"\n",
+ " '''\n",
+ " ]\n",
+ " print(\"\\n=== EXTRACTION ===\\n\")\n",
+ " print(\" \".join(cmd_extract))\n",
+ "\n",
+ " subprocess.run(cmd_extract, check=True)\n",
+ " \n",
+ " # Half-pixel correction (OTB bug fix)\n",
+ " corrected_ulx = LOCATION[\"ul_x\"] + resolution / 2\n",
+ " corrected_uly = LOCATION[\"ul_y\"] - resolution / 2\n",
+ "\n",
+ " # =================================================\n",
+ " # OTB resampling\n",
+ " # =================================================\n",
+ " cmd_otb = [\n",
+ " \"docker\", \"run\",\n",
+ " \"--rm\",\n",
+ " \"-v\", f\"{OUTPUT_FOLDER}:/output\",\n",
+ " \"orfeotoolbox/otb:9.0.0\",\n",
+ " \"otbcli_GridBasedImageResampling\",\n",
+ " \"-io.in\",\n",
+ " f\"/output/raw/{raw_name}\",\n",
+ " \"-io.out\",\n",
+ " f\"/output/otb_no_georef/{ortho_name}\",\n",
+ " \"-grid.in\",\n",
+ " f\"/output/INVERSE_GRID/{test_grid}\",\n",
+ " \"-grid.type\",\n",
+ " \"loc\",\n",
+ " \"-out.ulx\",\n",
+ " str(corrected_ulx),\n",
+ " \"-out.uly\",\n",
+ " str(corrected_uly),\n",
+ " \"-out.spacingx\",\n",
+ " str(resolution),\n",
+ " \"-out.spacingy\",\n",
+ " str(-resolution),\n",
+ " \"-out.sizex\",\n",
+ " str(width),\n",
+ " \"-out.sizey\",\n",
+ " str(height)\n",
+ " ]\n",
+ "\n",
+ " print(\"\\n=== OTB RESAMPLING ===\\n\")\n",
+ " print(\" \".join(cmd_otb))\n",
+ "\n",
+ " try:\n",
+ " subprocess.run(cmd_otb, check=True)\n",
+ " print(\"OTB success\")\n",
+ " \n",
+ " except subprocess.CalledProcessError:\n",
+ " print(\"OTB failed (outside grid) → skipping\")\n",
+ " continue\n",
+ "\n",
+ " # ==================a===============================\n",
+ " # Add georeferencing to correct otb missing one\n",
+ " # =================================================\n",
+ " cmd_georef = [\n",
+ " \"docker\", \"run\",\n",
+ " \"--rm\",\n",
+ " \"-v\", f\"{OUTPUT_FOLDER}:/output\",\n",
+ " \"gdal-latest\",\n",
+ " \"-c\",\n",
+ " f'''\n",
+ " gdal_translate \\\n",
+ " -a_srs EPSG:{UTM_EPSG} \\\n",
+ " /output/otb_no_georef/{ortho_name} \\\n",
+ " /output/output_georef/{georef_name}\n",
+ " '''\n",
+ " ]\n",
+ "\n",
+ " print(\"\\n=== ADD GEOREF ===\\n\")\n",
+ " print(\" \".join(cmd_georef))\n",
+ " subprocess.run(cmd_georef, check=True)\n",
+ " print(\"\\nFinished: \", georef_name)\n",
+ "\n",
+ "# =====================================================\n",
+ "# Mosaic\n",
+ "# =====================================================\n",
+ "print(\"\\n=== MOSAIC CREATION ===\\n\")\n",
+ "bands = set()\n",
+ "for f in glob.glob(os.path.join(GEOREF_DIR, \"*.tif\")):\n",
+ " match = re.search(r\"_B(\\d{2}[A]?)\", f)\n",
+ " if match:\n",
+ " bands.add(f\"B{match.group(1)}\")\n",
+ "print(\"Bands detected: \", bands)\n",
+ "\n",
+ "for band in bands:\n",
+ "\n",
+ " print(\"\\n----------------------------------------\")\n",
+ " print(\"Creating mosaic for band: \", band)\n",
+ " print(\"----------------------------------------\")\n",
+ " # fichiers correspondant à la bande\n",
+ " input_files = glob.glob(\n",
+ " os.path.join(GEOREF_DIR, f\"*_{band}_georef.tif\")\n",
+ " )\n",
+ " if len(input_files) == 0:\n",
+ " print(\"No ortho images found for band\", band)\n",
+ " continue\n",
+ " input_files = sorted(input_files)\n",
+ " \n",
+ " output_path = os.path.join(\n",
+ " MOSAIC_DIR,\n",
+ " f\"ORTHO_mosaic_{band}.tif\"\n",
+ " )\n",
+ " print(\"Found\", len(input_files), \"files\")\n",
+ " for f in input_files:\n",
+ " print(\" \", os.path.basename(f))\n",
+ " \n",
+ " gdal_cmd = \"gdal_merge.py \" \\\n",
+ " f\"-o /output/mosaic/ORTHO_mosaic_{band}.tif \" \\\n",
+ " \"-of GTiff \" \\\n",
+ " \"-co COMPRESS=LZW \" \\\n",
+ " \"-co TILED=YES \" \\\n",
+ " \"-ot UInt16 \" \\\n",
+ " \"-n 0 -a_nodata 0 \"\n",
+ "\n",
+ " for f in input_files:\n",
+ " fname = os.path.basename(f)\n",
+ " gdal_cmd += f\"/output/output_georef/{fname} \"\n",
+ "\n",
+ " cmd_mosaic = [\n",
+ " \"docker\", \"run\",\n",
+ " \"--rm\",\n",
+ " \"-v\", f\"{OUTPUT_FOLDER}:/output\",\n",
+ " \"gdal-latest\",\n",
+ " \"-c\",\n",
+ " gdal_cmd\n",
+ " ]\n",
+ "\n",
+ " print(\"\\nRunning mosaic:\\n\", gdal_cmd)\n",
+ " subprocess.run(cmd_mosaic, check=True)\n",
+ " print(\"Mosaic written →\", output_path)\n",
+ "\n",
+ "# =====================================================\n",
+ "# Cleanup docker image\n",
+ "# =====================================================\n",
+ "if REMOVE_DOCKER_IMAGE:\n",
+ " print(\"Removing gdal-latest image...\\n\")\n",
+ " subprocess.run([\"docker\", \"rmi\", \"-f\", \"gdal-latest\"], check=True)\n",
+ " print(\"GDAL image removed.\\n\")\n",
+ "else:\n",
+ " print(\"Docker images kept (faster next run, ~4 GB disk usage).\\n\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b281ab9e-0afd-4ef9-8ca0-0f4d368c7cb3",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.13.11"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/sen2vm-notebook/src/requirements.txt b/sen2vm-notebook/src/requirements.txt
index b8115cf4..d81a89d0 100644
--- a/sen2vm-notebook/src/requirements.txt
+++ b/sen2vm-notebook/src/requirements.txt
@@ -34,9 +34,11 @@ python-dateutil==2.9.0.post0
pyzmq==27.1.0
rasterio==1.4.3
requests==2.32.5
+scipy
six==1.17.0
stack-data==0.6.3
tornado==6.5.2
traitlets==5.14.3
urllib3==2.6.1
+utm==0.8.1
wcwidth==0.2.14
diff --git a/src/main/java/esa/sen2vm/input/gipp/GIPPFileManager.java b/src/main/java/esa/sen2vm/input/gipp/GIPPFileManager.java
index 777a9656..b814dad3 100644
--- a/src/main/java/esa/sen2vm/input/gipp/GIPPFileManager.java
+++ b/src/main/java/esa/sen2vm/input/gipp/GIPPFileManager.java
@@ -97,8 +97,9 @@ public static List findGippFiles(Path root, String dirNameRegex, List searchGIPFilesFromRegex(Path root, String dirNameRegex, String fileNameRegex, List validExtensions) throws IOException {
- final Pattern dirPattern = Pattern.compile(dirNameRegex);
+ public static List searchGIPFilesFromRegex(Path root, String dirNameRegex, String fileNameRegex, List validExtensions) throws IOException
+ {
+ final Pattern dirPattern = Pattern.compile(dirNameRegex);
final Pattern filePattern = Pattern.compile(fileNameRegex);
final List results = new ArrayList<>();
// Stack indicating whether we are currently in a qualified subtree
@@ -225,7 +226,7 @@ public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs) {
}
catch(IOException e)
{
- LOGGER.warning("The targz extraction of GIPP has failed:"+file.toString());
+ LOGGER.warning("The targz extraction of GIPP has failed: "+file.toString());
e.printStackTrace();
}
}
@@ -266,7 +267,7 @@ public static File findGippFile(Path root, String dirNameRegex, List gip
final List results = findGippFiles(root, dirNameRegex, gippList, fileNameRegex, validExtensions);
if(results.size()==0)
{
- throw new Sen2VMException("The directory must be contains keyword:"+fileNameRegex);
+ throw new Sen2VMException("The directory must be contains keyword: "+fileNameRegex);
}
else if(results.size()>1)
{
diff --git a/src/test/java/esa/sen2vm/Sen2VMCheckPointTest.java b/src/test/java/esa/sen2vm/Sen2VMCheckPointTest.java
index 94cf013b..27e9387b 100644
--- a/src/test/java/esa/sen2vm/Sen2VMCheckPointTest.java
+++ b/src/test/java/esa/sen2vm/Sen2VMCheckPointTest.java
@@ -155,7 +155,7 @@ public void geoTimeFirstLine()
LineSensor lineSensor = ruggedManager.getLineSensor("B01/D01");
String date = lineSensor.getDate(0.5).toString(TimeScalesFactory.getGPS());
- LOGGER.info("date line 0.5:" + date);
+ LOGGER.info("date line 0.5: " + date);
assertEquals(date, "2020-08-16T12:02:45.812731");
} catch ( SXGeoException e ) {
diff --git a/src/test/java/esa/sen2vm/Sen2VMDirectTest.java b/src/test/java/esa/sen2vm/Sen2VMDirectTest.java
index 0ae1e7b7..39fd196f 100644
--- a/src/test/java/esa/sen2vm/Sen2VMDirectTest.java
+++ b/src/test/java/esa/sen2vm/Sen2VMDirectTest.java
@@ -160,7 +160,7 @@ public void testAutoSelectTarGipp()
String config = Config.configAutoGippSelection(configTmpDirectTDS1, GIPP_2, true, outputDir);
String param = Config.changeParams(paramTmp, detectors, bands, outputDir);
String[] args = {"-c", config, "-p", param};
- LOGGER.info("config:"+config);
+ LOGGER.info("config: "+config);
Sen2VM.main(args);
Utils.verifyDirectLoc(config, refDir + "/" + nameTest);
} catch (Sen2VMException e) {
@@ -202,7 +202,7 @@ public void testAutoSelectWithMissingGipp()
String config = Config.configAutoGippSelection(configTmpDirectTDS1, GIPP_2, true, outputDir);
String param = Config.changeParams(paramTmp, detectors, bands, outputDir);
String[] args = {"-c", config, "-p", param};
- LOGGER.info("config:"+config);
+ LOGGER.info("config: "+config);
Sen2VM.main(args);
Utils.verifyDirectLoc(config, refDir + "/" + nameTest);
LOGGER.warning("Expecting an error.");
diff --git a/src/test/java/esa/sen2vm/Utils.java b/src/test/java/esa/sen2vm/Utils.java
index bac03d38..a326581b 100755
--- a/src/test/java/esa/sen2vm/Utils.java
+++ b/src/test/java/esa/sen2vm/Utils.java
@@ -84,9 +84,9 @@ public static void verifyStepDirectLoc(String configFilepath, int step) throws S
double res = BandInfo.getBandInfoFromIndex(b).getPixelHeight();
Dataset ds = gdal.Open(grid.getPath());
double[] transform = ds.GetGeoTransform();
- System.out.println("transform:" + String.valueOf(transform[1]));
- System.out.println("res:" + String.valueOf(res));
- System.out.println("step:" + String.valueOf(step));
+ System.out.println("transform: " + String.valueOf(transform[1]));
+ System.out.println("res: " + String.valueOf(res));
+ System.out.println("step: " + String.valueOf(step));
assertEquals(transform[1] * (res / 10), step);
assertEquals(transform[5] * (res / 10), step);