diff --git a/CHANGELOG.md b/CHANGELOG.md index f77fbb80..74285b6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Sen2VM Release history -## Merged in main (1.3.0.rc2) +## Merged in main ### Main feature @@ -10,6 +10,28 @@ * +* Feature: + + * + +* Doc: + + * + +## 1.3.0 (2026-07-17) + +### Main features + +* Allow to have mosaic DEM and not only cut in square degrees +* Notebooks improvments +* GIPP handling more robust + +### Updates + +* Fix: + + * Update GIPP handling (#70) + * Feature: * Handle mosaic DEM (multiple square degrees): #62 @@ -23,6 +45,9 @@ * Creation of a Notebook for Inverse location (#54) * Allow using OTB instead of gdal in direct location Notebook (through a new notebook) + * Change input description related to DEM and GIPP + * Addition of Unit test for GIPP handling (#51, #70) + ## 1.2.0 (2026-05-04) diff --git a/Dockerfile b/Dockerfile index 63c98c84..6261d1e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM ghcr.io/sen2vm/sen2vm-build-env:latest AS launcher -ENV SEN2VM_VERSION=1.3.0.rc2 +ENV SEN2VM_VERSION=1.3.0 WORKDIR /Sen2vm diff --git a/documentation/Input/DEM_CDSE_Download.md b/documentation/Input/DEM_CDSE_Download.md index 14d6b89d..2f29fd36 100644 --- a/documentation/Input/DEM_CDSE_Download.md +++ b/documentation/Input/DEM_CDSE_Download.md @@ -23,7 +23,7 @@ Digital Elevation Models (DEM) are essential for geolocation and orthorectification in Sen2VM. Users can use different types of DEM (cf [Inputs Description](../Input/input_description.md)). -Sen2VM requires DEM data **organized per square degree** (see §[DEM format requirements](../Input/input_description.md#131-dem)) but can now also handle mosaic of square degrees. The recommended way to obtain Copernicus DEM in the correct format is to use the **[CDSE-Copernicus-DEM-downloader](https://github.com/senbox-org/CDSE-Copernicus-DEM-downloader)** tool, which downloads individual 1°×1° geocells from the Copernicus Data Space Ecosystem. +Sen2VM requires DEM data **organized per square degree** (see §[DEM format requirements](../Input/input_description.md#131-dem)) but can now also handle **mosaic of square degrees**. The recommended way to obtain Copernicus DEM in the correct format is to use the **[CDSE-Copernicus-DEM-downloader](https://github.com/senbox-org/CDSE-Copernicus-DEM-downloader)** tool, which downloads individual 1°×1° geocells from the Copernicus Data Space Ecosystem. --- diff --git a/documentation/Input/input_description.md b/documentation/Input/input_description.md index ca49dc00..e8665d6f 100644 --- a/documentation/Input/input_description.md +++ b/documentation/Input/input_description.md @@ -146,9 +146,6 @@ The GIPP folder does not require a specific structure; the system searches throu If only .tar or .tar.gz archives of the GIPPs are available, the archives are extracted. -> [!CAUTION] -> Without automatic GIPP selection, the GIPP folder should contain only the required GIPP files. No archive extraction is performed in this mode. - The GIPP required are the following ones: * **GIP_VIEDIR**: contains Viewing Direction required by Rugged to create viewing model based on TAN_PSI_X/Y_LIST tags. There is one GIP_VIEDIR file **per band** and each file contains information per **detector** (in the following tags: _[DATA/VIEWING_DIRECTIONS_LIST/VIEWING_DIRECTIONS/TAN_PSI_X_LIST]_ and _[DATA/VIEWING_DIRECTIONS_LIST/VIEWING_DIRECTIONS/TAN_PSI_Y_LIST]_) * **GIP_SPAMOD**: contains transformations to apply to viewing direction from tags, available in the _[DATA]_ field: diff --git a/pom.xml b/pom.xml index e9152b8a..6ca7b389 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ esa.sen2vm sen2vm-core - 1.3.0.rc2 + 1.3.0 sen2vm-core https://github.com/sen2vm/sen2vm-core diff --git a/sen2vm-notebook/src/notebook-direct-grid-otb.ipynb b/sen2vm-notebook/src/notebook-direct-grid-otb.ipynb index fbafa227..c0c46dbb 100644 --- a/sen2vm-notebook/src/notebook-direct-grid-otb.ipynb +++ b/sen2vm-notebook/src/notebook-direct-grid-otb.ipynb @@ -30,7 +30,6 @@ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n", "# /!\\/!\\/!\\ If you do not have your own GIPP folder, you may use the inputs-download-notebook to download it, then you may indicate the path were you \n", "# /!\\/!\\/!\\ downloaded it here in PATH_GIPP\n", - "# /!\\/!\\/!\\ If you have your own GIPP folder, please note that this current notebook will search for a subfolder with mission S2[A/B/C] inside the GIPP folder\n", "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n", "\n", "# Path to DEM directory \n", @@ -275,7 +274,12 @@ "\n", "docker_l1b = \"/data/L1B\"\n", "docker_dem = \"/data/DEM\"\n", - "docker_gipp = f\"/data/GIPP/{mission}\"\n", + "\n", + "if os.path.isdir(os.path.join(PATH_GIPP, mission)):\n", + " docker_gipp = f\"/data/GIPP/{mission}\" \n", + "else:\n", + " docker_gipp = f\"/data/GIPP/\"\n", + "\n", "# =====================================================\n", "# 3. Geoid management\n", "# =====================================================\n", @@ -294,7 +298,6 @@ "\n", "if os.path.isfile(PATH_IERS):\n", " iers_host = PATH_IERS\n", - " PATH_IERS = os.path.dirname(PATH_IERS)\n", "elif os.path.isdir(PATH_IERS):\n", " for f in os.listdir(PATH_IERS):\n", " if f.startswith(\"bulletin\"):\n", @@ -484,7 +487,7 @@ " \"-v\", f\"{PATH_DEM}:/data/DEM\",\n", " \"-v\", f\"{PATH_GIPP}:/data/GIPP\",\n", " \"-v\", f\"{PATH_GEOID}:/data/GEOID\",\n", - " \"-v\", f\"{PATH_IERS}:/data/IERS\",\n", + " \"-v\", f\"{iers_host}:{docker_iers}\",\n", " \"-v\", f\"{WORKDIR}:/workspace\",\n", " \"sen2vm\",\n", " \"-c\", config_inside,\n", diff --git a/sen2vm-notebook/src/notebook-direct-grid.ipynb b/sen2vm-notebook/src/notebook-direct-grid.ipynb index 14431f3f..5f1eb496 100644 --- a/sen2vm-notebook/src/notebook-direct-grid.ipynb +++ b/sen2vm-notebook/src/notebook-direct-grid.ipynb @@ -30,7 +30,6 @@ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n", "# /!\\/!\\/!\\ If you do not have your own GIPP folder, you may use the inputs-download-notebook to download it, then you may indicate the path were you \n", "# /!\\/!\\/!\\ downloaded it here in PATH_GIPP\n", - "# /!\\/!\\/!\\ If you have your own GIPP folder, please note that this current notebook will search for a subfolder with mission S2[A/B/C] inside the GIPP folder\n", "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n", "\n", "# Path to DEM directory \n", @@ -275,7 +274,12 @@ "\n", "docker_l1b = \"/data/L1B\"\n", "docker_dem = \"/data/DEM\"\n", - "docker_gipp = f\"/data/GIPP/{mission}\"\n", + "\n", + "if os.path.isdir(os.path.join(PATH_GIPP, mission)):\n", + " docker_gipp = f\"/data/GIPP/{mission}\" \n", + "else:\n", + " docker_gipp = f\"/data/GIPP/\"\n", + "\n", "# =====================================================\n", "# 3. Geoid management\n", "# =====================================================\n", @@ -294,7 +298,6 @@ "\n", "if os.path.isfile(PATH_IERS):\n", " iers_host = PATH_IERS\n", - " PATH_IERS = os.path.dirname(PATH_IERS)\n", "elif os.path.isdir(PATH_IERS):\n", " for f in os.listdir(PATH_IERS):\n", " if f.startswith(\"bulletin\"):\n", @@ -487,7 +490,7 @@ " \"-v\", f\"{PATH_DEM}:/data/DEM\",\n", " \"-v\", f\"{PATH_GIPP}:/data/GIPP\",\n", " \"-v\", f\"{PATH_GEOID}:/data/GEOID\",\n", - " \"-v\", f\"{PATH_IERS}:/data/IERS\",\n", + " \"-v\", f\"{iers_host}:{docker_iers}\",\n", " \"-v\", f\"{WORKDIR}:/workspace\",\n", " \"sen2vm\",\n", " \"-c\", config_inside,\n", diff --git a/sen2vm-notebook/src/notebook-inverse-grid.ipynb b/sen2vm-notebook/src/notebook-inverse-grid.ipynb index de748503..63ae9c75 100644 --- a/sen2vm-notebook/src/notebook-inverse-grid.ipynb +++ b/sen2vm-notebook/src/notebook-inverse-grid.ipynb @@ -30,7 +30,6 @@ "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n", "# /!\\/!\\/!\\ If you do not have your own GIPP folder, you may use the inputs-download-notebook to download it, then you may indicate the path were you \n", "# /!\\/!\\/!\\ downloaded it here in PATH_GIPP\n", - "# /!\\/!\\/!\\ If you have your own GIPP folder, please note that this current notebook will search for a subfolder with mission S2[A/B/C] inside the GIPP folder\n", "# /!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\/!\\\n", "\n", "# Path to DEM directory \n", @@ -274,7 +273,12 @@ "\n", "docker_l1b = \"/data/L1B\"\n", "docker_dem = \"/data/DEM\"\n", - "docker_gipp = f\"/data/GIPP/{mission}\"\n", + "\n", + "if os.path.isdir(os.path.join(PATH_GIPP, mission)):\n", + " docker_gipp = f\"/data/GIPP/{mission}\" \n", + "else:\n", + " docker_gipp = f\"/data/GIPP/\"\n", + "\n", "# =====================================================\n", "# 3. Geoid management\n", "# =====================================================\n", @@ -293,7 +297,6 @@ "\n", "if os.path.isfile(PATH_IERS):\n", " iers_host = PATH_IERS\n", - " PATH_IERS = os.path.dirname(PATH_IERS)\n", "elif os.path.isdir(PATH_IERS):\n", " for f in os.listdir(PATH_IERS):\n", " if f.startswith(\"bulletin\"):\n", @@ -501,7 +504,7 @@ " \"-v\", f\"{PATH_DEM}:/data/DEM\",\n", " \"-v\", f\"{PATH_GIPP}:/data/GIPP\",\n", " \"-v\", f\"{PATH_GEOID}:/data/GEOID\",\n", - " \"-v\", f\"{PATH_IERS}:/data/IERS\",\n", + " \"-v\", f\"{iers_host}:{docker_iers}\",\n", " \"-v\", f\"{WORKDIR}:/workspace\",\n", " \"-v\", f\"{OUTPUT_FOLDER}:/output\",\n", " \"sen2vm\",\n", diff --git a/src/main/java/esa/sen2vm/input/gipp/GIPPFileManager.java b/src/main/java/esa/sen2vm/input/gipp/GIPPFileManager.java index b814dad3..d891c38e 100644 --- a/src/main/java/esa/sen2vm/input/gipp/GIPPFileManager.java +++ b/src/main/java/esa/sen2vm/input/gipp/GIPPFileManager.java @@ -102,6 +102,7 @@ public static List searchGIPFilesFromRegex(Path root, String dirNameRegex, final Pattern dirPattern = Pattern.compile(dirNameRegex); final Pattern filePattern = Pattern.compile(fileNameRegex); final List results = new ArrayList<>(); + final List tarExtension = Arrays.asList("TGZ", "tar.gz","tgz"); // Stack indicating whether we are currently in a qualified subtree final Deque qualifiedStack = new ArrayDeque<>(); Files.walkFileTree(root, new SimpleFileVisitor() { @@ -129,7 +130,30 @@ public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs) { if (inQualifiedSubtree || filePattern.matcher(fileName).matches()) { File file = filePath.toFile(); String extension = getFileExtension(file); - if(validExtensions.stream().anyMatch(item -> item.contains(extension))) + + if(tarExtension.stream().anyMatch(item -> item.contains(extension))) + { + try + { + List listPath = UntarGIPP.untarGz(file.toPath(), Paths.get(file.getParent())); + for(Path untarPath:listPath) + { + File untarFile = untarPath.toFile(); + String untarFileExtension = getFileExtension(untarFile); + if(validExtensions.stream().anyMatch(item -> item.contains(untarFileExtension))) + { + results.add(untarFile); + } + } + LOGGER.info("Untar GIPP: "+file.toString()); + } + catch(IOException e) + { + LOGGER.warning("The targz extraction of GIPP has failed: "+file.toString()); + e.printStackTrace(); + } + } + else if(validExtensions.stream().anyMatch(item -> item.contains(extension))) { results.add(file); } @@ -265,9 +289,16 @@ public static List typedGIPPList(List gippList, String gippType) */ public static File findGippFile(Path root, String dirNameRegex, List gippList, String fileNameRegex, List validExtensions) throws IOException, Sen2VMException { final List results = findGippFiles(root, dirNameRegex, gippList, fileNameRegex, validExtensions); - if(results.size()==0) + if(results.isEmpty()) { - throw new Sen2VMException("The directory must be contains keyword: "+fileNameRegex); + if (gippList.isEmpty()) + { + throw new Sen2VMException("GIPP directory(ies) must contains at list one GIPP of type: " + fileNameRegex); + } + else + { + throw new Sen2VMException("GIPP directory(ies) for GIPP type " + fileNameRegex + " must match GIPP list from Datastrip Metadata: " + Arrays.toString(gippList.toArray())); + } } else if(results.size()>1) { @@ -276,7 +307,7 @@ else if(results.size()>1) .map(File::getAbsolutePath) .collect(Collectors.joining(", ")); - LOGGER.info("GIPP "+dirNameRegex+" : "+message); + LOGGER.info("GIPP " + dirNameRegex + " : " + message); throw new Sen2VMException("Duplicate GIPP file type found"); } else diff --git a/src/main/java/esa/sen2vm/input/gipp/GIPPManager.java b/src/main/java/esa/sen2vm/input/gipp/GIPPManager.java index 9a66c933..dc2e1fe9 100644 --- a/src/main/java/esa/sen2vm/input/gipp/GIPPManager.java +++ b/src/main/java/esa/sen2vm/input/gipp/GIPPManager.java @@ -155,7 +155,7 @@ protected void loadAllGIPP(List bands) throws Sen2VMException } catch (Exception e) { - throw new Sen2VMException("Error when reading the blind pixel GIPP file: " + fileBlindPixel, e); + throw new Sen2VMException("Error when reading the blind pixel (GIP_BLINDP) GIPP file: " + fileBlindPixel + ": " + e.getMessage(), e); } // Load spacecraft model gipp @@ -179,7 +179,7 @@ protected void loadAllGIPP(List bands) throws Sen2VMException } catch (Exception e) { - throw new Sen2VMException("Error when reading spacecraft model GIPP file: " + fileSpaMod, e); + throw new Sen2VMException("Error when reading spacecraft model (GIP_SPAMOD) GIPP file: " + fileSpaMod + ": " + e.getMessage(), e); } // Load viewing directions gipp @@ -191,7 +191,7 @@ protected void loadAllGIPP(List bands) throws Sen2VMException File file = gippFilePathFromIndexBand(bands.get(i), gippFilePathList); if (file == null) { - throw new Sen2VMException("Viewing directions GIPP file missing for band "+ bands.get(i)); + throw new Sen2VMException("Viewing directions (GIP_VIEDIR) GIPP file missing for band "+ bands.get(i)); } // Load GIPP DATA @@ -211,7 +211,7 @@ protected void loadAllGIPP(List bands) throws Sen2VMException } catch (Exception e) { - throw new Sen2VMException("Error when reading viewing directions GIPP files from", e); + throw new Sen2VMException("Error when reading viewing directions GIPP (GIP_VIEDIR) files: " + e.getMessage(), e); } // Load prdloc model gipp, only for RAW mode diff --git a/src/test/java/esa/sen2vm/Sen2VMDirectTest.java b/src/test/java/esa/sen2vm/Sen2VMDirectTest.java index 39fd196f..3fb17b13 100644 --- a/src/test/java/esa/sen2vm/Sen2VMDirectTest.java +++ b/src/test/java/esa/sen2vm/Sen2VMDirectTest.java @@ -174,6 +174,56 @@ public void testAutoSelectTarGipp() } } + @Test + public void testNoAutoSelectTarGipp() + { + String[] detectors = new String[]{"01"}; + String[] bands = new String[]{"B01"}; + String GIPP_archive = "src/test/resources/tests/data/archive_GIPP/"; + String GIPP_2 = "src/test/resources/tests/data/test_GIPP/"; + File gippDir= new File(GIPP_2); + File sourceArchive= new File(GIPP_archive); + if(Files.exists(gippDir.toPath())) + { + + Config.deleteDirectory(gippDir); + } + gippDir.mkdir(); + try + { + Config.copyFolder(sourceArchive,gippDir,true); + + File fileToRemove = new File("src/test/resources/tests/data/test_GIPP/S2A_OPER_GIP_SPAMOD_MPC__20210419T000024_V20210421T233000_21000101T000000_B00.xml"); + LOGGER.info("File to remove: "+fileToRemove.toString()); + fileToRemove.delete(); + + fileToRemove = new File("src/test/resources/tests/data/test_GIPP/S2A_OPER_GIP_SPAMOD_MPC__20210419T000024_V20210421T233000_21000101T000000_B00.tar.gz"); + LOGGER.info("File to remove: "+fileToRemove.toString()); + fileToRemove.delete(); + + fileToRemove = new File("src/test/resources/tests/data/test_GIPP/S2A_OPER_GIP_BLINDP_MPC__20150605T094736_V20150622T000000_21000101T000000_B00/S2A_OPER_GIP_BLINDP_MPC__20150605T094736_V20150622T000000_21000101T000000_B00.DBL"); + LOGGER.info("File to remove: "+fileToRemove.toString()); + fileToRemove.delete(); + + String nameTest = "testDirectLoc"; + String outputDir = Config.createTestDir(Config.TDS.TDS1, nameTest, "direct"); + String config = Config.configAutoGippSelection(configTmpDirectTDS1, GIPP_2, false, outputDir); + String param = Config.changeParams(paramTmp, detectors, bands, outputDir); + String[] args = {"-c", config, "-p", param}; + LOGGER.info("config: "+config); + Sen2VM.main(args); + Utils.verifyDirectLoc(config, refDir + "/" + nameTest); + } catch (Sen2VMException e) { + LOGGER.warning(e.getMessage()); + e.printStackTrace(); + assert(false); + } catch (Exception e) { + LOGGER.warning(e.getMessage()); + e.printStackTrace(); + assert(false); + } + } + @Test public void testAutoSelectWithMissingGipp() { @@ -218,6 +268,61 @@ public void testAutoSelectWithMissingGipp() } } + @Test + public void testAutoSelectWithMissingUntarGipp() + { + String[] detectors = new String[]{"01"}; + String[] bands = new String[]{"B01"}; + String GIPP_archive = "src/test/resources/tests/data/archive_GIPP/"; + String GIPP_2 = "src/test/resources/tests/data/test_GIPP/"; + File gippDir= new File(GIPP_2); + File sourceArchive= new File(GIPP_archive); + if(Files.exists(gippDir.toPath())) + { + + Config.deleteDirectory(gippDir); + } + gippDir.mkdir(); + + try + { + Config.copyFolder(sourceArchive,gippDir,true); + + // remove a listed GIPP to check a test failure + File fileToRemove = new File("src/test/resources/tests/data/test_GIPP/S2A_OPER_GIP_SPAMOD_MPC__20210419T000024_V20210421T233000_21000101T000000_B00.xml"); + LOGGER.info("File to remove: "+fileToRemove.toString()); + fileToRemove.delete(); + + fileToRemove = new File("src/test/resources/tests/data/test_GIPP/S2A_OPER_GIP_SPAMOD_MPC__20210419T000024_V20210421T233000_21000101T000000_B00.tar.gz"); + LOGGER.info("File to remove: "+fileToRemove.toString()); + fileToRemove.delete(); + + fileToRemove = new File("src/test/resources/tests/data/test_GIPP/S2A_OPER_GIP_SPAMOD_MPC__20220120T000025_V20220125T022000_21000101T000000_B00.tar.gz"); + LOGGER.info("File to remove: "+fileToRemove.toString()); + fileToRemove.delete(); + + + String nameTest = "testDirectLoc"; + String outputDir = Config.createTestDir(Config.TDS.TDS1, nameTest, "direct"); + String config = Config.configAutoGippSelection(configTmpDirectTDS1, GIPP_2, true, outputDir); + String param = Config.changeParams(paramTmp, detectors, bands, outputDir); + String[] args = {"-c", config, "-p", param}; + LOGGER.info("config: "+config); + Sen2VM.main(args); + Utils.verifyDirectLoc(config, refDir + "/" + nameTest); + LOGGER.warning("Expecting an error."); + assert(false); + } catch (Sen2VMException e) { + LOGGER.warning(e.getMessage()); + e.printStackTrace(); + assert(true); + } catch (Exception e) { + LOGGER.warning(e.getMessage()); + e.printStackTrace(); + assert(false); + } + } + @Test public void testDirectGippError() {