diff --git a/CHANGELOG.md b/CHANGELOG.md index 35e70999..865731e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - --> -## [v3.0.9](https://github.com/TUDelftGeodesy/caroline/tree/main) (27-Nov-2025, [diff](https://github.com/TUDelftGeodesy/caroline/compare/07b3e59109de44d5c223806757416a132514f112...main)) +## [v3.0.10](https://github.com/TUDelftGeodesy/caroline/tree/main) (17-Dec-2025, [diff](https://github.com/TUDelftGeodesy/caroline/compare/6543634985e0be7fdaf3cf09202f031f584f3396...main)) + +### Fixed: +- `find-new-insar-files.sh` will now take new directories that lack orbit data with it to the next check if the reference timestamp file is updated (by updating the timestamps of the relevant `*.json` files) + + +## [v3.0.9](https://github.com/TUDelftGeodesy/caroline/tree/6543634985e0be7fdaf3cf09202f031f584f3396) (27-Nov-2025, [diff](https://github.com/TUDelftGeodesy/caroline/compare/07b3e59109de44d5c223806757416a132514f112...6543634985e0be7fdaf3cf09202f031f584f3396)) ### Added: - `s1_download` will now perform a ZIP-file integrity check after the call to `caroline-download` completes. If incomplete downloads are found, those are removed and the call to `caroline-download` is re-initiated. If this loop repeats over 20 times, `s1_download` aborts with exit code `5` to prevent infinite loops. diff --git a/pyproject.toml b/pyproject.toml index b1de73c6..74b180c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "Caroline" -version = "3.0.9" +version = "3.0.10" requires-python = ">=3.10" dependencies = [ "apache-airflow", diff --git a/scripts/find-new-insar-files.sh b/scripts/find-new-insar-files.sh index e8536319..8c8d7d38 100755 --- a/scripts/find-new-insar-files.sh +++ b/scripts/find-new-insar-files.sh @@ -21,6 +21,7 @@ TIMESTAMP_FILE="/project/caroline/Share/users/${USER}/run/find-new-insar-files-t main () { local NEW_DOWNLOADS=() + local DOWNLOADS_TO_TAKE_TO_NEXT_CHECK=() local OPTIONS='' local NEW_TIMESTAMP='' local TRACK='' @@ -102,6 +103,8 @@ main () { if is_download_complete "${DOWNLOAD_DIR}"; then if orbits_downloaded "${DOWNLOAD_DIR}"; then NEW_DOWNLOADS+=("${DOWNLOAD_DIR}") + else + DOWNLOADS_TO_TAKE_TO_NEXT_CHECK+=("${DOWNLOAD_DIR}") fi fi done @@ -112,7 +115,7 @@ main () { # we report this and return exit 1 so that if this script is used as # a test the test will fail if [ ${#NEW_DOWNLOADS[@]} -gt 0 ]; then - # Upate the timestamp file + # Update the timestamp file touch "${TIMESTAMP_FILE}" # List the zipfiles we found @@ -120,7 +123,19 @@ main () { ls "${DOWNLOAD_DIR}"/*.zip done - # Return succes so that if this script is used as a test + # If orbit data was lacking for one or more of the found data directories + # but was found for one or more others, the timestamp file is updated and + # we will no longer detect the data directories with the missing orbits. + # We take those ones with us to the next check by updating its timestamp to + # be later than the TIMESTAMP FILE. Only touch the json files so that the + # zip files still retain their original download date + # First sleep 2 seconds so the timestamps cannot be the same + sleep 2 + for DOWNLOAD_DIR in ${DOWNLOADS_TO_TAKE_TO_NEXT_CHECK[@]}; do + touch "${DOWNLOAD_DIR}"/*.json + done + + # Return success so that if this script is used as a test # the test will pass exit 0 else