diff --git a/scripts/update-h3.sh b/scripts/update-h3.sh index 5abdbca..f4c40ba 100755 --- a/scripts/update-h3.sh +++ b/scripts/update-h3.sh @@ -27,7 +27,6 @@ # Options: # --env-file PATH Path to .env file (default: ./scripts/.env) # --data-dir PATH Directory for import data (default: ./data) -# --jar-file PATH Path to PAIKKA jar (auto-detected if not provided) # --memory SIZE JVM heap size (default: 16g) # --threads NUM Import threads (default: 10) # --pbf-file PATH Use local PBF file instead of downloading @@ -58,7 +57,6 @@ DATA_DIR="${DATA_DIR:-$LOCAL_WORK_DIR}" # --- Import Settings --- IMPORT_MEMORY="${IMPORT_MEMORY:-16g}" IMPORT_THREADS="${IMPORT_THREADS:-10}" -JAR_FILE="${JAR_FILE:-}" # --- Bundle Settings --- VERSION="${VERSION:-$(date +%Y-%m-%d)-v1}" @@ -123,10 +121,6 @@ parse_args_and_configure() { DATA_DIR="$2" shift 2 ;; - --jar-file) - JAR_FILE="$2" - shift 2 - ;; --memory) IMPORT_MEMORY="$2" shift 2 @@ -157,7 +151,6 @@ parse_args_and_configure() { echo "Options:" echo " --env-file PATH Path to .env file (default: ./scripts/.env)" echo " --data-dir PATH Directory for import data (default: ./data)" - echo " --jar-file PATH Path to PAIKKA jar (auto-detected if not provided)" echo " --memory SIZE JVM heap size (default: 16g)" echo " --threads NUM Import threads (default: 10)" echo " --pbf-file PATH Use local PBF file instead of downloading" @@ -190,17 +183,6 @@ parse_args_and_configure() { exit 1 fi - # Auto-detect JAR file if not provided - if [ -z "$JAR_FILE" ]; then - JAR_FILE=$(find target -name "paikka-*.jar" -not -name "*-sources.jar" 2>/dev/null | head -1) - fi - - # Validate JAR file - if [ -n "$JAR_FILE" ] && [ ! -f "$JAR_FILE" ]; then - echo "Error: JAR file not found: $JAR_FILE" - exit 1 - fi - # Display configuration echo "==========================================" echo "H3 Bundle Pipeline Configuration" @@ -208,7 +190,6 @@ parse_args_and_configure() { echo " Data directory: $DATA_DIR" echo " Import memory: $IMPORT_MEMORY" echo " Import threads: $IMPORT_THREADS" - echo " JAR file: ${JAR_FILE:-auto-detect}" echo " Bundle version: $VERSION" echo " Bundle output: $BUNDLE_OUTPUT_DIR" echo " Skip upload: $NO_UPLOAD" @@ -276,52 +257,63 @@ local_pull_docker_image() { # LOCAL: Filters the PBF file using the Paikka container. ### local_filter_pbf() { + if [ -n "$PBF_INPUT_PATH" ]; then + log "Step 3: Skipping filter – using provided PBF directly" + return 0 + fi + log "Step 3: Filtering PBF file" + sudo docker run --rm \ + -v "$DOWNLOAD_DIR":/data \ + "$DOCKER_IMAGE" prepare-boundaries "/data/$PBF_INPUT_FILE" "/data/$PBF_FILTERED_FILE" +} +### +# LOCAL: Runs the Java H3 import. +### +local_import_h3() { + log "Step 4: Running H3 import" + if [ -n "$PBF_INPUT_PATH" ]; then INPUT_DIR="$(dirname "$PBF_INPUT_PATH")" INPUT_FILE="$(basename "$PBF_INPUT_PATH")" sudo docker run --rm \ -v "$INPUT_DIR":/input \ - -v "$DOWNLOAD_DIR":/data \ - "$DOCKER_IMAGE" prepare-boundaries "/input/$INPUT_FILE" "/data/$PBF_FILTERED_FILE" + -v "$DATA_DIR":/data \ + "$DOCKER_IMAGE" \ + import-boundaries \ + --data-dir /data \ + --memory "$IMPORT_MEMORY" \ + --threads "$IMPORT_THREADS" \ + "/input/$INPUT_FILE" else sudo docker run --rm \ - -v "$DOWNLOAD_DIR":/data \ - "$DOCKER_IMAGE" prepare-boundaries "/data/$PBF_INPUT_FILE" "/data/$PBF_FILTERED_FILE" + -v "$DOWNLOAD_DIR":/input \ + -v "$DATA_DIR":/data \ + "$DOCKER_IMAGE" \ + import-boundaries \ + --data-dir /data \ + --memory "$IMPORT_MEMORY" \ + --threads "$IMPORT_THREADS" \ + "/input/$PBF_FILTERED_FILE" fi } - -### -# LOCAL: Runs the Java H3 import. -### -local_import_h3() { - log "Step 4: Running H3 import" - - local PBF_TO_IMPORT="$DOWNLOAD_DIR/$PBF_FILTERED_FILE" - - cd "$LOCAL_WORK_DIR" - ./scripts/import-boundaries.sh \ - --jar-file "$JAR_FILE" \ - --data-dir "$DATA_DIR" \ - --memory "$IMPORT_MEMORY" \ - --threads "$IMPORT_THREADS" \ - "$PBF_TO_IMPORT" -} - ### # LOCAL: Removes intermediate PBF files. ### local_cleanup_pbf() { log "Step 5: Cleaning up intermediate PBF files" + + if [ -n "$PBF_INPUT_PATH" ]; then + echo "No intermediate files to clean up (user-provided PBF)" + return 0 + fi + cd "$DOWNLOAD_DIR" rm -f "$PBF_FILTERED_FILE" - if [ -z "$PBF_INPUT_PATH" ]; then - rm -f "$PBF_INPUT_FILE" - fi - echo "Cleaned up filtered PBF file" + rm -f "$PBF_INPUT_FILE" + echo "Cleaned up downloaded PBF files" } - ### # LOCAL: Creates the H3 RocksDB bundle ZIP and manifest. ### @@ -346,10 +338,15 @@ local_upload_bundle() { log "Step 7: Uploading bundle to R2" - ./scripts/upload-h3-bundle.sh \ + local upload_args=( --dist-dir "$BUNDLE_OUTPUT_DIR" -} + ) + if [ -n "$ENV_FILE" ] && [ -f "$ENV_FILE" ]; then + upload_args+=(--env-file "$ENV_FILE") + fi + ./scripts/upload-h3-bundle.sh "${upload_args}" +} # ============================================================================== # MAIN ORCHESTRATION FUNCTION # ============================================================================== @@ -358,7 +355,7 @@ main() { parse_args_and_configure "$@" local_prepare_directories local_download_planet_file -# local_pull_docker_image + local_pull_docker_image local_filter_pbf local_import_h3 local_cleanup_pbf diff --git a/scripts/upload-h3-bundle.sh b/scripts/upload-h3-bundle.sh index 878fbe4..90d0f06 100755 --- a/scripts/upload-h3-bundle.sh +++ b/scripts/upload-h3-bundle.sh @@ -4,8 +4,8 @@ set -e # --- Configuration & Defaults --- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ENV_FILE="$SCRIPT_DIR/.env" -DIST_DIR="$SCRIPT_DIR/dist" # Default fallback if no directory parameter is provided +ENV_FILE="" # No default - must be provided or use environment variables +DIST_DIR="$SCRIPT_DIR/dist" # Show help/usage instructions usage() { @@ -13,6 +13,7 @@ usage() { echo "" echo "Options:" echo " -d, --dist-dir Path to folder containing the ZIP file and manifest.json (Default: $DIST_DIR)" + echo " -e, --env-file Path to .env file with R2 credentials (optional if set via environment)" echo " -h, --help Show this help message" exit 1 } @@ -21,18 +22,21 @@ usage() { while [[ "$#" -gt 0 ]]; do case $1 in -d|--dist-dir) DIST_DIR="$2"; shift ;; + -e|--env-file) ENV_FILE="$2"; shift ;; -h|--help) usage ;; *) echo "Unknown parameter: $1"; usage ;; esac shift done -# Load credentials from .env file -if [ -f "$ENV_FILE" ]; then - source "$ENV_FILE" -else - echo "Error: Configuration file .env was not found at: $ENV_FILE" - exit 1 +# Load credentials from .env file (if provided) +if [ -n "$ENV_FILE" ]; then + if [ -f "$ENV_FILE" ]; then + source "$ENV_FILE" + else + echo "Error: Configuration file .env was not found at: $ENV_FILE" + exit 1 + fi fi # AWS CLI check @@ -89,7 +93,7 @@ echo "Uploading H3 RocksDB Bundle to R2" echo "Source Dir: $DIST_DIR_ABS" echo "Bundle: $ZIP_FILENAME" echo "Bucket: $R2_BUCKET" -echo "Prefix: ${REMOTE_PREFIX:-[root]}" +echo "Prefix: ${REMOTE_PREFIX:-}" echo "==========================================" # 1. Upload the heavy ZIP file first @@ -119,7 +123,7 @@ ZIPS_IN_BUCKET=$(aws s3api list-objects-v2 \ # Convert output into a Bash array read -r -a ZIP_ARRAY <<< "$ZIPS_IN_BUCKET" -TOTAL_ZIPS=${#ZIP_ARRAY[@]} +TOTAL_ZIPS=${#ZIP_ARRAY} echo "$TOTAL_ZIPS ZIP file(s) found in bucket." diff --git a/src/main/java/com/dedicatedcode/paikka/controller/GeocodingController.java b/src/main/java/com/dedicatedcode/paikka/controller/GeocodingController.java index 9891a3e..e419d9e 100644 --- a/src/main/java/com/dedicatedcode/paikka/controller/GeocodingController.java +++ b/src/main/java/com/dedicatedcode/paikka/controller/GeocodingController.java @@ -45,12 +45,12 @@ public class GeocodingController { private final ReverseGeocodingService reverseGeocodingService; private final PaikkaConfiguration config; - private final MetadataService metadataService; // Inject MetadataService + private final MetadataService metadataService; public GeocodingController(ReverseGeocodingService reverseGeocodingService, PaikkaConfiguration config, MetadataService metadataService) { this.reverseGeocodingService = reverseGeocodingService; this.config = config; - this.metadataService = metadataService; // Inject MetadataService + this.metadataService = metadataService; } /** @@ -64,8 +64,8 @@ public GeocodingController(ReverseGeocodingService reverseGeocodingService, Paik */ @GetMapping("/reverse") public ResponseEntity> reverse( - @RequestParam double lat, - @RequestParam double lon, + @RequestParam(required = false) Double lat, + @RequestParam(required = false) Double lon, @RequestParam(defaultValue = "en") String lang, @RequestParam(required = false) Integer limit) { @@ -73,7 +73,19 @@ public ResponseEntity> reverse( int effectiveLimit = (limit != null) ? Math.min(limit, config.getQueryConfiguration().getMaxResults()) : config.getQueryConfiguration().getDefaultResults(); logger.debug("Reverse geocoding request: lat={}, lon={}, lang={}, limit={}", lat, lon, lang, effectiveLimit); - + + if (lat == null || lon == null) { + Map error = new HashMap<>(); + if (lat == null && lon == null) { + error.put("error", "Missing required parameters: lat, lon"); + } else if (lat == null) { + error.put("error", "Missing required parameter: lat"); + } else { + error.put("error", "Missing required parameter: lon"); + } + return ResponseEntity.badRequest().body(error); + } + // Validate coordinates if (lat < -90 || lat > 90) { Map error = new HashMap<>(); @@ -111,7 +123,7 @@ public ResponseEntity> reverse( return ResponseEntity.ok() .header("X-Result-Count", String.valueOf(results.size())) - .header("Cache-Control", "no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0") // No caching + .header("Cache-Control", "max-age=86400") .body(response); } @@ -123,7 +135,7 @@ public ResponseEntity> health() { Map response = new HashMap<>(); response.put("status", "ok"); response.put("service", "paikka"); - response.put("metadata", metadataService.getMetadata()); // Include metadata + response.put("metadata", metadataService.getMetadata()); return ResponseEntity.ok() .header("Cache-Control", "no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0") // No caching .body(response); diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/BoundaryImportStatistics.java b/src/main/java/com/dedicatedcode/paikka/service/importer/BoundaryImportStatistics.java index 602e46c..88c3d5b 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/BoundaryImportStatistics.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/BoundaryImportStatistics.java @@ -192,7 +192,8 @@ public String getMemoryStats() { } public void startProgressReporter() { - boolean isTty = System.console() != null; + boolean isTty = System.console() != null && System.console().isTerminal(); + long sleepMillis = isTty ? 1000 : 5000; Thread.ofPlatform().daemon().start(() -> { while (isRunning()) { @@ -237,7 +238,7 @@ public void startProgressReporter() { } try { - Thread.sleep(1000); + Thread.sleep(sleepMillis); } catch (InterruptedException e) { break; } diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/ImportService.java b/src/main/java/com/dedicatedcode/paikka/service/importer/ImportService.java index 52e3190..52cd0d2 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/ImportService.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/ImportService.java @@ -793,9 +793,9 @@ private void processAdministrativeBoundariesFromIndex(RocksDB relIndexDb, RocksD RelRec rec = decodeRelRec(val, relId); semaphore.acquire(); - stats.incrementActiveThreads(); ecs.submit(() -> { try { + stats.incrementActiveThreads(); org.locationtech.jts.geom.Geometry geometry = buildGeometryFromRelRec(rec, nodeCache, wayIndexDb, stats); if (geometry == null) return null; @@ -908,11 +908,12 @@ private void processBuildingBoundariesFromIndex(RocksDB poiIndexDb, RocksDB node if (currentBatchIds.size() >= batchSize) { semaphore.acquire(); - stats.incrementActiveThreads(); List idsToProcess = new ArrayList<>(currentBatchIds); List recsToProcess = new ArrayList<>(currentBatchRecs); ecs.submit(() -> { try { + stats.incrementActiveThreads(); + List results = new ArrayList<>(); for (int i = 0; i < idsToProcess.size(); i++) { long wayIdInner = idsToProcess.get(i); @@ -972,11 +973,11 @@ private void processBuildingBoundariesFromIndex(RocksDB poiIndexDb, RocksDB node if (!currentBatchIds.isEmpty()) { semaphore.acquire(); - stats.incrementActiveThreads(); List idsToProcess = new ArrayList<>(currentBatchIds); List recsToProcess = new ArrayList<>(currentBatchRecs); ecs.submit(() -> { try { + stats.incrementActiveThreads(); List results = new ArrayList<>(); for (int i = 0; i < idsToProcess.size(); i++) { long wayIdInner = idsToProcess.get(i); @@ -1160,6 +1161,7 @@ private int serializePoiData(FlatBufferBuilder builder, PoiData poi) { private void compactBuildingShards(RocksDB appendDb, RocksDB buildingsDb, ImportStatistics stats) { stats.setCompactionStartTime(System.currentTimeMillis()); stats.setCompactionEntriesTotal(buildingSequence.get()); + stats.resetCompactionProgress(); Building reusableBuilding = new Building(); Geometry reusableGeom = new Geometry(); diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/ImportStatistics.java b/src/main/java/com/dedicatedcode/paikka/service/importer/ImportStatistics.java index 5d0ec0c..de7842e 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/ImportStatistics.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/ImportStatistics.java @@ -318,6 +318,9 @@ public void setTotalTime(long t) { public void setCompactionEntriesTotal(long v) { this.compactionEntriesTotal.set(v); } + public void resetCompactionProgress() { + this.compactionEntriesProcessed.set(0); + } public void incrementCompactionEntriesProcessed(long size) { this.compactionEntriesProcessed.addAndGet(size); @@ -489,8 +492,8 @@ public String getMemoryStats() { } public void startProgressReporter() { - boolean isTty = System.console() != null; - + boolean isTty = System.console() != null && System.console().isTerminal(); + long sleepMillis = isTty ? 1000 : 5000; Thread.ofPlatform().daemon().start(() -> { while (isRunning()) { long elapsed = System.currentTimeMillis() - getStartTime(); @@ -612,7 +615,7 @@ public void startProgressReporter() { } try { - Thread.sleep(1000); + Thread.sleep(sleepMillis); } catch (InterruptedException e) { break; }