44 feature request add explanation what went wrong

scripts/update-h3.sh

@@ -27,7 +27,6 @@
 # Options:
 #   --env-file PATH       Path to .env file (default: ./scripts/.env)
 #   --data-dir PATH       Directory for import data (default: ./data)
-#   --jar-file PATH       Path to PAIKKA jar (auto-detected if not provided)
 #   --memory SIZE         JVM heap size (default: 16g)
 #   --threads NUM         Import threads (default: 10)
 #   --pbf-file PATH       Use local PBF file instead of downloading
@@ -58,7 +57,6 @@ DATA_DIR="${DATA_DIR:-$LOCAL_WORK_DIR}"
 # --- Import Settings ---
 IMPORT_MEMORY="${IMPORT_MEMORY:-16g}"
 IMPORT_THREADS="${IMPORT_THREADS:-10}"
-JAR_FILE="${JAR_FILE:-}"
 
 # --- Bundle Settings ---
 VERSION="${VERSION:-$(date +%Y-%m-%d)-v1}"
@@ -123,10 +121,6 @@ parse_args_and_configure() {
                 DATA_DIR="$2"
                 shift 2
                 ;;
-            --jar-file)
-                JAR_FILE="$2"
-                shift 2
-                ;;
             --memory)
                 IMPORT_MEMORY="$2"
                 shift 2
@@ -157,7 +151,6 @@ parse_args_and_configure() {
                 echo "Options:"
                 echo "  --env-file PATH       Path to .env file (default: ./scripts/.env)"
                 echo "  --data-dir PATH       Directory for import data (default: ./data)"
-                echo "  --jar-file PATH       Path to PAIKKA jar (auto-detected if not provided)"
                 echo "  --memory SIZE         JVM heap size (default: 16g)"
                 echo "  --threads NUM         Import threads (default: 10)"
                 echo "  --pbf-file PATH       Use local PBF file instead of downloading"
@@ -190,25 +183,13 @@ parse_args_and_configure() {
         exit 1
     fi
 
-    # Auto-detect JAR file if not provided
-    if [ -z "$JAR_FILE" ]; then
-        JAR_FILE=$(find target -name "paikka-*.jar" -not -name "*-sources.jar" 2>/dev/null | head -1)
-    fi
-
-    # Validate JAR file
-    if [ -n "$JAR_FILE" ] && [ ! -f "$JAR_FILE" ]; then
-        echo "Error: JAR file not found: $JAR_FILE"
-        exit 1
-    fi
-
     # Display configuration
     echo "=========================================="
     echo "H3 Bundle Pipeline Configuration"
     echo "=========================================="
     echo "  Data directory:    $DATA_DIR"
     echo "  Import memory:     $IMPORT_MEMORY"
     echo "  Import threads:    $IMPORT_THREADS"
-    echo "  JAR file:          ${JAR_FILE:-auto-detect}"
     echo "  Bundle version:    $VERSION"
     echo "  Bundle output:     $BUNDLE_OUTPUT_DIR"
     echo "  Skip upload:       $NO_UPLOAD"
@@ -276,52 +257,63 @@ local_pull_docker_image() {
 # LOCAL: Filters the PBF file using the Paikka container.
 ###
 local_filter_pbf() {
+    if [ -n "$PBF_INPUT_PATH" ]; then
+        log "Step 3: Skipping filter – using provided PBF directly"
+        return 0
+    fi
+
     log "Step 3: Filtering PBF file"
 
+    sudo docker run --rm \
+        -v "$DOWNLOAD_DIR":/data \
+        "$DOCKER_IMAGE" prepare-boundaries "/data/$PBF_INPUT_FILE" "/data/$PBF_FILTERED_FILE"
+}
+###
+# LOCAL: Runs the Java H3 import.
+###
+local_import_h3() {
+    log "Step 4: Running H3 import"
+
     if [ -n "$PBF_INPUT_PATH" ]; then
         INPUT_DIR="$(dirname "$PBF_INPUT_PATH")"
         INPUT_FILE="$(basename "$PBF_INPUT_PATH")"
         sudo docker run --rm \
             -v "$INPUT_DIR":/input \
-            -v "$DOWNLOAD_DIR":/data \
-            "$DOCKER_IMAGE" prepare-boundaries "/input/$INPUT_FILE" "/data/$PBF_FILTERED_FILE"
+            -v "$DATA_DIR":/data \
+            "$DOCKER_IMAGE" \
+            import-boundaries \
+            --data-dir /data \
+            --memory "$IMPORT_MEMORY" \
+            --threads "$IMPORT_THREADS" \
+            "/input/$INPUT_FILE"
     else
         sudo docker run --rm \
-            -v "$DOWNLOAD_DIR":/data \
-            "$DOCKER_IMAGE" prepare-boundaries "/data/$PBF_INPUT_FILE" "/data/$PBF_FILTERED_FILE"
+            -v "$DOWNLOAD_DIR":/input \
+            -v "$DATA_DIR":/data \
+            "$DOCKER_IMAGE" \
+            import-boundaries \
+            --data-dir /data \
+            --memory "$IMPORT_MEMORY" \
+            --threads "$IMPORT_THREADS" \
+            "/input/$PBF_FILTERED_FILE"
     fi
 }
-
-###
-# LOCAL: Runs the Java H3 import.
-###
-local_import_h3() {
-    log "Step 4: Running H3 import"
-
-    local PBF_TO_IMPORT="$DOWNLOAD_DIR/$PBF_FILTERED_FILE"
-
-    cd "$LOCAL_WORK_DIR"
-    ./scripts/import-boundaries.sh \
-        --jar-file "$JAR_FILE" \
-        --data-dir "$DATA_DIR" \
-        --memory "$IMPORT_MEMORY" \
-        --threads "$IMPORT_THREADS" \
-        "$PBF_TO_IMPORT"
-}
-
 ###
 # LOCAL: Removes intermediate PBF files.
 ###
 local_cleanup_pbf() {
     log "Step 5: Cleaning up intermediate PBF files"
+
+    if [ -n "$PBF_INPUT_PATH" ]; then
+        echo "No intermediate files to clean up (user-provided PBF)"
+        return 0
+    fi
+
     cd "$DOWNLOAD_DIR"
     rm -f "$PBF_FILTERED_FILE"
-    if [ -z "$PBF_INPUT_PATH" ]; then
-        rm -f "$PBF_INPUT_FILE"
-    fi
-    echo "Cleaned up filtered PBF file"
+    rm -f "$PBF_INPUT_FILE"
+    echo "Cleaned up downloaded PBF files"
 }
-
 ###
 # LOCAL: Creates the H3 RocksDB bundle ZIP and manifest.
 ###
@@ -346,10 +338,15 @@ local_upload_bundle() {
 
     log "Step 7: Uploading bundle to R2"
 
-    ./scripts/upload-h3-bundle.sh \
+    local upload_args=(
         --dist-dir "$BUNDLE_OUTPUT_DIR"
-}
+    )
+    if [ -n "$ENV_FILE" ] && [ -f "$ENV_FILE" ]; then
+        upload_args+=(--env-file "$ENV_FILE")
+    fi
 
+    ./scripts/upload-h3-bundle.sh "${upload_args}"
+}
 # ==============================================================================
 # MAIN ORCHESTRATION FUNCTION
 # ==============================================================================
@@ -358,7 +355,7 @@ main() {
     parse_args_and_configure "$@"
     local_prepare_directories
     local_download_planet_file
-#    local_pull_docker_image
+    local_pull_docker_image
     local_filter_pbf
     local_import_h3
     local_cleanup_pbf

scripts/upload-h3-bundle.sh

@@ -4,15 +4,16 @@ set -e
 
 # --- Configuration & Defaults ---
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-ENV_FILE="$SCRIPT_DIR/.env"
-DIST_DIR="$SCRIPT_DIR/dist"  # Default fallback if no directory parameter is provided
+ENV_FILE=""  # No default - must be provided or use environment variables
+DIST_DIR="$SCRIPT_DIR/dist"
 
 # Show help/usage instructions
 usage() {
     echo "Usage: $0 [OPTIONS]"
     echo ""
     echo "Options:"
     echo "  -d, --dist-dir     Path to folder containing the ZIP file and manifest.json (Default: $DIST_DIR)"
+    echo "  -e, --env-file     Path to .env file with R2 credentials (optional if set via environment)"
     echo "  -h, --help         Show this help message"
     exit 1
 }
@@ -21,18 +22,21 @@ usage() {
 while [[ "$#" -gt 0 ]]; do
     case $1 in
         -d|--dist-dir) DIST_DIR="$2"; shift ;;
+        -e|--env-file) ENV_FILE="$2"; shift ;;
         -h|--help) usage ;;
         *) echo "Unknown parameter: $1"; usage ;;
     esac
     shift
 done
 
-# Load credentials from .env file
-if [ -f "$ENV_FILE" ]; then
-    source "$ENV_FILE"
-else
-    echo "Error: Configuration file .env was not found at: $ENV_FILE"
-    exit 1
+# Load credentials from .env file (if provided)
+if [ -n "$ENV_FILE" ]; then
+    if [ -f "$ENV_FILE" ]; then
+        source "$ENV_FILE"
+    else
+        echo "Error: Configuration file .env was not found at: $ENV_FILE"
+        exit 1
+    fi
 fi
 
 # AWS CLI check
@@ -89,7 +93,7 @@ echo "Uploading H3 RocksDB Bundle to R2"
 echo "Source Dir: $DIST_DIR_ABS"
 echo "Bundle:     $ZIP_FILENAME"
 echo "Bucket:     $R2_BUCKET"
-echo "Prefix:     ${REMOTE_PREFIX:-[root]}"
+echo "Prefix:     ${REMOTE_PREFIX:-}"
 echo "=========================================="
 
 # 1. Upload the heavy ZIP file first
@@ -119,7 +123,7 @@ ZIPS_IN_BUCKET=$(aws s3api list-objects-v2 \
 
 # Convert output into a Bash array
 read -r -a ZIP_ARRAY <<< "$ZIPS_IN_BUCKET"
-TOTAL_ZIPS=${#ZIP_ARRAY[@]}
+TOTAL_ZIPS=${#ZIP_ARRAY}
 
 echo "$TOTAL_ZIPS ZIP file(s) found in bucket."
 

src/main/java/com/dedicatedcode/paikka/controller/GeocodingController.java

@@ -45,12 +45,12 @@ public class GeocodingController {
 
     private final ReverseGeocodingService reverseGeocodingService;
     private final PaikkaConfiguration config;
-    private final MetadataService metadataService; // Inject MetadataService
+    private final MetadataService metadataService;
 
     public GeocodingController(ReverseGeocodingService reverseGeocodingService, PaikkaConfiguration config, MetadataService metadataService) {
         this.reverseGeocodingService = reverseGeocodingService;
         this.config = config;
-        this.metadataService = metadataService; // Inject MetadataService
+        this.metadataService = metadataService;
     }
 
     /**
@@ -64,16 +64,28 @@ public GeocodingController(ReverseGeocodingService reverseGeocodingService, Paik
      */
     @GetMapping("/reverse")
     public ResponseEntity<Map<String, Object>> reverse(
-            @RequestParam double lat,
-            @RequestParam double lon,
+            @RequestParam(required = false) Double lat,
+            @RequestParam(required = false) Double lon,
             @RequestParam(defaultValue = "en") String lang,
             @RequestParam(required = false) Integer limit) {
 
         // Determine effective limit
         int effectiveLimit = (limit != null) ? Math.min(limit, config.getQueryConfiguration().getMaxResults()) : config.getQueryConfiguration().getDefaultResults();
 
         logger.debug("Reverse geocoding request: lat={}, lon={}, lang={}, limit={}", lat, lon, lang, effectiveLimit);
-
+
+        if (lat == null || lon == null) {
+            Map<String, Object> error = new HashMap<>();
+            if (lat == null && lon == null) {
+                error.put("error", "Missing required parameters: lat, lon");
+            } else if (lat == null) {
+                error.put("error", "Missing required parameter: lat");
+            } else {
+                error.put("error", "Missing required parameter: lon");
+            }
+            return ResponseEntity.badRequest().body(error);
+        }
+
         // Validate coordinates
         if (lat < -90 || lat > 90) {
             Map<String, Object> error = new HashMap<>();
@@ -111,7 +123,7 @@ public ResponseEntity<Map<String, Object>> reverse(
 
         return ResponseEntity.ok()
             .header("X-Result-Count", String.valueOf(results.size()))
-            .header("Cache-Control", "no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0") // No caching
+            .header("Cache-Control", "max-age=86400")
             .body(response);
     }
 
@@ -123,7 +135,7 @@ public ResponseEntity<Map<String, Object>> health() {
         Map<String, Object> response = new HashMap<>();
         response.put("status", "ok");
         response.put("service", "paikka");
-        response.put("metadata", metadataService.getMetadata()); // Include metadata
+        response.put("metadata", metadataService.getMetadata());
         return ResponseEntity.ok()
             .header("Cache-Control", "no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0") // No caching
             .body(response);

src/main/java/com/dedicatedcode/paikka/service/importer/BoundaryImportStatistics.java

@@ -192,7 +192,8 @@ public String getMemoryStats() {
     }
 
     public void startProgressReporter() {
-        boolean isTty = System.console() != null;
+        boolean isTty = System.console() != null && System.console().isTerminal();
+        long sleepMillis = isTty ? 1000 : 5000;
 
         Thread.ofPlatform().daemon().start(() -> {
             while (isRunning()) {
@@ -237,7 +238,7 @@ public void startProgressReporter() {
                 }
 
                 try {
-                    Thread.sleep(1000);
+                    Thread.sleep(sleepMillis);
                 } catch (InterruptedException e) {
                     break;
                 }

src/main/java/com/dedicatedcode/paikka/service/importer/ImportService.java

@@ -793,9 +793,9 @@ private void processAdministrativeBoundariesFromIndex(RocksDB relIndexDb, RocksD
                     RelRec rec = decodeRelRec(val, relId);
 
                     semaphore.acquire();
-                    stats.incrementActiveThreads();
                     ecs.submit(() -> {
                         try {
+                            stats.incrementActiveThreads();
                             org.locationtech.jts.geom.Geometry geometry = buildGeometryFromRelRec(rec, nodeCache, wayIndexDb, stats);
                             if (geometry == null) return null;
 
@@ -908,11 +908,12 @@ private void processBuildingBoundariesFromIndex(RocksDB poiIndexDb, RocksDB node
 
                         if (currentBatchIds.size() >= batchSize) {
                             semaphore.acquire();
-                            stats.incrementActiveThreads();
                             List<Long> idsToProcess = new ArrayList<>(currentBatchIds);
                             List<PoiIndexRec> recsToProcess = new ArrayList<>(currentBatchRecs);
                             ecs.submit(() -> {
                                 try {
+                                    stats.incrementActiveThreads();
+
                                     List<BuildingData> results = new ArrayList<>();
                                     for (int i = 0; i < idsToProcess.size(); i++) {
                                         long wayIdInner = idsToProcess.get(i);
@@ -972,11 +973,11 @@ private void processBuildingBoundariesFromIndex(RocksDB poiIndexDb, RocksDB node
 
             if (!currentBatchIds.isEmpty()) {
                 semaphore.acquire();
-                stats.incrementActiveThreads();
                 List<Long> idsToProcess = new ArrayList<>(currentBatchIds);
                 List<PoiIndexRec> recsToProcess = new ArrayList<>(currentBatchRecs);
                 ecs.submit(() -> {
                     try {
+                        stats.incrementActiveThreads();
                         List<BuildingData> results = new ArrayList<>();
                         for (int i = 0; i < idsToProcess.size(); i++) {
                             long wayIdInner = idsToProcess.get(i);
@@ -1160,6 +1161,7 @@ private int serializePoiData(FlatBufferBuilder builder, PoiData poi) {
     private void compactBuildingShards(RocksDB appendDb, RocksDB buildingsDb, ImportStatistics stats) {
         stats.setCompactionStartTime(System.currentTimeMillis());
         stats.setCompactionEntriesTotal(buildingSequence.get());
+        stats.resetCompactionProgress();
 
         Building reusableBuilding = new Building();
         Geometry reusableGeom = new Geometry();