Skip to content

Add resolution to region meta data #51

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dgraf-gh merged 2 commits into from
Aug 2, 2026
Merged

Add resolution to region meta data #51

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c1805cc
test: add comprehensive integration tests for StandaloneBoundaryImpor…
dgraf-gh Aug 2, 2026
80b276b
refactor: enhance progress reporting with detailed phase durations an…
dgraf-gh Aug 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,3 +39,4 @@ build/
/.idea/
/data-boundaries/
/.env
/tmp/
11 changes: 11 additions & 0 deletions .run/PaikkaApplication (Import Schleswig-Holstein).run.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="PaikkaApplication (Import Schleswig-Holstein)" type="SpringBootApplicationConfigurationType" factoryName="Spring Boot">
<option name="ACTIVE_PROFILES" value="dev" />
<module name="paikka" />
<option name="PROGRAM_PARAMETERS" value="--import --data-dir data/ schleswig-holstein-filtered.pbf" />
<option name="SPRING_BOOT_MAIN_CLASS" value="com.dedicatedcode.paikka.PaikkaApplication" />
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
</component>
86 changes: 63 additions & 23 deletions src/main/java/com/dedicatedcode/paikka/service/importer/BoundaryImportStatistics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,16 @@ public String toString() {
private volatile long phaseStartTime = System.currentTimeMillis();
private long totalTime;

private final int TOTAL_STEPS = 2;
private final int TOTAL_STEPS = 3;
private int currentStep = 0;

private volatile long phase1Duration;
private volatile long phase2Duration;
private volatile long h3OsmSizeBytes;
private volatile long regionMetaSizeBytes;
private volatile long regionGeomSizeBytes;
private volatile long osmNamesSizeBytes;

public long getNodesCached() {
return nodesCached.get();
}
Expand Down Expand Up @@ -127,8 +134,14 @@ public String getCurrentPhase() {
}

public void setCurrentPhase(int step, String phase) {
long now = System.currentTimeMillis();
if (this.currentStep == 1 && step != 1) {
this.phase1Duration = now - this.phaseStartTime;
} else if (this.currentStep == 2 && step != 2) {
this.phase2Duration = now - this.phaseStartTime;
}
this.currentPhase = phase;
this.phaseStartTime = System.currentTimeMillis();
this.phaseStartTime = now;
this.currentStep = step;
}

Expand Down Expand Up @@ -186,9 +199,16 @@ public long getErrorsTotal() {

public String getMemoryStats() {
Runtime r = Runtime.getRuntime();
long used = (r.totalMemory() - r.freeMemory()) / 1024 / 1024 / 1024;
long max = r.maxMemory() / 1024 / 1024 / 1024;
return String.format("%dGB/%dGB", used, max);
long usedBytes = r.totalMemory() - r.freeMemory();
long maxBytes = r.maxMemory();
return String.format("%.1fG/%.1fG", usedBytes / (1024.0 * 1024.0 * 1024.0), maxBytes / (1024.0 * 1024.0 * 1024.0));
}

public void setOutputSizes(long h3OsmBytes, long regionMetaBytes, long regionGeomBytes, long osmNamesBytes) {
this.h3OsmSizeBytes = h3OsmBytes;
this.regionMetaSizeBytes = regionMetaBytes;
this.regionGeomSizeBytes = regionGeomBytes;
this.osmNamesSizeBytes = osmNamesBytes;
}

public void startProgressReporter() {
Expand All @@ -210,20 +230,23 @@ public void startProgressReporter() {

sb.append(String.format("\033[1;90m[%d/%d]\033[0m ", currentStep, TOTAL_STEPS));

if (phase.contains("1.1")) {
if (phase.contains("Caching")) {
long nodesPerSec = phaseSeconds > 0 ? (long) (getNodesCached() / phaseSeconds) : 0;
sb.append(String.format("\033[1;36m[%s]\033[0m \033[1mCaching Nodes & Ways\033[0m", formatTime(elapsed)));
sb.append(String.format(" │ \033[32mNodes:\033[0m %s \033[33m(%s/s)\033[0m",
formatCompactNumber(getNodesCached()), formatCompactRate(nodesPerSec)));
sb.append(String.format(" │ \033[34mWays:\033[0m %s", formatCompactNumber(getWaysCached())));
} else if (phase.contains("2.1")) {
} else if (phase.contains("Processing")) {
long relsPerSec = phaseSeconds > 0 ? (long) (getRelationsProcessed() / phaseSeconds) : 0;
double percentage = getRelationsFound() > 0 ? (double) getRelationsProcessed() / getRelationsFound() * 100.0 : 0.0;
sb.append(String.format("\033[1;36m[%s]\033[0m \033[1mProcessing Relations & H3\033[0m", formatTime(elapsed)));
sb.append(String.format(" │ \033[32mRelations:\033[0m %s/%s \033[33m(%s/s)\033[0m",
formatCompactNumber(getRelationsProcessed()), formatCompactNumber(getRelationsFound()), formatCompactRate(relsPerSec)));
sb.append(String.format(" │ \033[35mProgress:\033[0m %.2f%%", percentage));
sb.append(String.format(" │ \033[36mH3 Cells:\033[0m %s", formatCompactNumber(getH3CellsGenerated())));
if (getErrorsTotal() > 0) {
sb.append(String.format(" │ \033[31mErrors:\033[0m %d", getErrorsTotal()));
}
} else {
sb.append(String.format("\033[1;36m[%s]\033[0m %s", formatTime(elapsed), phase));
}
Expand All @@ -248,33 +271,43 @@ public void startProgressReporter() {
}

public void printFinalStatistics() {
System.out.println("\n\033[1;36m" + "═".repeat(80) + "\n" + centerText("🎯 BOUNDARY IMPORT STATISTICS") + "\n" + "═".repeat(80) + "\033[0m");
System.out.println("\n\033[1;36m" + "═".repeat(80) + "\n" + centerText("BOUNDARY IMPORT STATISTICS") + "\n" + "═".repeat(80) + "\033[0m");

long totalTime = Math.max(1, getTotalTime());
double totalSeconds = totalTime / 1000.0;
double phase1Seconds = Math.max(0.001, phase1Duration / 1000.0);
double phase2Seconds = Math.max(0.001, phase2Duration / 1000.0);

System.out.printf("\n\033[1;37m⏱️ Total Import Time:\033[0m \033[1;33m%s\033[0m%n%n", formatTime(getTotalTime()));
System.out.printf("\n\033[1;37mTotal Import Time:\033[0m \033[1;33m%s\033[0m%n%n", formatTime(getTotalTime()));

System.out.println("\033[1;37m📊 Processing Summary:\033[0m");
System.out.println("┌────────────────────┬─────────────────┬─────────────────┐");
System.out.println("│ \033[1mEntity Type\033[0m │ \033[1mTotal Count\033[0m │ \033[1mAvg Speed\033[0m │");
System.out.println("├────────────────────┼─────────────────┼─────────────────┤");
System.out.printf("│ \033[32mNodes Cached\033[0m │ %15s │ %13s/s │%n",
System.out.println("\033[1;37mProcessing Summary:\033[0m");
System.out.println("┌──────────────────────┬─────────────────┬─────────────────┐");
System.out.println("│ \033[1mEntity Type\033[0m │ \033[1mTotal Count\033[0m │ \033[1mAvg Speed\033[0m │");
System.out.println("├──────────────────────┼─────────────────┼─────────────────┤");
System.out.printf("│ \033[32mNodes Cached\033[0m │ %15s │ %13s/s │%n",
formatCompactNumber(getNodesCached()),
formatCompactNumber((long) (getNodesCached() / totalSeconds)));
System.out.printf("│ \033[34mWays Cached\033[0m │ %15s │ %13s/s │%n",
formatCompactNumber((long) (getNodesCached() / phase1Seconds)));
System.out.printf("│ \033[34mWays Cached\033[0m │ %15s │ %13s/s │%n",
formatCompactNumber(getWaysCached()),
formatCompactNumber((long) (getWaysCached() / totalSeconds)));
System.out.printf("│ \033[35mRelations Found\033[0m │ %15s │ %13s/s │%n",
formatCompactNumber((long) (getWaysCached() / phase1Seconds)));
System.out.printf("│ \033[35mRelations Found\033[0m │ %15s │ %13s/s │%n",
formatCompactNumber(getRelationsFound()),
formatCompactNumber((long) (getRelationsFound() / totalSeconds)));
System.out.printf("│ \033[36mRelations Processed\033[0m│ %15s │ %13s/s │%n",
System.out.printf("│ \033[36mRelations Processed\033[0m │ %15s │ %13s/s │%n",
formatCompactNumber(getRelationsProcessed()),
formatCompactNumber((long) (getRelationsProcessed() / totalSeconds)));
System.out.printf("│ \033[33mH3 Cells Generated\033[0m │ %15s │ %13s/s │%n",
formatCompactNumber((long) (getRelationsProcessed() / phase2Seconds)));
System.out.printf("│ \033[33mH3 Cells Generated\033[0m │ %15s │ %13s/s │%n",
formatCompactNumber(getH3CellsGenerated()),
formatCompactNumber((long) (getH3CellsGenerated() / totalSeconds)));
System.out.println("└────────────────────┴─────────────────┴─────────────────┘");
formatCompactNumber((long) (getH3CellsGenerated() / phase2Seconds)));
System.out.println("└──────────────────────┴─────────────────┴─────────────────┘");

if (h3OsmSizeBytes > 0) {
System.out.println("\n\033[1;37mOutput Database Sizes:\033[0m");
System.out.printf(" \033[36mh3_to_osm:\033[0m %s%n", formatSize(h3OsmSizeBytes));
System.out.printf(" \033[36mregion_metadata:\033[0m %s%n", formatSize(regionMetaSizeBytes));
System.out.printf(" \033[36mregion_geometry:\033[0m %s%n", formatSize(regionGeomSizeBytes));
System.out.printf(" \033[36mosm_names.tsv:\033[0m %s%n", formatSize(osmNamesSizeBytes));
}

System.out.println();
}
Expand Down Expand Up @@ -331,6 +364,13 @@ private String formatCompactRate(long n) {
return String.format("%.1fM", n / 1_000_000.0);
}

private String formatSize(long bytes) {
if (bytes < 1024) return bytes + " B";
if (bytes < 1024 * 1024) return String.format("%.1f KB", bytes / 1024.0);
if (bytes < 1024L * 1024 * 1024) return String.format("%.1f MB", bytes / (1024.0 * 1024.0));
return String.format("%.2f GB", bytes / (1024.0 * 1024.0 * 1024.0));
}

private String centerText(String text) {
int pad = (80 - text.length()) / 2;
return " ".repeat(Math.max(0, pad)) + text;
Expand Down
Loading
Loading