Add planet-dump-ng for planet file processing

Author: Rub21

images/backup-restore/start.sh

@@ -35,25 +35,26 @@ cloudStorageOps() {
 }
 
 backupDB() {
-	local LOCAL_BACKUP_FILE=${BACKUP_CLOUD_FILE}.sql.gz
-	local CLOUD_BACKUP_FILE="${BACKUP_CLOUD_FOLDER}/${BACKUP_CLOUD_FILE}.sql.gz"
+	local LOCAL_BACKUP_FILE=${BACKUP_CLOUD_FILE}.dump
+	local CLOUD_BACKUP_FILE="${BACKUP_CLOUD_FOLDER}/${BACKUP_CLOUD_FILE}.dump"
 	if [ "$SET_DATE_AT_NAME" == "true" ]; then
 		local CURRENT_DATE=$(date '+%Y%m%d-%H%M')
-		LOCAL_BACKUP_FILE="${BACKUP_CLOUD_FILE}-${CURRENT_DATE}.sql.gz"
-		CLOUD_BACKUP_FILE="${BACKUP_CLOUD_FOLDER}/${BACKUP_CLOUD_FILE}-${CURRENT_DATE}.sql.gz"
+		LOCAL_BACKUP_FILE="${BACKUP_CLOUD_FILE}-${CURRENT_DATE}.dump"
+		CLOUD_BACKUP_FILE="${BACKUP_CLOUD_FOLDER}/${BACKUP_CLOUD_FILE}-${CURRENT_DATE}.dump"
 	fi
 
 	# Backup database with max compression
 	echo "Backing up DB ${POSTGRES_DB} into ${LOCAL_BACKUP_FILE}"
-	pg_dump -h ${POSTGRES_HOST} -U ${POSTGRES_USER} ${POSTGRES_DB} | gzip -9 >${LOCAL_BACKUP_FILE}
+	# pg_dump -h ${POSTGRES_HOST} -U ${POSTGRES_USER} ${POSTGRES_DB} | gzip -9 >${LOCAL_BACKUP_FILE}
+	pg_dump -h ${POSTGRES_HOST} -U ${POSTGRES_USER} -Fc -f ${LOCAL_BACKUP_FILE} ${POSTGRES_DB}
 
 	# Handle cloud storage based on the provider
 	cloudStorageOps "${LOCAL_BACKUP_FILE}" "${CLOUD_BACKUP_FILE}"
 }
 
 restoreDB() {
 	local CURRENT_DATE=$(date '+%Y%m%d-%H%M')
-	local RESTORE_FILE="backup.sql.gz"
+	local RESTORE_FILE="backup.dump"
 	local LOG_RESULT_FILE="restore_results-${CURRENT_DATE}.log"
 	local flag=true
 
@@ -62,7 +63,7 @@ restoreDB() {
 		flag=false
 		wget -O ${RESTORE_FILE} ${RESTORE_URL_FILE}
 		echo "Restoring ${RESTORE_URL_FILE} in ${POSTGRES_DB}"
-		gunzip -c <${RESTORE_FILE} | psql -h ${POSTGRES_HOST} -U ${POSTGRES_USER} -d ${POSTGRES_DB} | tee ${LOG_RESULT_FILE}
+		pg_restore -h ${POSTGRES_HOST} -U ${POSTGRES_USER} -d ${POSTGRES_DB} --create --no-owner ${RESTORE_FILE} | tee ${LOG_RESULT_FILE}
 		# aws s3 cp ${LOG_RESULT_FILE} s3://${AWS_S3_BUCKET}/${LOG_RESULT_FILE}
 		echo "Import data to ${POSTGRES_DB} has finished ..."
 	done

images/osm-processor/Dockerfile

@@ -7,17 +7,45 @@ RUN set -ex \
     && apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install \
     -y --no-install-recommends \
-        "osmosis" \
-        "osmium-tool" \
+        osmosis \
+        osmium-tool \
         # Cloud provider CLIs
-        "awscli" \
-        "gsutil" \
-        "azure-cli" \
+        awscli \
+        gsutil \
+        azure-cli \
         # PostgreSQL client
-        "postgresql-client" \
+        postgresql-client \
         # Other useful packages
-        "rsync" \
-        "pyosmium" \
-        "tmux" \
-        "zsh" \
+        rsync \
+        pyosmium \
+        tmux \
+        zsh  \
+        git \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install planet-dump-ng
+RUN set -ex \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        build-essential \
+        automake \
+        autoconf \
+        libxml2-dev \
+        libboost-dev \
+        libboost-program-options-dev \
+        libboost-date-time-dev \
+        libboost-filesystem-dev \
+        libboost-thread-dev \
+        libboost-iostreams-dev \
+        libosmpbf-dev \
+        osmpbf-bin \
+        libprotobuf-dev \
+        pkg-config \
+        git \
+    && git clone https://github.com/zerebubuth/planet-dump-ng.git /opt/planet-dump-ng \
+    && cd /opt/planet-dump-ng \
+    && ./autogen.sh \
+    && ./configure \
+    && make \
+    && ln -s /opt/planet-dump-ng/planet-dump-ng /usr/local/bin/planet-dump-ng \
     && rm -rf /var/lib/apt/lists/*