Backup sync using importer (Post-Launch)

CHANGELOG.md

@@ -4,14 +4,17 @@ All notable changes to this project will be documented in this file.
 
 The changelog format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
-## [0.4.3] - NEXT
+## [0.4.4] - NEXT
 
 **Milestone**: Hippopotamus(0.10.0.7)
 
 | Package          | Version | Link                                                               |
 | ---------------- | ------- | ------------------------------------------------------------------ |
 | Symbol Bootstrap | v0.4.4  | [symbol-bootstrap](https://www.npmjs.com/package/symbol-bootstrap) |
 
+-   Added `--backupSync` to `config` and `start` commands. It downloads a backup with the Mongo and RocksDb databases for faster synchronization.
+-   Added `backup` command. The command backups the Mongo and RocksDb data folder into a Zip file that can be used for `--backupSync` feature.
+-   Splitting `userconfig` into `server-config` and `broker-config` for each service. 
 -   Added `--ready` to `link` and `enrolRewardProgram` commands.
 -   Fixed how seed is copied to node folders when `--upgrade` and `resetData` are used
 -   Moved Reward Program Agent to its own service/container in docker-compose.yml.

README.md

@@ -265,6 +265,7 @@ General users should install this tool like any other node module.
 # Command Topics
 
 * [`symbol-bootstrap autocomplete`](docs/autocomplete.md) - display autocomplete installation instructions
+* [`symbol-bootstrap backup`](docs/backup.md) - The command backs up the Mongo and RocksDb data folder into a Zip file that can then be used by the `--backupSync` feature. Bootstrap compose services must be stopped before calling this command.
 * [`symbol-bootstrap clean`](docs/clean.md) - It removes the target folder deleting the generated configuration and data
 * [`symbol-bootstrap compose`](docs/compose.md) - It generates the `docker-compose.yml` file from the configured network.
 * [`symbol-bootstrap config`](docs/config.md) - Command used to set up the configuration files and the nemesis block for the current network

cmds/backup-full-dual.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+rm -rf testnet-backup
+mkdir -p testnet-backup
+cp -rf target/databases/db testnet-backup/mongo
+cp -rf target/nodes/api-node/data testnet-backup/data
+rm -rf testnet-backup/data/spool
+
+cd testnet-backup
+zip -r testnet-full-backup.zip *
+cd ..
+
+# push testnet-backup/testnet-full-backup.zip into S3

cmds/backup-full-sync-testnet-backup.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+rm -rf backup-sync/testnet-backup
+mkdir -p backup-sync/testnet-backup
+cp -rf target/testnet-dual/databases/db backup-sync/testnet-backup/mongo
+cp -rf target/testnet-dual/nodes/api-node/data backup-sync/testnet-backup/data
+rm -rf backup-sync/testnet-backup/data/spool
+
+cd backup-sync/testnet-backup
+zip -r testnet-local-full-backup.zip *
+cd ../..

cmds/backup-minimal-sync-testnet-backup.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e -x
+rm -rf backup-sync/testnet-backup
+mkdir -p backup-sync/testnet-backup/data
+cp -rf target/testnet-dual/nodes/api-node/data/0* backup-sync/testnet-backup/data
+cp -rf target/testnet-dual/nodes/api-node/data/proof.index.dat backup-sync/testnet-backup/data
+cp -rf target/testnet-dual/nodes/api-node/data/index.dat backup-sync/testnet-backup/data
+rm -rf backup-sync/testnet-backup/data/spool
+touch backup-sync/testnet-backup/data/server.lock # force docker compose to run a recover
+
+cd backup-sync/testnet-backup
+zip -r testnet-local-minimal-backup.zip *
+cd ../..

cmds/backup-testnet-dual.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -e
+symbol-bootstrap backup -t target/testnet-dual --destinationFile ./backup-sync/testnet-local-backup.zip $1 $2 $3

cmds/start-testnet-dual.sh

@@ -1,3 +1,16 @@
 #!/bin/bash
 set -e
-symbol-bootstrap start -p testnet -a dual -t target/testnet-dual $1 $2 $3
+#rm -rf target/testnet-dual/databases/db
+#mkdir -p target/testnet-dual/databases/db
+#rm -rf target/testnet-dual/nodes/api-node/data/recovery.lock
+#rm -rf target/testnet-dual/nodes/api-node/data/broker.started
+#rm -rf target/testnet-dual/nodes/api-node/logs
+#mkdir -p target/testnet-dual/nodes/api-node/logs
+#
+##rm -rf target/testnet-dual/nodes/api-node/data
+##cp target/data target/testnet-dual/nodes/api-node  -r
+#
+#rm -rf target/testnet-dual/nodes/api-node/data/state
+#rm -rf target/testnet-dual/nodes/api-node/data/statedb
+
+symbol-bootstrap start -p testnet -a dual -t target/testnet-dual -c  test/testnet-custom-preset.yml $1 $2 $3

cmds/start-testnet-voting.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 set -e
-symbol-bootstrap start -p testnet -a dual -t target/testnet -c test/voting_preset.yml  $1
+symbol-bootstrap start -p testnet -a dual -t target/testnet -c test/voting_preset.yml  $1 $2 $3

config/docker/server/startBroker.sh.mustache

@@ -3,17 +3,37 @@ set -e
 
 name=$1
 mode=$2
+application="broker"
+config="./$application-config"
 
-echo "RUNNING startBroker.sh $name $mode"
+echo "RUNNING $application $name $mode"
+export LD_LIBRARY_PATH=/usr/catapult/lib:/usr/catapult/deps
 
 ulimit -c unlimited
 
 if [ "$mode" == "DEBUG" ]; then
   echo "Setting up core dump..."
   mkdir -p ./logs
-  echo './logs/broker.%e.%t' > /proc/sys/kernel/core_pattern
+  echo "./logs/$application.%e.%t" >/proc/sys/kernel/core_pattern
 fi
 
-touch ./data/broker.started
+if [ -e "{{{dataDirectory}}}/$application-importer.run" ]; then
+  echo "!!!! Have Importer file present, going to run importer in $application mode...."
+  {{{catapultAppFolder}}}/bin/catapult.importer "$config"
+  echo "!!!! Finished running importer, should be moving on to start $application..."
+fi
+
+rm -rf "{{{dataDirectory}}}/$application-importer.run"
+
+if [ -e "{{{dataDirectory}}}/$application.lock" ]; then
+  echo "!!!! Have lock file present, going to run recovery in $application mode...."
+  {{{catapultAppFolder}}}/bin/catapult.recovery "$config"
+  echo "!!!! Finished running recovery, should be moving on to start $application..."
+fi
+
+rm -f "{{{dataDirectory}}}/$application.lock"
+rm -f "{{{dataDirectory}}}/recovery.lock"
+
+touch "{{{dataDirectory}}}/$application.started"
 
-exec env LD_LIBRARY_PATH={{{catapultAppFolder}}}/lib:{{{catapultAppFolder}}}/deps {{{catapultAppFolder}}}/bin/catapult.broker ./userconfig
+{{{catapultAppFolder}}}/bin/catapult.broker "$config"

config/docker/server/startServer.sh.mustache

@@ -3,15 +3,37 @@ set -e
 
 name=$1
 mode=$2
+application="server"
+config="./$application-config"
 
-echo "RUNNING startServer.sh $name $mode"
+echo "RUNNING $application $name $mode"
+export LD_LIBRARY_PATH=/usr/catapult/lib:/usr/catapult/deps
 
 ulimit -c unlimited
 
 if [ "$mode" == "DEBUG" ]; then
   echo "Setting up core dump..."
   mkdir -p ./logs
-  echo './logs/server.%e.%t' >/proc/sys/kernel/core_pattern
+  echo "./logs/$application.%e.%t" >/proc/sys/kernel/core_pattern
 fi
 
-exec env LD_LIBRARY_PATH={{{catapultAppFolder}}}/lib:{{{catapultAppFolder}}}/deps {{{catapultAppFolder}}}/bin/catapult.server ./userconfig
+if [ -e "{{{dataDirectory}}}/$application-importer.run" ]; then
+  echo "!!!! Have Importer file present, going to run importer in $application mode...."
+  {{{catapultAppFolder}}}/bin/catapult.importer "$config"
+  echo "!!!! Finished running importer, should be moving on to start $application..."
+fi
+
+rm -rf "{{{dataDirectory}}}/$application-importer.run"
+
+if [ -e "{{{dataDirectory}}}/$application.lock" ]; then
+  echo "!!!! Have lock file present, going to run recovery in $application mode...."
+  {{{catapultAppFolder}}}/bin/catapult.recovery "$config"
+  echo "!!!! Finished running recovery, should be moving on to start $application..."
+fi
+
+rm -f "{{{dataDirectory}}}/$application.lock"
+rm -f "{{{dataDirectory}}}/recovery.lock"
+
+touch "{{{dataDirectory}}}/$application.started"
+
+{{{catapultAppFolder}}}/bin/catapult.server "$config"

config/docker/server/waitWhile.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+file=$1
+while [ -f $file ] ;
+do
+      sleep 0.5
+done

config/node/resources/config-extensions-recovery.properties.mustache

@@ -1,8 +1,9 @@
 [extensions]
 
 # addressextraction must be first because mongo and zeromq depend on extracted addresses
-extension.addressextraction = {{addressextraction}}
-extension.mongo = {{mongo}}
-extension.zeromq = {{zeromq}}
+extension.addressextraction = {{addressextractionRecovery}}
+extension.mongo = {{mongoRecovery}}
+extension.zeromq = {{zeromqRecovery}}
 
-extension.hashcache = true
+extension.filespooling = {{filespoolingRecovery}}
+extension.hashcache = {{hashcacheRecovery}}

docs/backup.md

@@ -0,0 +1,39 @@
+`symbol-bootstrap backup`
+=========================
+
+The command backs up the Mongo and RocksDb data folder into a Zip file that can then be used by the `--backupSync` feature. Bootstrap compose services must be stopped before calling this command.
+
+Note: This command is designed for NGL to be used when running public main or public test networks. It's not backing up any node specific information.
+
+* [`symbol-bootstrap backup`](#symbol-bootstrap-backup)
+
+## `symbol-bootstrap backup`
+
+The command backs up the Mongo and RocksDb data folder into a Zip file that can then be used by the `--backupSync` feature. Bootstrap compose services must be stopped before calling this command.
+
+```
+USAGE
+  $ symbol-bootstrap backup
+
+OPTIONS
+  -h, --help                         It shows the help of this command.
+  -t, --target=target                [default: target] The target folder where the symbol-bootstrap network is generated
+
+  --destinationFile=destinationFile  The file location where the backup zip file will be created. Default destination is
+                                     target/backup.zip.
+
+  --[no-]fullBackup                  If the restore/backup to be performed is a full backup (RocksDB + Mongo) or partial
+                                     backup (RocksDB + Catapult's Importer)
+
+  --nodeName=nodeName                The dual/api node name to be used to backup the data. If not provided, the first
+                                     configured api/dual node would be used.
+
+DESCRIPTION
+  Note: This command is designed for NGL to be used when running public main or public test networks. It's not backing 
+  up any node specific information.
+
+EXAMPLE
+  $ symbol-bootstrap backup
+```
+
+_See code: [src/commands/backup.ts](https://github.com/nemtech/symbol-bootstrap/blob/v0.4.4/src/commands/backup.ts)_

docs/config.md

@@ -30,6 +30,15 @@ OPTIONS
   -u, --user=user                   [default: current] User used to run docker images when creating configuration files
                                     like certificates or nemesis block. "current" means the current user.
 
+  --backupSync                      It downloads a backup with the RocksDb and Mongo databases for faster
+                                    synchronization.
+
+                                    The location of the backup can be found and changed using the 'backupSyncLocation'
+                                    preset configuration. This configuration allows local files and remote URLs
+
+  --[no-]fullBackup                 If the restore/backup to be performed is a full backup (RocksDB + Mongo) or partial
+                                    backup (RocksDB + Catapult's Importer)
+
   --pullImages                      It pulls the utility images from DockerHub when running the configuration. It only
                                     affects alpha/dev docker images.
 

docs/start.md

@@ -46,6 +46,16 @@ OPTIONS
   --args=args
       Add extra arguments to the docker-compose up command. Check out https://docs.docker.com/compose/reference/up.
 
+  --backupSync
+      It downloads a backup with the RocksDb and Mongo databases for faster synchronization.
+
+      The location of the backup can be found and changed using the 'backupSyncLocation' preset configuration. This 
+      configuration allows local files and remote URLs
+
+  --[no-]fullBackup
+      If the restore/backup to be performed is a full backup (RocksDB + Mongo) or partial backup (RocksDB + Catapult's 
+      Importer)
+
   --healthCheck
       It checks if the services created with docker compose are up and running.