diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fdcfe1d4..73702cfe4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The changelog format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## [0.4.3] - NEXT +## [0.4.4] - NEXT **Milestone**: Hippopotamus(0.10.0.7) @@ -12,6 +12,9 @@ The changelog format is based on [Keep a Changelog](https://keepachangelog.com/e | ---------------- | ------- | ------------------------------------------------------------------ | | Symbol Bootstrap | v0.4.4 | [symbol-bootstrap](https://www.npmjs.com/package/symbol-bootstrap) | +- Added `--backupSync` to `config` and `start` commands. It downloads a backup with the Mongo and RocksDb databases for faster synchronization. +- Added `backup` command. The command backups the Mongo and RocksDb data folder into a Zip file that can be used for `--backupSync` feature. +- Splitting `userconfig` into `server-config` and `broker-config` for each service. - Added `--ready` to `link` and `enrolRewardProgram` commands. - Fixed how seed is copied to node folders when `--upgrade` and `resetData` are used - Moved Reward Program Agent to its own service/container in docker-compose.yml. diff --git a/README.md b/README.md index 1f25d784b..5aacfa792 100644 --- a/README.md +++ b/README.md @@ -265,6 +265,7 @@ General users should install this tool like any other node module. # Command Topics * [`symbol-bootstrap autocomplete`](docs/autocomplete.md) - display autocomplete installation instructions +* [`symbol-bootstrap backup`](docs/backup.md) - The command backs up the Mongo and RocksDb data folder into a Zip file that can then be used by the `--backupSync` feature. Bootstrap compose services must be stopped before calling this command. * [`symbol-bootstrap clean`](docs/clean.md) - It removes the target folder deleting the generated configuration and data * [`symbol-bootstrap compose`](docs/compose.md) - It generates the `docker-compose.yml` file from the configured network. * [`symbol-bootstrap config`](docs/config.md) - Command used to set up the configuration files and the nemesis block for the current network diff --git a/cmds/backup-full-dual.sh b/cmds/backup-full-dual.sh new file mode 100644 index 000000000..1f7173b9b --- /dev/null +++ b/cmds/backup-full-dual.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e +rm -rf testnet-backup +mkdir -p testnet-backup +cp -rf target/databases/db testnet-backup/mongo +cp -rf target/nodes/api-node/data testnet-backup/data +rm -rf testnet-backup/data/spool + +cd testnet-backup +zip -r testnet-full-backup.zip * +cd .. + +# push testnet-backup/testnet-full-backup.zip into S3 diff --git a/cmds/backup-full-sync-testnet-backup.sh b/cmds/backup-full-sync-testnet-backup.sh new file mode 100755 index 000000000..be5714510 --- /dev/null +++ b/cmds/backup-full-sync-testnet-backup.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e +rm -rf backup-sync/testnet-backup +mkdir -p backup-sync/testnet-backup +cp -rf target/testnet-dual/databases/db backup-sync/testnet-backup/mongo +cp -rf target/testnet-dual/nodes/api-node/data backup-sync/testnet-backup/data +rm -rf backup-sync/testnet-backup/data/spool + +cd backup-sync/testnet-backup +zip -r testnet-local-full-backup.zip * +cd ../.. diff --git a/cmds/backup-minimal-sync-testnet-backup.sh b/cmds/backup-minimal-sync-testnet-backup.sh new file mode 100755 index 000000000..f7d535d10 --- /dev/null +++ b/cmds/backup-minimal-sync-testnet-backup.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e -x +rm -rf backup-sync/testnet-backup +mkdir -p backup-sync/testnet-backup/data +cp -rf target/testnet-dual/nodes/api-node/data/0* backup-sync/testnet-backup/data +cp -rf target/testnet-dual/nodes/api-node/data/proof.index.dat backup-sync/testnet-backup/data +cp -rf target/testnet-dual/nodes/api-node/data/index.dat backup-sync/testnet-backup/data +rm -rf backup-sync/testnet-backup/data/spool +touch backup-sync/testnet-backup/data/server.lock # force docker compose to run a recover + +cd backup-sync/testnet-backup +zip -r testnet-local-minimal-backup.zip * +cd ../.. diff --git a/cmds/backup-testnet-dual.sh b/cmds/backup-testnet-dual.sh new file mode 100755 index 000000000..16720dd6f --- /dev/null +++ b/cmds/backup-testnet-dual.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -e +symbol-bootstrap backup -t target/testnet-dual --destinationFile ./backup-sync/testnet-local-backup.zip $1 $2 $3 diff --git a/cmds/start-testnet-dual.sh b/cmds/start-testnet-dual.sh index c17256f62..03c4395b5 100755 --- a/cmds/start-testnet-dual.sh +++ b/cmds/start-testnet-dual.sh @@ -1,3 +1,16 @@ #!/bin/bash set -e -symbol-bootstrap start -p testnet -a dual -t target/testnet-dual $1 $2 $3 +#rm -rf target/testnet-dual/databases/db +#mkdir -p target/testnet-dual/databases/db +#rm -rf target/testnet-dual/nodes/api-node/data/recovery.lock +#rm -rf target/testnet-dual/nodes/api-node/data/broker.started +#rm -rf target/testnet-dual/nodes/api-node/logs +#mkdir -p target/testnet-dual/nodes/api-node/logs +# +##rm -rf target/testnet-dual/nodes/api-node/data +##cp target/data target/testnet-dual/nodes/api-node -r +# +#rm -rf target/testnet-dual/nodes/api-node/data/state +#rm -rf target/testnet-dual/nodes/api-node/data/statedb + +symbol-bootstrap start -p testnet -a dual -t target/testnet-dual -c test/testnet-custom-preset.yml $1 $2 $3 diff --git a/cmds/start-testnet-voting.sh b/cmds/start-testnet-voting.sh index 776a78d05..e08d46305 100755 --- a/cmds/start-testnet-voting.sh +++ b/cmds/start-testnet-voting.sh @@ -1,3 +1,3 @@ #!/bin/bash set -e -symbol-bootstrap start -p testnet -a dual -t target/testnet -c test/voting_preset.yml $1 +symbol-bootstrap start -p testnet -a dual -t target/testnet -c test/voting_preset.yml $1 $2 $3 diff --git a/config/docker/server/runServerRecover.sh.mustache b/config/docker/server/runServerRecover.sh.mustache deleted file mode 100644 index 7c5196c18..000000000 --- a/config/docker/server/runServerRecover.sh.mustache +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -e - -name=$1 -echo "RUNNING runServerRecover.sh $name" - -ulimit -c unlimited -if [ -e "{{{dataDirectory}}}/broker.lock" ] || [ -e "{{{dataDirectory}}}/server.lock" ]; then - echo "!!!! Have lock file present, going to run recovery...." - exec env LD_LIBRARY_PATH={{{catapultAppFolder}}}/lib:{{{catapultAppFolder}}}/deps {{{catapultAppFolder}}}/bin/catapult.recovery ./userconfig - echo "!!!! Finished running recovery, should be moving on to start server..." -else - echo "!!!! DO NOT HAVE ANY LOCK FILE... There is no need to recover" - exit 0; -fi - diff --git a/config/docker/server/startBroker.sh.mustache b/config/docker/server/startBroker.sh.mustache index 8d3317563..ec086cdc8 100644 --- a/config/docker/server/startBroker.sh.mustache +++ b/config/docker/server/startBroker.sh.mustache @@ -3,17 +3,37 @@ set -e name=$1 mode=$2 +application="broker" +config="./$application-config" -echo "RUNNING startBroker.sh $name $mode" +echo "RUNNING $application $name $mode" +export LD_LIBRARY_PATH=/usr/catapult/lib:/usr/catapult/deps ulimit -c unlimited if [ "$mode" == "DEBUG" ]; then echo "Setting up core dump..." mkdir -p ./logs - echo './logs/broker.%e.%t' > /proc/sys/kernel/core_pattern + echo "./logs/$application.%e.%t" >/proc/sys/kernel/core_pattern fi -touch ./data/broker.started +if [ -e "{{{dataDirectory}}}/$application-importer.run" ]; then + echo "!!!! Have Importer file present, going to run importer in $application mode...." + {{{catapultAppFolder}}}/bin/catapult.importer "$config" + echo "!!!! Finished running importer, should be moving on to start $application..." +fi + +rm -rf "{{{dataDirectory}}}/$application-importer.run" + +if [ -e "{{{dataDirectory}}}/$application.lock" ]; then + echo "!!!! Have lock file present, going to run recovery in $application mode...." + {{{catapultAppFolder}}}/bin/catapult.recovery "$config" + echo "!!!! Finished running recovery, should be moving on to start $application..." +fi + +rm -f "{{{dataDirectory}}}/$application.lock" +rm -f "{{{dataDirectory}}}/recovery.lock" + +touch "{{{dataDirectory}}}/$application.started" -exec env LD_LIBRARY_PATH={{{catapultAppFolder}}}/lib:{{{catapultAppFolder}}}/deps {{{catapultAppFolder}}}/bin/catapult.broker ./userconfig +{{{catapultAppFolder}}}/bin/catapult.broker "$config" diff --git a/config/docker/server/startServer.sh.mustache b/config/docker/server/startServer.sh.mustache index d90ef5ca7..a46e0a87c 100644 --- a/config/docker/server/startServer.sh.mustache +++ b/config/docker/server/startServer.sh.mustache @@ -3,15 +3,37 @@ set -e name=$1 mode=$2 +application="server" +config="./$application-config" -echo "RUNNING startServer.sh $name $mode" +echo "RUNNING $application $name $mode" +export LD_LIBRARY_PATH=/usr/catapult/lib:/usr/catapult/deps ulimit -c unlimited if [ "$mode" == "DEBUG" ]; then echo "Setting up core dump..." mkdir -p ./logs - echo './logs/server.%e.%t' >/proc/sys/kernel/core_pattern + echo "./logs/$application.%e.%t" >/proc/sys/kernel/core_pattern fi -exec env LD_LIBRARY_PATH={{{catapultAppFolder}}}/lib:{{{catapultAppFolder}}}/deps {{{catapultAppFolder}}}/bin/catapult.server ./userconfig +if [ -e "{{{dataDirectory}}}/$application-importer.run" ]; then + echo "!!!! Have Importer file present, going to run importer in $application mode...." + {{{catapultAppFolder}}}/bin/catapult.importer "$config" + echo "!!!! Finished running importer, should be moving on to start $application..." +fi + +rm -rf "{{{dataDirectory}}}/$application-importer.run" + +if [ -e "{{{dataDirectory}}}/$application.lock" ]; then + echo "!!!! Have lock file present, going to run recovery in $application mode...." + {{{catapultAppFolder}}}/bin/catapult.recovery "$config" + echo "!!!! Finished running recovery, should be moving on to start $application..." +fi + +rm -f "{{{dataDirectory}}}/$application.lock" +rm -f "{{{dataDirectory}}}/recovery.lock" + +touch "{{{dataDirectory}}}/$application.started" + +{{{catapultAppFolder}}}/bin/catapult.server "$config" diff --git a/config/docker/server/waitWhile.sh b/config/docker/server/waitWhile.sh new file mode 100755 index 000000000..5f38e87da --- /dev/null +++ b/config/docker/server/waitWhile.sh @@ -0,0 +1,6 @@ +#!/bin/bash +file=$1 +while [ -f $file ] ; +do + sleep 0.5 +done diff --git a/config/node/resources/config-extensions-recovery.properties.mustache b/config/node/resources/config-extensions-recovery.properties.mustache index 0eca8c25c..f55be4d73 100644 --- a/config/node/resources/config-extensions-recovery.properties.mustache +++ b/config/node/resources/config-extensions-recovery.properties.mustache @@ -1,8 +1,9 @@ [extensions] # addressextraction must be first because mongo and zeromq depend on extracted addresses -extension.addressextraction = {{addressextraction}} -extension.mongo = {{mongo}} -extension.zeromq = {{zeromq}} +extension.addressextraction = {{addressextractionRecovery}} +extension.mongo = {{mongoRecovery}} +extension.zeromq = {{zeromqRecovery}} -extension.hashcache = true +extension.filespooling = {{filespoolingRecovery}} +extension.hashcache = {{hashcacheRecovery}} diff --git a/docs/backup.md b/docs/backup.md new file mode 100644 index 000000000..24bed0cb8 --- /dev/null +++ b/docs/backup.md @@ -0,0 +1,39 @@ +`symbol-bootstrap backup` +========================= + +The command backs up the Mongo and RocksDb data folder into a Zip file that can then be used by the `--backupSync` feature. Bootstrap compose services must be stopped before calling this command. + +Note: This command is designed for NGL to be used when running public main or public test networks. It's not backing up any node specific information. + +* [`symbol-bootstrap backup`](#symbol-bootstrap-backup) + +## `symbol-bootstrap backup` + +The command backs up the Mongo and RocksDb data folder into a Zip file that can then be used by the `--backupSync` feature. Bootstrap compose services must be stopped before calling this command. + +``` +USAGE + $ symbol-bootstrap backup + +OPTIONS + -h, --help It shows the help of this command. + -t, --target=target [default: target] The target folder where the symbol-bootstrap network is generated + + --destinationFile=destinationFile The file location where the backup zip file will be created. Default destination is + target/backup.zip. + + --[no-]fullBackup If the restore/backup to be performed is a full backup (RocksDB + Mongo) or partial + backup (RocksDB + Catapult's Importer) + + --nodeName=nodeName The dual/api node name to be used to backup the data. If not provided, the first + configured api/dual node would be used. + +DESCRIPTION + Note: This command is designed for NGL to be used when running public main or public test networks. It's not backing + up any node specific information. + +EXAMPLE + $ symbol-bootstrap backup +``` + +_See code: [src/commands/backup.ts](https://github.com/nemtech/symbol-bootstrap/blob/v0.4.4/src/commands/backup.ts)_ diff --git a/docs/config.md b/docs/config.md index 0136208c8..3c55da360 100644 --- a/docs/config.md +++ b/docs/config.md @@ -30,6 +30,15 @@ OPTIONS -u, --user=user [default: current] User used to run docker images when creating configuration files like certificates or nemesis block. "current" means the current user. + --backupSync It downloads a backup with the RocksDb and Mongo databases for faster + synchronization. + + The location of the backup can be found and changed using the 'backupSyncLocation' + preset configuration. This configuration allows local files and remote URLs + + --[no-]fullBackup If the restore/backup to be performed is a full backup (RocksDB + Mongo) or partial + backup (RocksDB + Catapult's Importer) + --pullImages It pulls the utility images from DockerHub when running the configuration. It only affects alpha/dev docker images. diff --git a/docs/start.md b/docs/start.md index e2e323280..d0f865f67 100644 --- a/docs/start.md +++ b/docs/start.md @@ -46,6 +46,16 @@ OPTIONS --args=args Add extra arguments to the docker-compose up command. Check out https://docs.docker.com/compose/reference/up. + --backupSync + It downloads a backup with the RocksDb and Mongo databases for faster synchronization. + + The location of the backup can be found and changed using the 'backupSyncLocation' preset configuration. This + configuration allows local files and remote URLs + + --[no-]fullBackup + If the restore/backup to be performed is a full backup (RocksDB + Mongo) or partial backup (RocksDB + Catapult's + Importer) + --healthCheck It checks if the services created with docker compose are up and running. diff --git a/package-lock.json b/package-lock.json index 8d79e7924..021ca57a8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -547,6 +547,15 @@ "fancy-test": "^1.4.3" } }, + "@types/archiver": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@types/archiver/-/archiver-5.1.0.tgz", + "integrity": "sha512-baFOhanb/hxmcOd1Uey2TfFg43kTSmM6py1Eo7Rjbv/ivcl7PXLhY0QgXGf50Hx/eskGCFqPfhs/7IZLb15C5g==", + "dev": true, + "requires": { + "@types/glob": "*" + } + }, "@types/chai": { "version": "4.2.12", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.2.12.tgz", @@ -842,6 +851,80 @@ "default-require-extensions": "^2.0.0" } }, + "archiver": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-5.2.0.tgz", + "integrity": "sha512-QEAKlgQuAtUxKeZB9w5/ggKXh21bZS+dzzuQ0RPBC20qtDCbTyzqmisoeJP46MP39fg4B4IcyvR+yeyEBdblsQ==", + "requires": { + "archiver-utils": "^2.1.0", + "async": "^3.2.0", + "buffer-crc32": "^0.2.1", + "readable-stream": "^3.6.0", + "readdir-glob": "^1.0.0", + "tar-stream": "^2.1.4", + "zip-stream": "^4.0.4" + }, + "dependencies": { + "tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "requires": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + } + } + } + }, + "archiver-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz", + "integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==", + "requires": { + "glob": "^7.1.4", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^2.0.0" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "archy": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz", @@ -928,8 +1011,7 @@ "base64-js": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", - "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==", - "dev": true + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" }, "bcrypt-pbkdf": { "version": "1.0.2", @@ -950,7 +1032,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz", "integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==", - "dev": true, "requires": { "buffer": "^5.5.0", "inherits": "^2.0.4", @@ -989,12 +1070,16 @@ "version": "5.6.0", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", - "dev": true, "requires": { "base64-js": "^1.0.2", "ieee754": "^1.1.4" } }, + "buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" + }, "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", @@ -1425,6 +1510,17 @@ "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=", "dev": true }, + "compress-commons": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-4.0.2.tgz", + "integrity": "sha512-qhd32a9xgzmpfoga1VQEiLEwdKZ6Plnpx5UCgIsf89FSolyJ7WnifY4Gtjgv5WR6hWAyRaHxC5MiEhU/38U70A==", + "requires": { + "buffer-crc32": "^0.2.13", + "crc32-stream": "^4.0.1", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + } + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -1496,6 +1592,24 @@ } } }, + "crc-32": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.0.tgz", + "integrity": "sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA==", + "requires": { + "exit-on-epipe": "~1.0.1", + "printj": "~1.1.0" + } + }, + "crc32-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-4.0.1.tgz", + "integrity": "sha512-FN5V+weeO/8JaXsamelVYO1PHyeCsuL3HcG4cqsj0ceARcocxalaShCsohZMSAF+db7UYFwBy1rARK/0oFItUw==", + "requires": { + "crc-32": "^1.2.0", + "readable-stream": "^3.4.0" + } + }, "create-ts-index": { "version": "1.13.6", "resolved": "https://registry.npmjs.org/create-ts-index/-/create-ts-index-1.13.6.tgz", @@ -1717,7 +1831,6 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dev": true, "requires": { "once": "^1.4.0" } @@ -1971,6 +2084,11 @@ } } }, + "exit-on-epipe": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz", + "integrity": "sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw==" + }, "extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -2266,8 +2384,7 @@ "fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "dev": true + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, "fs-extra": { "version": "7.0.1", @@ -2551,8 +2668,7 @@ "ieee754": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==", - "dev": true + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" }, "ignore": { "version": "5.1.8", @@ -2941,6 +3057,43 @@ "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" }, + "lazystream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.0.tgz", + "integrity": "sha1-9plf4PggOS9hOWvolGJAe7dxaOQ=", + "requires": { + "readable-stream": "^2.0.5" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "lcov-parse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/lcov-parse/-/lcov-parse-1.0.0.tgz", @@ -3014,12 +3167,32 @@ "resolved": "https://registry.npmjs.org/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz", "integrity": "sha1-DM8tiRZq8Ds2Y8eWU4t1rG4RTZ0=" }, + "lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" + }, + "lodash.difference": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", + "integrity": "sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw=" + }, + "lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" + }, "lodash.flattendeep": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz", "integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=", "dev": true }, + "lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=" + }, "lodash.template": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.template/-/lodash.template-4.5.0.tgz", @@ -3037,6 +3210,11 @@ "lodash._reinterpolate": "^3.0.0" } }, + "lodash.union": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", + "integrity": "sha1-SLtQiECfFvGCFmZkHETdGqrjzYg=" + }, "log-driver": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/log-driver/-/log-driver-1.2.7.tgz", @@ -3374,6 +3552,11 @@ "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz", "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==" }, + "node-stream-zip": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.12.0.tgz", + "integrity": "sha512-HZ3XehqShTFj9gHauRJ3Bri9eiCTOII7/crtXzURtT14NdnOFs9Ia5E82W7z3izVBNx760tqwddxrBJVG52Y1Q==" + }, "normalize-package-data": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", @@ -3386,6 +3569,11 @@ "validate-npm-package-license": "^3.0.1" } }, + "normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" + }, "npm-run-path": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", @@ -3696,6 +3884,11 @@ "integrity": "sha512-rFA1lnek1FYkMGthm4xBKME41qUKItTovuo24bCGZu/Vu1n3gW71UPLAkIdwewwkZCe29gRVweSOPXvAdckFuw==", "dev": true }, + "printj": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/printj/-/printj-1.1.2.tgz", + "integrity": "sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ==" + }, "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -3892,6 +4085,14 @@ "util-deprecate": "^1.0.1" } }, + "readdir-glob": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.1.tgz", + "integrity": "sha512-91/k1EzZwDx6HbERR+zucygRFfiPl2zkIYZtv3Jjr6Mn7SkKcVct8aVO+sSRiGMc6fLf72du3d92/uY63YPdEA==", + "requires": { + "minimatch": "^3.0.4" + } + }, "rechoir": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", @@ -5021,6 +5222,16 @@ "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", "dev": true + }, + "zip-stream": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-4.0.4.tgz", + "integrity": "sha512-a65wQ3h5gcQ/nQGWV1mSZCEzCML6EK/vyVPcrPNynySP1j3VBbQKh3nhC8CbORb+jfl2vXvh56Ul5odP1bAHqw==", + "requires": { + "archiver-utils": "^2.1.0", + "compress-commons": "^4.0.2", + "readable-stream": "^3.6.0" + } } } } diff --git a/package.json b/package.json index 545a87fde..288f7ad2c 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "@oclif/config": "^1.16.0", "@oclif/plugin-autocomplete": "^0.3.0", "@oclif/plugin-help": "^3.1.0", + "archiver": "^5.2.0", "figlet": "^1.2.4", "handlebars": "^4.7.6", "inquirer": "^7.3.3", @@ -19,6 +20,7 @@ "lodash": "^4.17.20", "memorystream": "^0.3.1", "node-forge": "^0.10.0", + "node-stream-zip": "^1.12.0", "rxjs": "^6.6.3", "shx": "^0.3.2", "sshpk": "^1.16.1", @@ -30,6 +32,7 @@ "devDependencies": { "@oclif/dev-cli": "^1.22.2", "@oclif/test": "^1.2.6", + "@types/archiver": "^5.1.0", "@types/chai": "^4.2.12", "@types/figlet": "^1.2.0", "@types/handlebars": "^4.1.0", diff --git a/presets/shared.yml b/presets/shared.yml index 3c7f90743..7893cbcbd 100644 --- a/presets/shared.yml +++ b/presets/shared.yml @@ -71,7 +71,7 @@ symbolWalletImage: symbolplatform/symbol-desktop-wallet:0.13.6 symbolFaucetImage: symbolplatform/symbol-faucet:0.4.0 symbolAgentImage: symbolplatform/symbol-node-rewards-agent:0.1.0 mongoImage: mongo:4.4.3-bionic -mongoComposeRunParam: --wiredTigerCacheSizeGB 2 +mongoComposeRunParam: --wiredTigerCacheSizeGB 2 --quiet logLevel: 'Info' # brokerLogLevel: 'Info' # recoveryLogLevel: 'Info' @@ -79,9 +79,9 @@ logLevel: 'Info' transactionsDirectory: './transactions' binDirectory: ./seed seedDirectory: ./seed -certificateDirectory: ./userconfig/resources/cert +certificateDirectory: ./cert dataDirectory: ./data -votingKeysDirectory: ./userconfig/votingkeys +votingKeysDirectory: ./votingkeys enableDelegatedHarvestersAutoDetection: true catapultAppFolder: /usr/catapult diff --git a/presets/testnet/network.yml b/presets/testnet/network.yml index 06190e970..7900be962 100644 --- a/presets/testnet/network.yml +++ b/presets/testnet/network.yml @@ -37,6 +37,7 @@ knownRestGateways: - 'http://api-01.us-east-1.testnet.symboldev.network:3000' - 'http://api-01.us-west-1.testnet.symboldev.network:3000' trustedHosts: +backupSyncLocation: https://symbol-bootstrap.s3-eu-west-1.amazonaws.com/testnet/backup.zip inflation: starting-at-height-2: 95998521 starting-at-height-200: 91882261 diff --git a/src/commands/backup.ts b/src/commands/backup.ts new file mode 100644 index 000000000..aa115fa37 --- /dev/null +++ b/src/commands/backup.ts @@ -0,0 +1,43 @@ +/* + * Copyright 2021 NEM + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Command, flags } from '@oclif/command'; +import { BootstrapService, BootstrapUtils } from '../service'; + +export default class Backup extends Command { + static description = `The command backs up the Mongo and RocksDb data folder into a Zip file that can then be used by the \`--backupSync\` feature. Bootstrap compose services must be stopped before calling this command. + +Note: This command is designed for NGL to be used when running public main or public test networks. It's not backing up any node specific information.`; + + static examples = [`$ symbol-bootstrap backup`]; + + static flags = { + help: BootstrapUtils.helpFlag, + target: BootstrapUtils.targetFlag, + nodeName: flags.string({ + description: `The dual/api node name to be used to backup the data. If not provided, the first configured api/dual node would be used.`, + }), + fullBackup: BootstrapUtils.fullBackupFlag, + destinationFile: flags.string({ + description: `The file location where the backup zip file will be created. Default destination is target/backup.zip.`, + }), + }; + + public async run(): Promise { + const { flags } = this.parse(Backup); + BootstrapUtils.showBanner(); + await new BootstrapService(this.config.root).backup(flags); + } +} diff --git a/src/commands/config.ts b/src/commands/config.ts index b59898026..b5bcd71ba 100644 --- a/src/commands/config.ts +++ b/src/commands/config.ts @@ -58,6 +58,15 @@ export default class Config extends Command { default: ConfigService.defaultParams.report, }), + backupSync: flags.boolean({ + description: `It downloads a backup with the RocksDb and Mongo databases for faster synchronization. + +The location of the backup can be found and changed using the 'backupSyncLocation' preset configuration. This configuration allows local files and remote URLs`, + default: ConfigService.defaultParams.backupSync, + }), + + fullBackup: BootstrapUtils.fullBackupFlag, + pullImages: flags.boolean({ description: 'It pulls the utility images from DockerHub when running the configuration. It only affects alpha/dev docker images.', diff --git a/src/model/ConfigPreset.ts b/src/model/ConfigPreset.ts index 782a447af..cbddf3ccc 100644 --- a/src/model/ConfigPreset.ts +++ b/src/model/ConfigPreset.ts @@ -188,4 +188,6 @@ export interface ConfigPreset { votingKeyEndEpoch: number; rewardProgramControllerPublicKey?: string; votingKeyLinkV2: number | undefined; + backupSyncLocation?: string; + backupSyncLocalCacheFileName?: string; } diff --git a/src/service/BackupSyncService.ts b/src/service/BackupSyncService.ts new file mode 100644 index 000000000..f5496bf8c --- /dev/null +++ b/src/service/BackupSyncService.ts @@ -0,0 +1,237 @@ +/* + * Copyright 2021 NEM + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as archiver from 'archiver'; +import { EntryDataFunction } from 'archiver'; +import { createWriteStream } from 'fs'; +import * as StreamZip from 'node-stream-zip'; +import { join } from 'path'; +import { LogType } from '../logger'; +import Logger from '../logger/Logger'; +import LoggerFactory from '../logger/LoggerFactory'; +import { ConfigPreset, DatabasePreset, NodePreset } from '../model'; +import { BootstrapUtils, KnownError } from './BootstrapUtils'; +import { ConfigLoader } from './ConfigLoader'; + +export type BackupSyncParams = { + readonly target: string; + readonly nodeName?: string; + readonly fullBackup?: boolean; + readonly destinationFile?: string; +}; + +const logger: Logger = LoggerFactory.getLogger(LogType.System); + +export class BackupSyncService { + public static defaultParams: BackupSyncParams = { + fullBackup: true, + target: BootstrapUtils.defaultTargetFolder, + }; + + constructor(private readonly root: string, protected readonly params: BackupSyncParams) {} + + public async run(presetData: ConfigPreset): Promise { + if (!presetData.backupSyncLocation) { + throw new Error(`Backup Sync cannot be executed. backupSyncLocation has not been defined.`); + } + await BootstrapUtils.mkdir(join(this.root, 'backup-sync')); + const downloadLocation = join( + this.root, + 'backup-sync', + presetData.backupSyncLocalCacheFileName || `backup-${presetData.nemesisGenerationHashSeed}.zip`, + ); + const fileLocation = (await BootstrapUtils.download(presetData.backupSyncLocation, downloadLocation)).fileLocation; + logger.info(`Restoring data from zip backup '${fileLocation}'`); + if (this.params.fullBackup) + await Promise.all( + (presetData.databases || []).map(async (db) => { + const destinationFolder = BootstrapUtils.getTargetDatabasesFolder(this.params.target, false, db.name); + await BootstrapUtils.mkdir(destinationFolder); + await this.unzip(fileLocation, 'mongo', destinationFolder); + }), + ); + await Promise.all( + (presetData.nodes || []).map(async (node) => { + const destinationFolder = BootstrapUtils.getTargetNodesFolder(this.params.target, false, node.name, 'data'); + + await BootstrapUtils.mkdir(destinationFolder); + await this.unzip(fileLocation, 'data', destinationFolder); + if (!this.params.fullBackup) { + if (node.brokerName) await BootstrapUtils.writeTextFile(join(destinationFolder, 'broker-importer.run'), ''); + else await BootstrapUtils.writeTextFile(join(destinationFolder, 'server-importer.run'), ''); + await BootstrapUtils.deleteFolder(join(destinationFolder, 'state')); + await BootstrapUtils.deleteFolder(join(destinationFolder, 'statedb')); + } + }), + ); + logger.info( + `Zip backup '${fileLocation}' has been restored. HINT: You can remove this file if you want to reclaim the disk space for future use.`, + ); + } + + private unzip(globalDestination: string, innerFolder: string, targetFolder: string): Promise { + const zip = new StreamZip({ + file: globalDestination, + storeEntries: true, + }); + logger.info(`Unzipping Backup Sync's '${innerFolder}' into '${targetFolder}'. This could take a while!`); + let totalFiles = 0; + let process = 0; + return new Promise((resolve, reject) => { + zip.on('entry', (entry) => { + if (!entry.isDirectory && totalFiles) { + process++; + const percentage = ((process * 100) / totalFiles).toFixed(2); + const message = `${percentage}% | ${process} files unzipped out of ${totalFiles}`; + BootstrapUtils.logSameLineMessage(message); + } + if (BootstrapUtils.stopProcess) { + zip.close(); + reject(new Error('Process cancelled!')); + } + }); + zip.on('ready', () => { + totalFiles = zip.entriesCount; + zip.extract(innerFolder, targetFolder, (err) => { + zip.close(); + if (err) { + reject(err); + } else { + logger.info(`Unzipped '${targetFolder}' created`); + resolve(); + } + }); + }); + }); + } + + private zip(destination: string, node: NodePreset, database: DatabasePreset): Promise { + // create a file to stream archive data to. + const output = createWriteStream(destination); + const archive = archiver('zip', { + zlib: { level: 9 }, // Sets the compression level. + }); + logger.info(`Creating zip file ${destination}. This could take a while!`); + return new Promise(async (resolve, reject) => { + // listen for all archive data to be written + // 'close' event is fired only when a file descriptor is involved + output.on('close', () => { + console.log(''); + logger.info( + `Zip file ${destination} size ${Math.floor( + archive.pointer() / 1024, + )} MB has been created. You can now share it for --backupSync.`, + ); + resolve(); + }); + + // This event is fired when the data source is drained no matter what was the data source. + // It is not part of this library but rather from the NodeJS Stream API. + // @see: https://nodejs.org/api/stream.html#stream_event_end + output.on('end', () => { + console.log(''); + logger.warn('Data has been drained'); + }); + + // good practice to catch warnings (ie stat failures and other non-blocking errors) + archive.on('warning', (err: any) => { + console.log(''); + if (err.code === 'ENOENT') { + // log warning + logger.warn(`There has been an warning creating ZIP file '${destination}' ${err.message || err}`); + } else { + // throw error + logger.error(`There has been an error creating ZIP file '${destination}' ${err.message || err}`); + reject(err); + } + }); + + // good practice to catch this error explicitly + archive.on('error', function (err: any) { + logger.error(`There has been an error creating ZIP file '${destination}' ${err.message || err}`); + reject(err); + }); + + // pipe archive data to the file + archive.pipe(output); + + const filter: (totalFiles: number, ignoreFiles: string[], ignoreDirectories: string[]) => EntryDataFunction = ( + totalFiles: number, + ignoreFiles: string[], + ignoreDirectories: string[], + ) => { + return (entry) => { + if (ignoreFiles.indexOf(entry.name) > -1) { + console.log(`\nExcluding file '${entry.name}'`); + return false; + } + + const ignoreEntryDirectory = ignoreDirectories.find((d) => entry.name.startsWith(d)); + if (ignoreEntryDirectory) { + if (entry.name === ignoreEntryDirectory) console.log(`\nExcluding directory '${entry.name}'`); + return false; + } + return entry; + }; + }; + + if (this.params.fullBackup) { + const mongoFolder = BootstrapUtils.getTargetDatabasesFolder(this.params.target, false, database.name); + const mongoTotalFiles = BootstrapUtils.getFilesRecursively(mongoFolder).length; + logger.info(`Adding '${mongoFolder}' to zip file ${destination}`); + archive.directory(mongoFolder, 'mongo', filter(mongoTotalFiles, [], [])); + } + const dataFolder = BootstrapUtils.getTargetNodesFolder(this.params.target, false, `${node.name}/`, 'data'); + const dataTotalFiles = BootstrapUtils.getFilesRecursively(dataFolder).length; + logger.info(`Adding '${dataFolder}' to zip file ${destination}`); + const ignoreDataFiles = [ + 'server.lock', + 'server.started', + 'server-importer.run', + 'broker.lock', + 'broker.started', + 'broker-importer.run', + 'recovery.lock', + ]; + const ignoreDataDirectories = ['spool', ...(this.params.fullBackup ? [] : ['state', 'statedb'])]; + archive.directory(dataFolder, 'data', filter(dataTotalFiles, ignoreDataFiles, ignoreDataDirectories)); + archive.on('progress', (progress) => { + const message = `${progress.entries.processed} entries zipped!`; + BootstrapUtils.logSameLineMessage(message); + }); + await archive.finalize(); + }); + } + + public async createBackup(passedPresetData?: ConfigPreset): Promise { + const configLoader = new ConfigLoader(); + const presetData = passedPresetData ?? configLoader.loadExistingPresetData(this.params.target, false); + + const node = presetData.nodes?.find((node) => (!this.params.nodeName || node.name == this.params.nodeName) && node.api); + if (!node && this.params.nodeName) { + throw new KnownError(`Api/Dual node with name '${this.params.nodeName}' has not been configured in this instance!`); + } + if (!node) { + throw new KnownError(`Api/Dual has not been configured in this instance!`); + } + const database = presetData.databases?.find((db) => db.name == node.databaseHost || db.host == node.databaseHost); + if (!database) { + throw new KnownError(`Database with name/host '${node.databaseHost}' does not exist!`); + } + const destination = this.params.destinationFile || join(this.params.target, 'backup.zip'); + BootstrapUtils.deleteFile(destination); + await this.zip(destination, node, database); + } +} diff --git a/src/service/BootstrapService.ts b/src/service/BootstrapService.ts index 3fc4149f8..a962eaae1 100644 --- a/src/service/BootstrapService.ts +++ b/src/service/BootstrapService.ts @@ -16,6 +16,7 @@ import { Addresses, ConfigPreset } from '../model'; import { DockerCompose } from '../model/DockerCompose'; +import { BackupSyncParams, BackupSyncService } from './BackupSyncService'; import { ComposeParams, ComposeService } from './ComposeService'; import { ConfigParams, ConfigResult, ConfigService } from './ConfigService'; import { LinkParams, LinkService } from './LinkService'; @@ -57,6 +58,18 @@ export class BootstrapService { return new ComposeService(this.root, config).run(passedPresetData, passedAddresses); } + /** + * It creates a zip backup with the node's data and mongo database folders ready for backupSync. + * + * Docker Compose must not be running when executing this operation. + * + * @param config the params of the compose command. + * @param passedPresetData the created preset if you know it, otherwise will load the latest one resolved from the target folder. + */ + public backup(config: BackupSyncParams = BackupSyncService.defaultParams, passedPresetData?: ConfigPreset): Promise { + return new BackupSyncService(this.root, config).createBackup(passedPresetData); + } + /** * It calls a running server announcing all the node transactions like VRF and Voting. * diff --git a/src/service/BootstrapUtils.ts b/src/service/BootstrapUtils.ts index 9c030784b..8163632c1 100644 --- a/src/service/BootstrapUtils.ts +++ b/src/service/BootstrapUtils.ts @@ -76,10 +76,10 @@ export class BootstrapUtils { public static readonly CURRENT_USER = 'current'; private static readonly pulledImages: string[] = []; - public static readonly VERSION = version; - public static stopProcess = false; + public static readonly VERSION = version; + public static helpFlag = flags.help({ char: 'h', description: 'It shows the help of this command.' }); public static targetFlag = flags.string({ @@ -94,6 +94,12 @@ export class BootstrapUtils { hidden: true, }); + public static fullBackupFlag = flags.boolean({ + description: `If the restore/backup to be performed is a full backup (RocksDB + Mongo) or partial backup (RocksDB + Catapult's Importer)`, + default: false, + allowNo: true, + }); + private static onProcessListener = (() => { process.on('SIGINT', () => { BootstrapUtils.stopProcess = true; @@ -238,7 +244,7 @@ export class BootstrapUtils { } public static showBanner(): void { - console.log(textSync('symbol-bootstrap', { horizontalLayout: 'full' })); + console.log(textSync('symbol-bootstrap', { horizontalLayout: 'fitted' })); } // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types diff --git a/src/service/CertificateService.ts b/src/service/CertificateService.ts index f2d0520ad..096135449 100644 --- a/src/service/CertificateService.ts +++ b/src/service/CertificateService.ts @@ -76,8 +76,7 @@ export class CertificateService { customCertFolder?: string, ): Promise { const copyFrom = `${this.root}/config/cert`; - const certFolder = - customCertFolder || BootstrapUtils.getTargetNodesFolder(this.params.target, false, name, 'userconfig', 'resources', 'cert'); + const certFolder = customCertFolder || BootstrapUtils.getTargetNodesFolder(this.params.target, false, name, 'cert'); await BootstrapUtils.mkdir(certFolder); const newCertsFolder = join(certFolder, 'new_certs'); await BootstrapUtils.mkdir(newCertsFolder); diff --git a/src/service/ComposeService.ts b/src/service/ComposeService.ts index 15c97b119..f9252df1b 100644 --- a/src/service/ComposeService.ts +++ b/src/service/ComposeService.ts @@ -29,6 +29,7 @@ export type ComposeParams = { target: string; user?: string; upgrade?: boolean; const logger: Logger = LoggerFactory.getLogger(LogType.System); const targetNodesFolder = BootstrapUtils.targetNodesFolder; +const targetNemesisFolder = BootstrapUtils.targetNemesisFolder; const targetDatabasesFolder = BootstrapUtils.targetDatabasesFolder; const targetGatewaysFolder = BootstrapUtils.targetGatewaysFolder; const targetExplorersFolder = BootstrapUtils.targetExplorersFolder; @@ -197,16 +198,14 @@ export class ComposeService { const serverDebugMode = presetData.dockerComposeDebugMode || n.dockerComposeDebugMode ? debugFlag : ''; const brokerDebugMode = presetData.dockerComposeDebugMode || n.brokerDockerComposeDebugMode ? debugFlag : ''; const waitForBroker = `/bin/bash ${nodeCommandsDirectory}/wait.sh ./data/broker.started`; - const recoverServerCommand = `/bin/bash ${nodeCommandsDirectory}/runServerRecover.sh ${n.name}`; const serverCommand = `/bin/bash ${nodeCommandsDirectory}/startServer.sh ${n.name}${serverDebugMode}`; const brokerCommand = `/bin/bash ${nodeCommandsDirectory}/startBroker.sh ${n.brokerName || 'broker'}${brokerDebugMode}`; - const recoverBrokerCommand = `/bin/bash ${nodeCommandsDirectory}/runServerRecover.sh ${n.brokerName || ''}`; const portConfigurations = [{ internalPort: 7900, openPort: n.openPort }]; - const serverServiceCommands = n.brokerName ? [waitForBroker, serverCommand] : [recoverServerCommand, serverCommand]; + const serverServiceCommands = n.brokerName ? [waitForBroker, serverCommand] : [serverCommand]; const serverServiceCommand = `bash -c "${serverServiceCommands.join(' && ')}"`; - const brokerServiceCommand = `bash -c "${[recoverBrokerCommand, brokerCommand].join(' && ')}"`; + const brokerServiceCommand = `bash -c "${[brokerCommand].join(' && ')}"`; const serverDependsOn: string[] = []; const brokerDependsOn: string[] = []; @@ -218,10 +217,10 @@ export class ComposeService { if (n.brokerName) { serverDependsOn.push(n.brokerName); } - const volumes = [ vol(`../${targetNodesFolder}/${n.name}`, nodeWorkingDirectory, false), vol(`./server`, nodeCommandsDirectory, true), + vol(`../${targetNemesisFolder}/seed`, '/seed', true), ]; const nodeService = await resolveService(n, { user: serverDebugMode === debugFlag ? undefined : user, // if debug on, run as root diff --git a/src/service/ConfigService.ts b/src/service/ConfigService.ts index df71de970..613cdc004 100644 --- a/src/service/ConfigService.ts +++ b/src/service/ConfigService.ts @@ -34,6 +34,7 @@ import Logger from '../logger/Logger'; import LoggerFactory from '../logger/LoggerFactory'; import { Addresses, ConfigPreset, NodeAccount, NodePreset, NodeType } from '../model'; import { AgentCertificateService } from './AgentCertificateService'; +import { BackupSyncService } from './BackupSyncService'; import { BootstrapUtils, KnownError } from './BootstrapUtils'; import { CertificateService } from './CertificateService'; import { ConfigLoader } from './ConfigLoader'; @@ -58,6 +59,8 @@ export interface ConfigParams { target: string; password?: string; user: string; + backupSync?: boolean; + fullBackup?: boolean; pullImages?: boolean; assembly?: string; customPreset?: string; @@ -73,12 +76,14 @@ const logger: Logger = LoggerFactory.getLogger(LogType.System); export class ConfigService { public static defaultParams: ConfigParams = { + ...BackupSyncService.defaultParams, target: BootstrapUtils.defaultTargetFolder, report: false, preset: Preset.bootstrap, reset: false, upgrade: false, pullImages: false, + backupSync: false, user: BootstrapUtils.CURRENT_USER, }; private readonly configLoader: ConfigLoader; @@ -250,7 +255,8 @@ export class ConfigService { const copyFrom = join(this.root, 'config', 'node'); const name = account.name; - const outputFolder = BootstrapUtils.getTargetNodesFolder(this.params.target, false, name, 'userconfig'); + const serverConfig = BootstrapUtils.getTargetNodesFolder(this.params.target, false, name, 'server-config'); + const brokerConfig = BootstrapUtils.getTargetNodesFolder(this.params.target, false, name, 'broker-config'); const nodePreset = (presetData.nodes || [])[index]; const generatedContext = { @@ -292,11 +298,28 @@ export class ConfigService { await BootstrapUtils.generateConfiguration(templateContext, copyFrom, agentConfig, []); } - await BootstrapUtils.generateConfiguration(templateContext, copyFrom, outputFolder, excludeFiles); + const serverRecoveryConfig = { + addressextractionRecovery: false, + mongoRecovery: false, + zeromqRecovery: false, + filespoolingRecovery: true, + hashcacheRecovery: true, + }; + + const brokerRecoveryConfig = { + addressextractionRecovery: true, + mongoRecovery: true, + zeromqRecovery: true, + filespoolingRecovery: false, + hashcacheRecovery: true, + }; + + logger.info('Generating server configuration'); + await BootstrapUtils.generateConfiguration({ ...serverRecoveryConfig, ...templateContext }, copyFrom, serverConfig, excludeFiles); await this.generateP2PFile( presetData, addresses, - outputFolder, + serverConfig, NodeType.PEER_NODE, (nodePresetData) => nodePresetData.harvesting, 'peers-p2p.json', @@ -304,11 +327,38 @@ export class ConfigService { await this.generateP2PFile( presetData, addresses, - outputFolder, + serverConfig, NodeType.API_NODE, (nodePresetData) => nodePresetData.api, 'peers-api.json', ); + + if (nodePreset.brokerName) { + logger.info('Generating broker configuration'); + await BootstrapUtils.generateConfiguration( + { ...brokerRecoveryConfig, ...templateContext }, + copyFrom, + brokerConfig, + excludeFiles, + ); + await this.generateP2PFile( + presetData, + addresses, + brokerConfig, + NodeType.PEER_NODE, + (nodePresetData) => nodePresetData.harvesting, + 'peers-p2p.json', + ); + await this.generateP2PFile( + presetData, + addresses, + brokerConfig, + NodeType.API_NODE, + (nodePresetData) => nodePresetData.api, + 'peers-api.json', + ); + } + await new VotingService(this.params).run(presetData, account, nodePreset); } @@ -356,7 +406,7 @@ export class ConfigService { const transactionsDirectory = join(nemesisWorkingDir, presetData.nemesis.transactionsDirectory || presetData.transactionsDirectory); await BootstrapUtils.mkdir(transactionsDirectory); const copyFrom = join(this.root, `config`, `nemesis`); - const moveTo = join(nemesisWorkingDir, `userconfig`); + const moveTo = join(nemesisWorkingDir, `server-config`); const templateContext = { ...(presetData as any), addresses }; await Promise.all( (addresses.nodes || []).filter((n) => n.vrf).map((n) => this.createVrfTransaction(transactionsDirectory, presetData, n)), @@ -522,15 +572,23 @@ export class ConfigService { this.params.target, false, gatewayPreset.apiNodeName, - 'userconfig', + 'server-config', 'resources', ); + const apiNodeCertFolder = BootstrapUtils.getTargetNodesFolder(this.params.target, false, gatewayPreset.apiNodeName, 'cert'); await BootstrapUtils.generateConfiguration( {}, apiNodeConfigFolder, join(moveTo, 'api-node-config'), [], - ['node.crt.pem', 'node.key.pem', 'ca.cert.pem', 'config-network.properties', 'config-node.properties'], + ['config-network.properties', 'config-node.properties'], + ); + await BootstrapUtils.generateConfiguration( + {}, + apiNodeCertFolder, + join(moveTo, 'api-node-config', 'cert'), + [], + ['node.crt.pem', 'node.key.pem', 'ca.cert.pem'], ); }), ); @@ -589,8 +647,15 @@ export class ConfigService { private cleanUpConfiguration(presetData: ConfigPreset) { const target = this.params.target; (presetData.nodes || []).forEach(({ name }) => { - const configFolder = BootstrapUtils.getTargetNodesFolder(target, false, name, 'userconfig'); - BootstrapUtils.deleteFolder(configFolder); + const serverConfigFolder = BootstrapUtils.getTargetNodesFolder(target, false, name, 'server-config'); + BootstrapUtils.deleteFolder(serverConfigFolder); + + const brokerConfigFolder = BootstrapUtils.getTargetNodesFolder(target, false, name, 'broker-config'); + BootstrapUtils.deleteFolder(brokerConfigFolder); + + const certConfigFolder = BootstrapUtils.getTargetNodesFolder(target, false, name, 'cert'); + BootstrapUtils.deleteFolder(certConfigFolder); + const seedFolder = BootstrapUtils.getTargetNodesFolder(target, false, name, 'seed'); BootstrapUtils.deleteFolder(seedFolder); }); diff --git a/src/service/NemgenService.ts b/src/service/NemgenService.ts index 94e054685..becaef8d5 100644 --- a/src/service/NemgenService.ts +++ b/src/service/NemgenService.ts @@ -44,18 +44,18 @@ export class NemgenService { await BootstrapUtils.mkdir(nemesisSeedFolder); await promises.copyFile(join(this.root, `config`, `hashes.dat`), join(nemesisSeedFolder, `hashes.dat`)); const name = presetData.nodes[0].name; - const userConfigWorkingDir = BootstrapUtils.getTargetNodesFolder(target, true, name, 'userconfig'); + const serverConfigWorkingDir = BootstrapUtils.getTargetNodesFolder(target, true, name, 'server-config'); BootstrapUtils.validateFolder(nemesisWorkingDir); - BootstrapUtils.validateFolder(userConfigWorkingDir); + BootstrapUtils.validateFolder(serverConfigWorkingDir); const cmd = [ `${presetData.catapultAppFolder}/bin/catapult.tools.nemgen`, - '--resources=/userconfig', - '--nemesisProperties=./userconfig/block-properties-file.properties', + '--resources=/server-config', + '--nemesisProperties=./server-config/block-properties-file.properties', ]; - const binds = [`${userConfigWorkingDir}:/userconfig`, `${nemesisWorkingDir}:/nemesis`]; + const binds = [`${serverConfigWorkingDir}:/server-config`, `${nemesisWorkingDir}:/nemesis`]; const userId = await BootstrapUtils.resolveDockerUserFromParam(this.params.user); const { stdout, stderr } = await BootstrapUtils.runImageUsingExec({ diff --git a/src/service/ReportService.ts b/src/service/ReportService.ts index 25ab2448c..7e612c9c7 100644 --- a/src/service/ReportService.ts +++ b/src/service/ReportService.ts @@ -102,7 +102,7 @@ export class ReportService { const target = join(workingDir, this.params.target); const descriptions = await BootstrapUtils.loadYaml(join(this.root, 'presets', 'descriptions.yml'), false); const promises: Promise[] = (presetData.nodes || []).map(async (n) => { - const resourcesFolder = join(BootstrapUtils.getTargetNodesFolder(target, false, n.name), 'userconfig', 'resources'); + const resourcesFolder = join(BootstrapUtils.getTargetNodesFolder(target, false, n.name), 'server-config', 'resources'); const files = await fsPromises.readdir(resourcesFolder); const reportFiles = files .filter((fileName) => fileName.indexOf('.properties') > -1) diff --git a/test/composes/expected-docker-compose-bootstrap-custom-compose.yml b/test/composes/expected-docker-compose-bootstrap-custom-compose.yml index 46f20bd9a..6e4e050a2 100644 --- a/test/composes/expected-docker-compose-bootstrap-custom-compose.yml +++ b/test/composes/expected-docker-compose-bootstrap-custom-compose.yml @@ -6,7 +6,7 @@ services: MONGO_INITDB_DATABASE: catapult container_name: db-0 image: 'mongo:4.4.3-bionic' - command: mongod --dbpath=/dbdata --bind_ip=db-0 --wiredTigerCacheSizeGB 2 + command: mongod --dbpath=/dbdata --bind_ip=db-0 --wiredTigerCacheSizeGB 2 --quiet stop_signal: SIGINT working_dir: /docker-entrypoint-initdb.d ports: @@ -24,7 +24,7 @@ services: user: '1000:1000' container_name: peer-node-0 image: 'symbolplatform/symbol-server:gcc-0.10.0.7' - command: bash -c "/bin/bash /symbol-commands/runServerRecover.sh peer-node-0 && /bin/bash /symbol-commands/startServer.sh peer-node-0" + command: bash -c "/bin/bash /symbol-commands/startServer.sh peer-node-0" stop_signal: SIGINT working_dir: /symbol-workdir restart: 'on-failure:2' @@ -33,6 +33,7 @@ services: volumes: - '../nodes/peer-node-0:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' cpu_count: 4 shm_size: 64M deploy: @@ -48,7 +49,7 @@ services: user: '1000:1000' container_name: peer-node-1 image: 'symbolplatform/symbol-server:gcc-0.10.0.7' - command: bash -c "/bin/bash /symbol-commands/runServerRecover.sh peer-node-1 && /bin/bash /symbol-commands/startServer.sh peer-node-1" + command: bash -c "/bin/bash /symbol-commands/startServer.sh peer-node-1" stop_signal: SIGINT working_dir: /symbol-workdir restart: 'on-failure:2' @@ -57,6 +58,7 @@ services: volumes: - '../nodes/peer-node-1:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' cpu_count: 4 shm_size: 64M deploy: @@ -79,6 +81,7 @@ services: volumes: - '../nodes/api-node-0:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db-0 - api-node-broker-0 @@ -92,9 +95,7 @@ services: container_name: api-node-broker-0 image: 'symbolplatform/symbol-server:gcc-0.10.0.7' working_dir: /symbol-workdir - command: >- - bash -c "/bin/bash /symbol-commands/runServerRecover.sh api-node-broker-0 && /bin/bash /symbol-commands/startBroker.sh - api-node-broker-0" + command: bash -c "/bin/bash /symbol-commands/startBroker.sh api-node-broker-0" ports: - '8002:7902' stop_signal: SIGINT @@ -102,6 +103,7 @@ services: volumes: - '../nodes/api-node-0:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db-0 rest-gateway-0: diff --git a/test/composes/expected-docker-compose-bootstrap-custom.yml b/test/composes/expected-docker-compose-bootstrap-custom.yml index b8af2b011..f9e32beb2 100644 --- a/test/composes/expected-docker-compose-bootstrap-custom.yml +++ b/test/composes/expected-docker-compose-bootstrap-custom.yml @@ -6,7 +6,7 @@ services: MONGO_INITDB_DATABASE: catapult container_name: db-0 image: 'mongo:4.4.3-bionic' - command: mongod --dbpath=/dbdata --bind_ip=db-0 --wiredTigerCacheSizeGB 2 + command: mongod --dbpath=/dbdata --bind_ip=db-0 --wiredTigerCacheSizeGB 2 --quiet stop_signal: SIGINT working_dir: /docker-entrypoint-initdb.d ports: @@ -17,9 +17,7 @@ services: peer-node-0: container_name: peer-node-0 image: 'symbolplatform/symbol-server:gcc-0.10.0.7' - command: >- - bash -c "/bin/bash /symbol-commands/runServerRecover.sh peer-node-0 && /bin/bash /symbol-commands/startServer.sh peer-node-0 - DEBUG" + command: bash -c "/bin/bash /symbol-commands/startServer.sh peer-node-0 DEBUG" stop_signal: SIGINT working_dir: /symbol-workdir restart: 'on-failure:2' @@ -28,6 +26,7 @@ services: volumes: - '../nodes/peer-node-0:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' security_opt: - 'seccomp:unconfined' cap_add: @@ -41,9 +40,7 @@ services: peer-node-1: container_name: peer-node-1 image: 'symbolplatform/symbol-server:gcc-0.10.0.7' - command: >- - bash -c "/bin/bash /symbol-commands/runServerRecover.sh peer-node-1 && /bin/bash /symbol-commands/startServer.sh peer-node-1 - DEBUG" + command: bash -c "/bin/bash /symbol-commands/startServer.sh peer-node-1 DEBUG" stop_signal: SIGINT working_dir: /symbol-workdir restart: 'on-failure:2' @@ -52,6 +49,7 @@ services: volumes: - '../nodes/peer-node-1:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' security_opt: - 'seccomp:unconfined' cap_add: @@ -72,6 +70,7 @@ services: volumes: - '../nodes/api-node-0:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db-0 - api-node-broker-0 @@ -89,9 +88,7 @@ services: container_name: api-node-broker-0 image: 'symbolplatform/symbol-server:gcc-0.10.0.7' working_dir: /symbol-workdir - command: >- - bash -c "/bin/bash /symbol-commands/runServerRecover.sh api-node-broker-0 && /bin/bash /symbol-commands/startBroker.sh - api-node-broker-0 DEBUG" + command: bash -c "/bin/bash /symbol-commands/startBroker.sh api-node-broker-0 DEBUG" ports: - '8002:7902' stop_signal: SIGINT @@ -99,6 +96,7 @@ services: volumes: - '../nodes/api-node-0:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db-0 security_opt: diff --git a/test/composes/expected-docker-compose-bootstrap-full.yml b/test/composes/expected-docker-compose-bootstrap-full.yml index 8f1b39979..8743072d8 100644 --- a/test/composes/expected-docker-compose-bootstrap-full.yml +++ b/test/composes/expected-docker-compose-bootstrap-full.yml @@ -6,7 +6,7 @@ services: MONGO_INITDB_DATABASE: catapult container_name: db-0 image: 'mongo:4.4.3-bionic' - command: mongod --dbpath=/dbdata --bind_ip=db-0 --wiredTigerCacheSizeGB 2 + command: mongod --dbpath=/dbdata --bind_ip=db-0 --wiredTigerCacheSizeGB 2 --quiet stop_signal: SIGINT working_dir: /docker-entrypoint-initdb.d ports: @@ -22,9 +22,7 @@ services: peer-node-0: container_name: peer-node-0 image: 'symbolplatform/symbol-server:gcc-0.10.0.7' - command: >- - bash -c "/bin/bash /symbol-commands/runServerRecover.sh peer-node-0 && /bin/bash /symbol-commands/startServer.sh peer-node-0 - DEBUG" + command: bash -c "/bin/bash /symbol-commands/startServer.sh peer-node-0 DEBUG" stop_signal: SIGINT working_dir: /symbol-workdir restart: 'on-failure:2' @@ -33,6 +31,7 @@ services: volumes: - '../nodes/peer-node-0:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' security_opt: - 'seccomp:unconfined' cap_add: @@ -46,9 +45,7 @@ services: peer-node-1: container_name: peer-node-1 image: 'symbolplatform/symbol-server:gcc-0.10.0.7' - command: >- - bash -c "/bin/bash /symbol-commands/runServerRecover.sh peer-node-1 && /bin/bash /symbol-commands/startServer.sh peer-node-1 - DEBUG" + command: bash -c "/bin/bash /symbol-commands/startServer.sh peer-node-1 DEBUG" stop_signal: SIGINT working_dir: /symbol-workdir restart: 'on-failure:2' @@ -57,6 +54,7 @@ services: volumes: - '../nodes/peer-node-1:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' security_opt: - 'seccomp:unconfined' cap_add: @@ -77,6 +75,7 @@ services: volumes: - '../nodes/api-node-0:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db-0 - api-node-broker-0 @@ -94,9 +93,7 @@ services: container_name: api-node-broker-0 image: 'symbolplatform/symbol-server:gcc-0.10.0.7' working_dir: /symbol-workdir - command: >- - bash -c "/bin/bash /symbol-commands/runServerRecover.sh api-node-broker-0 && /bin/bash /symbol-commands/startBroker.sh - api-node-broker-0 DEBUG" + command: bash -c "/bin/bash /symbol-commands/startBroker.sh api-node-broker-0 DEBUG" ports: - '8002:7902' stop_signal: SIGINT @@ -104,6 +101,7 @@ services: volumes: - '../nodes/api-node-0:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db-0 security_opt: diff --git a/test/composes/expected-docker-compose-bootstrap-repeat.yml b/test/composes/expected-docker-compose-bootstrap-repeat.yml index f3b24d713..a305a756a 100644 --- a/test/composes/expected-docker-compose-bootstrap-repeat.yml +++ b/test/composes/expected-docker-compose-bootstrap-repeat.yml @@ -6,7 +6,7 @@ services: MONGO_INITDB_DATABASE: catapult container_name: db-0 image: 'mongo:4.4.3-bionic' - command: mongod --dbpath=/dbdata --bind_ip=db-0 --wiredTigerCacheSizeGB 2 + command: mongod --dbpath=/dbdata --bind_ip=db-0 --wiredTigerCacheSizeGB 2 --quiet stop_signal: SIGINT working_dir: /docker-entrypoint-initdb.d ports: @@ -20,7 +20,7 @@ services: MONGO_INITDB_DATABASE: catapult container_name: db-1 image: 'mongo:4.4.3-bionic' - command: mongod --dbpath=/dbdata --bind_ip=db-1 --wiredTigerCacheSizeGB 2 + command: mongod --dbpath=/dbdata --bind_ip=db-1 --wiredTigerCacheSizeGB 2 --quiet stop_signal: SIGINT working_dir: /docker-entrypoint-initdb.d ports: @@ -34,7 +34,7 @@ services: MONGO_INITDB_DATABASE: catapult container_name: db-2 image: 'mongo:4.4.3-bionic' - command: mongod --dbpath=/dbdata --bind_ip=db-2 --wiredTigerCacheSizeGB 2 + command: mongod --dbpath=/dbdata --bind_ip=db-2 --wiredTigerCacheSizeGB 2 --quiet stop_signal: SIGINT working_dir: /docker-entrypoint-initdb.d ports: @@ -48,7 +48,7 @@ services: MONGO_INITDB_DATABASE: catapult container_name: db-3 image: 'mongo:4.4.3-bionic' - command: mongod --dbpath=/dbdata --bind_ip=db-3 --wiredTigerCacheSizeGB 2 + command: mongod --dbpath=/dbdata --bind_ip=db-3 --wiredTigerCacheSizeGB 2 --quiet stop_signal: SIGINT working_dir: /docker-entrypoint-initdb.d ports: @@ -60,7 +60,7 @@ services: user: '1000:1000' container_name: peer-node-0 image: 'symbolplatform/symbol-server:gcc-0.10.0.7' - command: bash -c "/bin/bash /symbol-commands/runServerRecover.sh peer-node-0 && /bin/bash /symbol-commands/startServer.sh peer-node-0" + command: bash -c "/bin/bash /symbol-commands/startServer.sh peer-node-0" stop_signal: SIGINT working_dir: /symbol-workdir restart: 'on-failure:2' @@ -69,6 +69,7 @@ services: volumes: - '../nodes/peer-node-0:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' networks: default: aliases: @@ -78,7 +79,7 @@ services: user: '1000:1000' container_name: peer-node-1 image: 'symbolplatform/symbol-server:gcc-0.10.0.7' - command: bash -c "/bin/bash /symbol-commands/runServerRecover.sh peer-node-1 && /bin/bash /symbol-commands/startServer.sh peer-node-1" + command: bash -c "/bin/bash /symbol-commands/startServer.sh peer-node-1" stop_signal: SIGINT working_dir: /symbol-workdir restart: 'on-failure:2' @@ -87,6 +88,7 @@ services: volumes: - '../nodes/peer-node-1:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' networks: default: aliases: @@ -96,7 +98,7 @@ services: user: '1000:1000' container_name: peer-node-2 image: 'symbolplatform/symbol-server:gcc-0.10.0.7' - command: bash -c "/bin/bash /symbol-commands/runServerRecover.sh peer-node-2 && /bin/bash /symbol-commands/startServer.sh peer-node-2" + command: bash -c "/bin/bash /symbol-commands/startServer.sh peer-node-2" stop_signal: SIGINT working_dir: /symbol-workdir restart: 'on-failure:2' @@ -105,6 +107,7 @@ services: volumes: - '../nodes/peer-node-2:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' networks: default: aliases: @@ -121,6 +124,7 @@ services: volumes: - '../nodes/api-node-0:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db-0 - api-node-broker-0 @@ -140,6 +144,7 @@ services: volumes: - '../nodes/api-node-1:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db-1 - api-node-broker-1 @@ -159,6 +164,7 @@ services: volumes: - '../nodes/api-node-2:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db-2 - api-node-broker-2 @@ -178,6 +184,7 @@ services: volumes: - '../nodes/api-node-3:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db-3 - api-node-broker-3 @@ -191,9 +198,7 @@ services: container_name: api-node-broker-0 image: 'symbolplatform/symbol-server:gcc-0.10.0.7' working_dir: /symbol-workdir - command: >- - bash -c "/bin/bash /symbol-commands/runServerRecover.sh api-node-broker-0 && /bin/bash /symbol-commands/startBroker.sh - api-node-broker-0" + command: bash -c "/bin/bash /symbol-commands/startBroker.sh api-node-broker-0" ports: - '8002:7902' stop_signal: SIGINT @@ -201,6 +206,7 @@ services: volumes: - '../nodes/api-node-0:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db-0 api-node-broker-1: @@ -208,9 +214,7 @@ services: container_name: api-node-broker-1 image: 'symbolplatform/symbol-server:gcc-0.10.0.7' working_dir: /symbol-workdir - command: >- - bash -c "/bin/bash /symbol-commands/runServerRecover.sh api-node-broker-1 && /bin/bash /symbol-commands/startBroker.sh - api-node-broker-1" + command: bash -c "/bin/bash /symbol-commands/startBroker.sh api-node-broker-1" ports: - '8003:7902' stop_signal: SIGINT @@ -218,6 +222,7 @@ services: volumes: - '../nodes/api-node-1:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db-1 api-node-broker-2: @@ -225,9 +230,7 @@ services: container_name: api-node-broker-2 image: 'symbolplatform/symbol-server:gcc-0.10.0.7' working_dir: /symbol-workdir - command: >- - bash -c "/bin/bash /symbol-commands/runServerRecover.sh api-node-broker-2 && /bin/bash /symbol-commands/startBroker.sh - api-node-broker-2" + command: bash -c "/bin/bash /symbol-commands/startBroker.sh api-node-broker-2" ports: - '8004:7902' stop_signal: SIGINT @@ -235,6 +238,7 @@ services: volumes: - '../nodes/api-node-2:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db-2 api-node-broker-3: @@ -242,9 +246,7 @@ services: container_name: api-node-broker-3 image: 'symbolplatform/symbol-server:gcc-0.10.0.7' working_dir: /symbol-workdir - command: >- - bash -c "/bin/bash /symbol-commands/runServerRecover.sh api-node-broker-3 && /bin/bash /symbol-commands/startBroker.sh - api-node-broker-3" + command: bash -c "/bin/bash /symbol-commands/startBroker.sh api-node-broker-3" ports: - '8005:7902' stop_signal: SIGINT @@ -252,6 +254,7 @@ services: volumes: - '../nodes/api-node-3:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db-3 rest-gateway-0: diff --git a/test/composes/expected-docker-compose-bootstrap.yml b/test/composes/expected-docker-compose-bootstrap.yml index d7f49388b..e1d4c477a 100644 --- a/test/composes/expected-docker-compose-bootstrap.yml +++ b/test/composes/expected-docker-compose-bootstrap.yml @@ -6,7 +6,7 @@ services: MONGO_INITDB_DATABASE: catapult container_name: db-0 image: 'mongo:4.4.3-bionic' - command: mongod --dbpath=/dbdata --bind_ip=db-0 --wiredTigerCacheSizeGB 2 + command: mongod --dbpath=/dbdata --bind_ip=db-0 --wiredTigerCacheSizeGB 2 --quiet stop_signal: SIGINT working_dir: /docker-entrypoint-initdb.d ports: @@ -18,7 +18,7 @@ services: user: '1000:1000' container_name: peer-node-0 image: 'symbolplatform/symbol-server:gcc-0.10.0.7' - command: bash -c "/bin/bash /symbol-commands/runServerRecover.sh peer-node-0 && /bin/bash /symbol-commands/startServer.sh peer-node-0" + command: bash -c "/bin/bash /symbol-commands/startServer.sh peer-node-0" stop_signal: SIGINT working_dir: /symbol-workdir restart: 'on-failure:2' @@ -27,6 +27,7 @@ services: volumes: - '../nodes/peer-node-0:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' networks: default: aliases: @@ -36,7 +37,7 @@ services: user: '1000:1000' container_name: peer-node-1 image: 'symbolplatform/symbol-server:gcc-0.10.0.7' - command: bash -c "/bin/bash /symbol-commands/runServerRecover.sh peer-node-1 && /bin/bash /symbol-commands/startServer.sh peer-node-1" + command: bash -c "/bin/bash /symbol-commands/startServer.sh peer-node-1" stop_signal: SIGINT working_dir: /symbol-workdir restart: 'on-failure:2' @@ -45,6 +46,7 @@ services: volumes: - '../nodes/peer-node-1:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' networks: default: aliases: @@ -61,6 +63,7 @@ services: volumes: - '../nodes/api-node-0:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db-0 - api-node-broker-0 @@ -74,9 +77,7 @@ services: container_name: api-node-broker-0 image: 'symbolplatform/symbol-server:gcc-0.10.0.7' working_dir: /symbol-workdir - command: >- - bash -c "/bin/bash /symbol-commands/runServerRecover.sh api-node-broker-0 && /bin/bash /symbol-commands/startBroker.sh - api-node-broker-0" + command: bash -c "/bin/bash /symbol-commands/startBroker.sh api-node-broker-0" ports: - '8002:7902' stop_signal: SIGINT @@ -84,6 +85,7 @@ services: volumes: - '../nodes/api-node-0:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db-0 rest-gateway-0: diff --git a/test/composes/expected-testnet-dual-compose.yml b/test/composes/expected-testnet-dual-compose.yml index bd8b365e5..a3d2ed160 100644 --- a/test/composes/expected-testnet-dual-compose.yml +++ b/test/composes/expected-testnet-dual-compose.yml @@ -6,7 +6,7 @@ services: MONGO_INITDB_DATABASE: catapult container_name: db image: 'mongo:4.4.3-bionic' - command: mongod --dbpath=/dbdata --bind_ip=db --wiredTigerCacheSizeGB 2 + command: mongod --dbpath=/dbdata --bind_ip=db --wiredTigerCacheSizeGB 2 --quiet stop_signal: SIGINT working_dir: /docker-entrypoint-initdb.d volumes: @@ -25,6 +25,7 @@ services: volumes: - '../nodes/api-node:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db - api-broker @@ -33,7 +34,7 @@ services: container_name: api-broker image: 'symbolplatform/symbol-server:gcc-0.10.1.7' working_dir: /symbol-workdir - command: bash -c "/bin/bash /symbol-commands/runServerRecover.sh api-broker && /bin/bash /symbol-commands/startBroker.sh api-broker" + command: bash -c "/bin/bash /symbol-commands/startBroker.sh api-broker" ports: - '7902:7902' stop_signal: SIGINT @@ -41,6 +42,7 @@ services: volumes: - '../nodes/api-node:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db rest-gateway: diff --git a/test/composes/expected-testnet-supernode-compose.yml b/test/composes/expected-testnet-supernode-compose.yml index 2f7f789f8..f7a35e18c 100644 --- a/test/composes/expected-testnet-supernode-compose.yml +++ b/test/composes/expected-testnet-supernode-compose.yml @@ -6,7 +6,7 @@ services: MONGO_INITDB_DATABASE: catapult container_name: db image: 'mongo:4.4.3-bionic' - command: mongod --dbpath=/dbdata --bind_ip=db --wiredTigerCacheSizeGB 2 + command: mongod --dbpath=/dbdata --bind_ip=db --wiredTigerCacheSizeGB 2 --quiet stop_signal: SIGINT working_dir: /docker-entrypoint-initdb.d volumes: @@ -25,6 +25,7 @@ services: volumes: - '../nodes/api-node:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db - api-broker @@ -38,7 +39,7 @@ services: container_name: api-broker image: 'symbolplatform/symbol-server:gcc-0.10.1.7' working_dir: /symbol-workdir - command: bash -c "/bin/bash /symbol-commands/runServerRecover.sh api-broker && /bin/bash /symbol-commands/startBroker.sh api-broker" + command: bash -c "/bin/bash /symbol-commands/startBroker.sh api-broker" ports: - '7902:7902' stop_signal: SIGINT @@ -46,6 +47,7 @@ services: volumes: - '../nodes/api-node:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db api-node-agent: diff --git a/test/composes/expected-testnet-voting-compose.yml b/test/composes/expected-testnet-voting-compose.yml index bd8b365e5..a3d2ed160 100644 --- a/test/composes/expected-testnet-voting-compose.yml +++ b/test/composes/expected-testnet-voting-compose.yml @@ -6,7 +6,7 @@ services: MONGO_INITDB_DATABASE: catapult container_name: db image: 'mongo:4.4.3-bionic' - command: mongod --dbpath=/dbdata --bind_ip=db --wiredTigerCacheSizeGB 2 + command: mongod --dbpath=/dbdata --bind_ip=db --wiredTigerCacheSizeGB 2 --quiet stop_signal: SIGINT working_dir: /docker-entrypoint-initdb.d volumes: @@ -25,6 +25,7 @@ services: volumes: - '../nodes/api-node:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db - api-broker @@ -33,7 +34,7 @@ services: container_name: api-broker image: 'symbolplatform/symbol-server:gcc-0.10.1.7' working_dir: /symbol-workdir - command: bash -c "/bin/bash /symbol-commands/runServerRecover.sh api-broker && /bin/bash /symbol-commands/startBroker.sh api-broker" + command: bash -c "/bin/bash /symbol-commands/startBroker.sh api-broker" ports: - '7902:7902' stop_signal: SIGINT @@ -41,6 +42,7 @@ services: volumes: - '../nodes/api-node:/symbol-workdir:rw' - './server:/symbol-commands:ro' + - '../nemesis/seed:/seed:ro' depends_on: - db rest-gateway: diff --git a/test/reports/bootstrap-voting/api-node-0-config.csv b/test/reports/bootstrap-voting/api-node-0-config.csv index 7ba5160ab..92a8f5b05 100644 --- a/test/reports/bootstrap-voting/api-node-0-config.csv +++ b/test/reports/bootstrap-voting/api-node-0-config.csv @@ -32,9 +32,10 @@ extension.hashcache; true config-extensions-recovery.properties extensions -extension.addressextraction; true -extension.mongo; true -extension.zeromq; true +extension.addressextraction; false +extension.mongo; false +extension.zeromq; false +extension.filespooling; true extension.hashcache; true @@ -428,7 +429,7 @@ enableDelegatedHarvestersAutoDetection; true storage seedDirectory; ./seed -certificateDirectory; ./userconfig/resources/cert +certificateDirectory; ./cert dataDirectory; ./data pluginsDirectory; /usr/catapult/lib -votingKeysDirectory; ./userconfig/votingkeys +votingKeysDirectory; ./votingkeys \ No newline at end of file diff --git a/test/reports/bootstrap-voting/api-node-0-config.rst b/test/reports/bootstrap-voting/api-node-0-config.rst index 760d2ac25..67dfb9dab 100644 --- a/test/reports/bootstrap-voting/api-node-0-config.rst +++ b/test/reports/bootstrap-voting/api-node-0-config.rst @@ -44,9 +44,10 @@ config-extensions-recovery.properties :delim: ; **extensions**; - extension.addressextraction; true - extension.mongo; true - extension.zeromq; true + extension.addressextraction; false + extension.mongo; false + extension.zeromq; false + extension.filespooling; true extension.hashcache; true config-extensions-server.properties @@ -447,7 +448,7 @@ config-user.properties enableDelegatedHarvestersAutoDetection; true **storage**; seedDirectory; ./seed - certificateDirectory; ./userconfig/resources/cert + certificateDirectory; ./cert dataDirectory; ./data pluginsDirectory; /usr/catapult/lib - votingKeysDirectory; ./userconfig/votingkeys + votingKeysDirectory; ./votingkeys \ No newline at end of file diff --git a/test/reports/bootstrap-voting/peer-node-0-config.csv b/test/reports/bootstrap-voting/peer-node-0-config.csv index 9aa6da59e..12b776a82 100644 --- a/test/reports/bootstrap-voting/peer-node-0-config.csv +++ b/test/reports/bootstrap-voting/peer-node-0-config.csv @@ -35,6 +35,7 @@ extensions extension.addressextraction; false extension.mongo; false extension.zeromq; false +extension.filespooling; true extension.hashcache; true @@ -438,7 +439,7 @@ enableDelegatedHarvestersAutoDetection; true storage seedDirectory; ./seed -certificateDirectory; ./userconfig/resources/cert +certificateDirectory; ./cert dataDirectory; ./data pluginsDirectory; /usr/catapult/lib -votingKeysDirectory; ./userconfig/votingkeys +votingKeysDirectory; ./votingkeys \ No newline at end of file diff --git a/test/reports/bootstrap-voting/peer-node-0-config.rst b/test/reports/bootstrap-voting/peer-node-0-config.rst index 914777dd9..156f94822 100644 --- a/test/reports/bootstrap-voting/peer-node-0-config.rst +++ b/test/reports/bootstrap-voting/peer-node-0-config.rst @@ -47,6 +47,7 @@ config-extensions-recovery.properties extension.addressextraction; false extension.mongo; false extension.zeromq; false + extension.filespooling; true extension.hashcache; true config-extensions-server.properties @@ -461,7 +462,7 @@ config-user.properties enableDelegatedHarvestersAutoDetection; true **storage**; seedDirectory; ./seed - certificateDirectory; ./userconfig/resources/cert + certificateDirectory; ./cert dataDirectory; ./data pluginsDirectory; /usr/catapult/lib - votingKeysDirectory; ./userconfig/votingkeys + votingKeysDirectory; ./votingkeys \ No newline at end of file diff --git a/test/reports/bootstrap-voting/peer-node-1-config.csv b/test/reports/bootstrap-voting/peer-node-1-config.csv index 2f39cb868..e2d2734f4 100644 --- a/test/reports/bootstrap-voting/peer-node-1-config.csv +++ b/test/reports/bootstrap-voting/peer-node-1-config.csv @@ -35,6 +35,7 @@ extensions extension.addressextraction; false extension.mongo; false extension.zeromq; false +extension.filespooling; true extension.hashcache; true @@ -438,7 +439,7 @@ enableDelegatedHarvestersAutoDetection; true storage seedDirectory; ./seed -certificateDirectory; ./userconfig/resources/cert +certificateDirectory; ./cert dataDirectory; ./data pluginsDirectory; /usr/catapult/lib -votingKeysDirectory; ./userconfig/votingkeys +votingKeysDirectory; ./votingkeys \ No newline at end of file diff --git a/test/reports/bootstrap-voting/peer-node-1-config.rst b/test/reports/bootstrap-voting/peer-node-1-config.rst index 00c6848a7..63c8127b0 100644 --- a/test/reports/bootstrap-voting/peer-node-1-config.rst +++ b/test/reports/bootstrap-voting/peer-node-1-config.rst @@ -47,6 +47,7 @@ config-extensions-recovery.properties extension.addressextraction; false extension.mongo; false extension.zeromq; false + extension.filespooling; true extension.hashcache; true config-extensions-server.properties @@ -461,7 +462,7 @@ config-user.properties enableDelegatedHarvestersAutoDetection; true **storage**; seedDirectory; ./seed - certificateDirectory; ./userconfig/resources/cert + certificateDirectory; ./cert dataDirectory; ./data pluginsDirectory; /usr/catapult/lib - votingKeysDirectory; ./userconfig/votingkeys + votingKeysDirectory; ./votingkeys \ No newline at end of file diff --git a/test/reports/testnet-api/api-node-config.csv b/test/reports/testnet-api/api-node-config.csv index ac979bb8c..fba9a64fe 100644 --- a/test/reports/testnet-api/api-node-config.csv +++ b/test/reports/testnet-api/api-node-config.csv @@ -32,9 +32,10 @@ extension.hashcache; true config-extensions-recovery.properties extensions -extension.addressextraction; true -extension.mongo; true -extension.zeromq; true +extension.addressextraction; false +extension.mongo; false +extension.zeromq; false +extension.filespooling; true extension.hashcache; true @@ -848,7 +849,7 @@ enableDelegatedHarvestersAutoDetection; true storage seedDirectory; ./seed -certificateDirectory; ./userconfig/resources/cert +certificateDirectory; ./cert dataDirectory; ./data pluginsDirectory; /usr/catapult/lib -votingKeysDirectory; ./userconfig/votingkeys +votingKeysDirectory; ./votingkeys \ No newline at end of file diff --git a/test/reports/testnet-api/api-node-config.rst b/test/reports/testnet-api/api-node-config.rst index cee55ace8..ef8c532b6 100644 --- a/test/reports/testnet-api/api-node-config.rst +++ b/test/reports/testnet-api/api-node-config.rst @@ -44,9 +44,10 @@ config-extensions-recovery.properties :delim: ; **extensions**; - extension.addressextraction; true - extension.mongo; true - extension.zeromq; true + extension.addressextraction; false + extension.mongo; false + extension.zeromq; false + extension.filespooling; true extension.hashcache; true config-extensions-server.properties @@ -867,7 +868,7 @@ config-user.properties enableDelegatedHarvestersAutoDetection; true **storage**; seedDirectory; ./seed - certificateDirectory; ./userconfig/resources/cert + certificateDirectory; ./cert dataDirectory; ./data pluginsDirectory; /usr/catapult/lib - votingKeysDirectory; ./userconfig/votingkeys + votingKeysDirectory; ./votingkeys \ No newline at end of file diff --git a/test/reports/testnet-dual-voting/api-node-config.csv b/test/reports/testnet-dual-voting/api-node-config.csv index 4528100ca..9627a2c86 100644 --- a/test/reports/testnet-dual-voting/api-node-config.csv +++ b/test/reports/testnet-dual-voting/api-node-config.csv @@ -32,9 +32,10 @@ extension.hashcache; true config-extensions-recovery.properties extensions -extension.addressextraction; true -extension.mongo; true -extension.zeromq; true +extension.addressextraction; false +extension.mongo; false +extension.zeromq; false +extension.filespooling; true extension.hashcache; true @@ -858,7 +859,7 @@ enableDelegatedHarvestersAutoDetection; true storage seedDirectory; ./seed -certificateDirectory; ./userconfig/resources/cert +certificateDirectory; ./cert dataDirectory; ./data pluginsDirectory; /usr/catapult/lib -votingKeysDirectory; ./userconfig/votingkeys +votingKeysDirectory; ./votingkeys \ No newline at end of file diff --git a/test/reports/testnet-dual-voting/api-node-config.rst b/test/reports/testnet-dual-voting/api-node-config.rst index 8ad52dd0c..bf0394820 100644 --- a/test/reports/testnet-dual-voting/api-node-config.rst +++ b/test/reports/testnet-dual-voting/api-node-config.rst @@ -44,9 +44,10 @@ config-extensions-recovery.properties :delim: ; **extensions**; - extension.addressextraction; true - extension.mongo; true - extension.zeromq; true + extension.addressextraction; false + extension.mongo; false + extension.zeromq; false + extension.filespooling; true extension.hashcache; true config-extensions-server.properties @@ -881,7 +882,7 @@ config-user.properties enableDelegatedHarvestersAutoDetection; true **storage**; seedDirectory; ./seed - certificateDirectory; ./userconfig/resources/cert + certificateDirectory; ./cert dataDirectory; ./data pluginsDirectory; /usr/catapult/lib - votingKeysDirectory; ./userconfig/votingkeys + votingKeysDirectory; ./votingkeys \ No newline at end of file diff --git a/test/reports/testnet-peer/peer-node-config.csv b/test/reports/testnet-peer/peer-node-config.csv index 237a6a2c5..b4fbce071 100644 --- a/test/reports/testnet-peer/peer-node-config.csv +++ b/test/reports/testnet-peer/peer-node-config.csv @@ -35,6 +35,7 @@ extensions extension.addressextraction; false extension.mongo; false extension.zeromq; false +extension.filespooling; true extension.hashcache; true @@ -858,7 +859,7 @@ enableDelegatedHarvestersAutoDetection; true storage seedDirectory; ./seed -certificateDirectory; ./userconfig/resources/cert +certificateDirectory; ./cert dataDirectory; ./data pluginsDirectory; /usr/catapult/lib -votingKeysDirectory; ./userconfig/votingkeys +votingKeysDirectory; ./votingkeys \ No newline at end of file diff --git a/test/reports/testnet-peer/peer-node-config.rst b/test/reports/testnet-peer/peer-node-config.rst index 579e6fafd..073c9e846 100644 --- a/test/reports/testnet-peer/peer-node-config.rst +++ b/test/reports/testnet-peer/peer-node-config.rst @@ -47,6 +47,7 @@ config-extensions-recovery.properties extension.addressextraction; false extension.mongo; false extension.zeromq; false + extension.filespooling; true extension.hashcache; true config-extensions-server.properties @@ -881,7 +882,7 @@ config-user.properties enableDelegatedHarvestersAutoDetection; true **storage**; seedDirectory; ./seed - certificateDirectory; ./userconfig/resources/cert + certificateDirectory; ./cert dataDirectory; ./data pluginsDirectory; /usr/catapult/lib - votingKeysDirectory; ./userconfig/votingkeys + votingKeysDirectory; ./votingkeys \ No newline at end of file diff --git a/test/service/BackupSyncService.test.ts b/test/service/BackupSyncService.test.ts new file mode 100644 index 000000000..78e9e32d7 --- /dev/null +++ b/test/service/BackupSyncService.test.ts @@ -0,0 +1,80 @@ +/* + * Copyright 2021 NEM + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from '@oclif/test'; +import { existsSync } from 'fs'; +import 'mocha'; +import { BootstrapService, BootstrapUtils, ConfigLoader, ConfigService, Preset, StartParams } from '../../src/service'; +import { BackupSyncService } from '../../src/service/BackupSyncService'; + +describe('BackupSyncService', () => { + it('run', async () => { + const target = 'target/BackupSyncService.test'; + await BootstrapUtils.deleteFolder(target); + await BootstrapUtils.mkdir(target); + const preset = Preset.testnet; + const service = new BackupSyncService('.', { target: target, fullBackup: true }); + + const presetData = new ConfigLoader().createPresetData({ + root: '.', + preset: preset, + assembly: 'dual', + password: undefined, + customPresetObject: { + backupSyncLocation: 'https://symbol-bootstrap.s3-eu-west-1.amazonaws.com/testnet/testnet-unit-test.zip', + backupSyncLocalCacheFileName: 'testnet-unit-test.zip', + }, + }); + await service.run(presetData); + expect(existsSync(`${target}/nodes/api-node/data/00000/00002.dat`)).eq(true); + expect(existsSync(`${target}/databases/db/mongod.lock`)).eq(true); + }); + + it('run, stop, create backup', async () => { + const target = 'target/BackupSyncService.e2e'; + const backupSyncLocation = './backup-sync/testnet-unittest-backup.zip'; + BootstrapUtils.deleteFile(backupSyncLocation); + const config: StartParams = { + ...ConfigService.defaultParams, + preset: Preset.testnet, + reset: true, + detached: true, + healthCheck: true, + assembly: 'dual', + pullImages: true, + target, + customPresetObject: { + backupSyncLocation: backupSyncLocation, + backupSyncLocalCacheFileName: 'testnet-unittest-backup.zip', + }, + }; + + const service = new BootstrapService('.'); + + await service.start(config); + + await BootstrapUtils.sleep(5000); + + await service.stop(config); + + await service.backup({ + target: target, + fullBackup: true, + destinationFile: backupSyncLocation, + }); + expect(existsSync(backupSyncLocation)).eq(true); + }); +}); diff --git a/test/testnet-custom-preset.yml b/test/testnet-custom-preset.yml new file mode 100644 index 000000000..9d201a38c --- /dev/null +++ b/test/testnet-custom-preset.yml @@ -0,0 +1,2 @@ +backupSyncLocation: ./backup-sync/testnet-local-backup.zip +backupSyncLocalCacheFileName: testnet-local-backup.zip diff --git a/test/voting_preset.yml b/test/voting_preset.yml index d631a1e6e..a827610a2 100644 --- a/test/voting_preset.yml +++ b/test/voting_preset.yml @@ -1,3 +1,5 @@ +backupSyncLocation: ./backup-sync/testnet-local-backup.zip +backupSyncLocalCacheFileName: testnet-local-backup.zip databases: - compose: mem_limit: ~