diff --git a/.gitignore b/.gitignore index b71c64271..268de67a3 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,6 @@ vendor/ scripts/bifrost/node_modules scripts/bifrost/yarn.lock +# Test environment .testenv.local +/testenv-artifacts \ No newline at end of file diff --git a/build/build/Dockerfile-DiadataBuild-114-Dev b/build/build/Dockerfile-DiadataBuild-114-Dev index 5f584845b..b86fef1a4 100755 --- a/build/build/Dockerfile-DiadataBuild-114-Dev +++ b/build/build/Dockerfile-DiadataBuild-114-Dev @@ -1,4 +1,4 @@ -FROM public.ecr.aws/docker/library/golang:1.14 as build +FROM public.ecr.aws/docker/library/golang:1.14 AS build RUN apt update && apt upgrade -y diff --git a/build/build/Dockerfile-DiadataBuild-117-Dev b/build/build/Dockerfile-DiadataBuild-117-Dev index 699c42cfc..729d5a98a 100755 --- a/build/build/Dockerfile-DiadataBuild-117-Dev +++ b/build/build/Dockerfile-DiadataBuild-117-Dev @@ -1,4 +1,4 @@ -FROM public.ecr.aws/docker/library/golang:1.17 as build +FROM public.ecr.aws/docker/library/golang:1.17 AS build RUN apt update && apt upgrade -y diff --git a/build/build/Dockerfile-DiadataBuild-119-Dev b/build/build/Dockerfile-DiadataBuild-119-Dev index 3642eeec0..6421d84b8 100755 --- a/build/build/Dockerfile-DiadataBuild-119-Dev +++ b/build/build/Dockerfile-DiadataBuild-119-Dev @@ -1,4 +1,4 @@ -FROM public.ecr.aws/docker/library/golang:1.19 as build +FROM public.ecr.aws/docker/library/golang:1.19 AS build RUN apt update && apt upgrade -y diff --git a/build/build/Dockerfile-DiadataBuild-120-Dev b/build/build/Dockerfile-DiadataBuild-120-Dev index 5a90c3679..b8e7a4494 100755 --- a/build/build/Dockerfile-DiadataBuild-120-Dev +++ b/build/build/Dockerfile-DiadataBuild-120-Dev @@ -1,4 +1,4 @@ -FROM public.ecr.aws/docker/library/golang:1.20 as build +FROM public.ecr.aws/docker/library/golang:1.20 AS build RUN apt update && apt upgrade -y diff --git a/build/build/Dockerfile-DiadataBuild-121-Dev b/build/build/Dockerfile-DiadataBuild-121-Dev new file mode 100644 index 000000000..4f0f2891a --- /dev/null +++ b/build/build/Dockerfile-DiadataBuild-121-Dev @@ -0,0 +1,13 @@ +FROM public.ecr.aws/docker/library/golang:1.21 AS build + +RUN apt update && apt upgrade -y + +COPY ./config/ /config/ + +WORKDIR $GOPATH/src/ +COPY ./go.mod ./ + +ENV GO111MODULE="on" +RUN go mod download + +RUN go get github.com/karalabe/usb@v0.0.0-20210518091819-4ea20957c210 \ No newline at end of file diff --git a/build/build/Dockerfile-DiadataBuild-122-Dev b/build/build/Dockerfile-DiadataBuild-122-Dev new file mode 100644 index 000000000..d2528a4b9 --- /dev/null +++ b/build/build/Dockerfile-DiadataBuild-122-Dev @@ -0,0 +1,13 @@ +FROM public.ecr.aws/docker/library/golang:1.22 AS build + +RUN apt update && apt upgrade -y + +COPY ./config/ /config/ + +WORKDIR $GOPATH/src/ +COPY ./go.mod ./ + +ENV GO111MODULE="on" +RUN go mod download + +RUN go get github.com/karalabe/usb@v0.0.0-20210518091819-4ea20957c210 \ No newline at end of file diff --git a/deployments/config/pginit.sql b/deployments/config/pginit.sql index 677471a59..fd01d2cd7 100644 --- a/deployments/config/pginit.sql +++ b/deployments/config/pginit.sql @@ -57,6 +57,22 @@ CREATE TABLE exchange ( UNIQUE (name) ); +CREATE TABLE nftexchange ( + exchange_id UUID DEFAULT gen_random_uuid(), + name text NOT NULL, + centralized boolean default false, + bridge boolean default false, + contract text, + blockchain text, + rest_api text, + ws_api text, + pairs_api text, + watchdog_delay numeric NOT NULL, + scraper_active boolean, + UNIQUE(exchange_id), + UNIQUE (name) +); + CREATE TABLE pool ( pool_id UUID DEFAULT gen_random_uuid(), exchange text NOT NULL, @@ -269,9 +285,19 @@ CREATE TABLE exchange_pairs ( +CREATE TABLE plans ( + plan_id SERIAL PRIMARY KEY, + plan_name VARCHAR(50) NOT NULL UNIQUE, + plan_description TEXT, + plan_price NUMERIC(10, 2) NOT NULL, + plan_features TEXT +); +ALTER TABLE plans ADD COLUMN total_feeds integer default 3; +ALTER TABLE plans ADD COLUMN total_oracles integer default 3; - CREATE TABLE customers ( + +CREATE TABLE customers ( customer_id SERIAL PRIMARY KEY, email VARCHAR(255) NOT NULL, account_creation_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP, @@ -400,18 +426,6 @@ CREATE TABLE loop_payment_responses ( ); -CREATE TABLE plans ( - plan_id SERIAL PRIMARY KEY, - plan_name VARCHAR(50) NOT NULL UNIQUE, - plan_description TEXT, - plan_price NUMERIC(10, 2) NOT NULL, - plan_features TEXT -); - -ALTER TABLE plans ADD COLUMN total_feeds integer default 3; -ALTER TABLE plans ADD COLUMN total_oracles integer default 3; - - diff --git a/deployments/k8s-yaml/data-kafka-ui.yaml b/deployments/k8s-yaml/data-kafka-ui.yaml new file mode 100755 index 000000000..91bcda9a5 --- /dev/null +++ b/deployments/k8s-yaml/data-kafka-ui.yaml @@ -0,0 +1,47 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kafka-ui + labels: + app: kafka-ui +spec: + replicas: 1 + selector: + matchLabels: + app: kafka-ui + template: + metadata: + labels: + app: kafka-ui + spec: + containers: + - name: kafka-ui + image: provectuslabs/kafka-ui:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + name: http + env: + - name: KAFKA_CLUSTERS_0_NAME + value: local + - name: KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS + value: data-kafka.default.svc.cluster.local:9094 + - name: KAFKA_CLUSTERS_0_ZOOKEEPER + value: data-kafka.default.svc.cluster.local:2181 +--- +apiVersion: v1 +kind: Service +metadata: + name: kafka-ui + labels: + app: kafka-ui +spec: + type: NodePort + ports: + - port: 8080 + targetPort: http + nodePort: 30080 + name: http + selector: + app: kafka-ui diff --git a/deployments/k8s-yaml/job-prepare.yaml b/deployments/k8s-yaml/job-prepare.yaml index e2ef3db68..ba8295bc5 100755 --- a/deployments/k8s-yaml/job-prepare.yaml +++ b/deployments/k8s-yaml/job-prepare.yaml @@ -39,12 +39,14 @@ spec: value: "password" - name: POSTGRES_DB value: "postgres" + - name: REDISURL + value: "data-redis.default.svc.cluster.local:6379" - name: job-prepare-assetcollectionservice-nonethassets image: dia.assetcollectionservice.dev:latest imagePullPolicy: Never command: ["assetCollectionService"] args: - ["-source=assetlists", "-secret=non_eth_assets", "-caching=true"] + ["-source=assetlists", "-secret=non_eth_assets", "-caching=true"] # skipcq: SCT-A000 env: - name: USE_ENV value: "true" @@ -56,11 +58,13 @@ spec: value: "password" - name: POSTGRES_DB value: "postgres" + - name: REDISURL + value: "data-redis.default.svc.cluster.local:6379" - name: job-prepare-assetcollectionservice-fiatassets image: dia.assetcollectionservice.dev:latest imagePullPolicy: Never command: ["assetCollectionService"] - args: ["-source=assetlists", "-secret=fiat_assets", "-caching=true"] + args: ["-source=assetlists", "-secret=fiat_assets", "-caching=true"] # skipcq: SCT-A000 env: - name: USE_ENV value: "true" @@ -72,6 +76,8 @@ spec: value: "password" - name: POSTGRES_DB value: "postgres" + - name: REDISURL + value: "data-redis.default.svc.cluster.local:6379" initContainers: - name: check-postgres-connection image: public.ecr.aws/docker/library/postgres:15-alpine diff --git a/deployments/k8s-yaml/scraper-cex-coinbase.yaml b/deployments/k8s-yaml/scraper-cex-coinbase.yaml index 5223b28c9..3f160f864 100755 --- a/deployments/k8s-yaml/scraper-cex-coinbase.yaml +++ b/deployments/k8s-yaml/scraper-cex-coinbase.yaml @@ -8,7 +8,7 @@ spec: image: dia.genericcollector.dev:latest imagePullPolicy: Never command: ["collector"] - args: ["-exchange=CoinBase", "-mode=current", "-pairsfile=true"] + args: ["-exchange=CoinBase"] # "-mode=storeTrades|current", "-pairsfile=true|false" env: - name: USE_ENV value: "true" @@ -30,6 +30,8 @@ spec: value: "data-redis.default.svc.cluster.local:6379" - name: KAFKAURL value: "data-kafka.default.svc.cluster.local:9094" + - name: EXEC_MODE + value: production volumeMounts: - name: env-context mountPath: /mnt/env-context diff --git a/deployments/k8s-yaml/scraper-cex-kraken.yaml b/deployments/k8s-yaml/scraper-cex-kraken.yaml new file mode 100755 index 000000000..118e8e103 --- /dev/null +++ b/deployments/k8s-yaml/scraper-cex-kraken.yaml @@ -0,0 +1,109 @@ +apiVersion: "v1" +kind: Pod +metadata: + name: scraper-exchange-kraken +spec: + containers: + - name: scraper-exchange-kraken + image: dia.genericcollector.dev:latest + imagePullPolicy: Never + command: ["collector"] + args: ["-exchange=Kraken"] # "-mode=storeTrades|current", "-pairsfile=true|false" + env: + - name: USE_ENV + value: "true" + - name: POSTGRES_USER + value: "postgres" + - name: POSTGRES_PASSWORD + value: "password" + - name: POSTGRES_DB + value: "postgres" + - name: POSTGRES_HOST + value: "data-postgres.default.svc.cluster.local" + - name: INFLUXURL + value: "http://data-influx.default.svc.cluster.local:8086" + - name: INFLUXUSER + value: "test" + - name: INFLUXPASSWORD + value: "testtest" + - name: REDISURL + value: "data-redis.default.svc.cluster.local:6379" + - name: KAFKAURL + value: "data-kafka.default.svc.cluster.local:9094" + - name: EXEC_MODE + value: production + volumeMounts: + - name: env-context + mountPath: /mnt/env-context + readOnly: true + initContainers: + - name: check-postgres-connection + image: public.ecr.aws/docker/library/postgres:15-alpine + env: + - name: PGHOST + value: "data-postgres.default.svc.cluster.local" + - name: PGUSER + value: "postgres" + - name: PGPASSWORD + value: "password" + command: + ["sh", "-c", "until pg_isready -h $PGHOST -U $PGUSER; do sleep 1; done"] + - name: check-redis-connection + image: public.ecr.aws/docker/library/redis:7 + env: + - name: REDISCLI_URL + value: "redis://data-redis.default.svc.cluster.local:6379/0" + command: + ["sh", "-c", "until redis-cli -u $REDISCLI_URL ping; do sleep 1; done"] + - name: check-influxdb-connection + image: public.ecr.aws/docker/library/influxdb:1.8 + env: + - name: INFLUXDB_URL + value: "data-influx.default.svc.cluster.local" + - name: INFLUXDB_USERNAME + value: "test" + - name: INFLUXDB_PASSWORD + value: "testtest" + command: + [ + "sh", + "-c", + "until influx -host $INFLUXDB_URL -username $INFLUXDB_USERNAME -password $INFLUXDB_PASSWORD -execute 'SHOW DATABASES'; do sleep 1; done", + ] + - name: check-kafka-connection + image: confluentinc/cp-kafkacat:6.2.1 + env: + - name: KAFKA_BROKER_URL + value: "data-kafka.default.svc.cluster.local:9094" + command: + ["sh", "-c", "until kafkacat -b $KAFKA_BROKER_URL -L; do sleep 1; done"] + # - name: scraper-exchange-example-kraken-pairdiscovery + # image: dia.pairdiscoveryservice.dev:latest + # imagePullPolicy: Never + # command: ["pairDiscoveryService"] + # args: ["-exchange=Kraken", "-mode=verification"] + # env: + # - name: USE_ENV + # value: "true" + # - name: POSTGRES_USER + # value: "postgres" + # - name: POSTGRES_PASSWORD + # value: "password" + # - name: POSTGRES_DB + # value: "postgres" + # - name: POSTGRES_HOST + # value: "data-postgres.default.svc.cluster.local" + # - name: INFLUXURL + # value: "http://data-influx.default.svc.cluster.local:8086" + # - name: INFLUXUSER + # value: "test" + # - name: INFLUXPASSWORD + # value: "testtest" + # - name: REDISURL + # value: "data-redis.default.svc.cluster.local:6379" + # - name: KAFKAURL + # value: "data-kafka.default.svc.cluster.local:9094" + volumes: + - name: env-context + hostPath: + path: /mnt/diadata diff --git a/deployments/k8s-yaml/service-filtersblockservice.yaml b/deployments/k8s-yaml/service-filtersblockservice.yaml index a571035da..de7ccace4 100755 --- a/deployments/k8s-yaml/service-filtersblockservice.yaml +++ b/deployments/k8s-yaml/service-filtersblockservice.yaml @@ -18,7 +18,7 @@ spec: image: dia.filtersblockservice.dev:latest imagePullPolicy: Never command: ["filtersBlockService"] - args: ["-replayInflux=false", "-historical=true", "-testing=true"] + args: ["-replayInflux=false", "-historical=true", "-testing=true"] # "-testing=true|false" env: - name: USE_ENV value: "true" diff --git a/deployments/k8s-yaml/service-tradesblockservice.yaml b/deployments/k8s-yaml/service-tradesblockservice.yaml index 5cd88bc69..a98f1efb3 100755 --- a/deployments/k8s-yaml/service-tradesblockservice.yaml +++ b/deployments/k8s-yaml/service-tradesblockservice.yaml @@ -18,7 +18,7 @@ spec: image: dia.tradesblockservice.dev:latest imagePullPolicy: Never command: ["tradesBlockService"] - args: ["-historical=true", "-testing=true"] + args: ["-historical=true", "-testing=true"] # "-historical=true|false", "-testing=true|false" env: - name: USE_ENV value: "true" @@ -42,6 +42,8 @@ spec: value: "data-kafka.default.svc.cluster.local:9094" - name: TRADE_VOLUME_THRESHOLD_EXPONENT value: "4" + - name: TRADE_VOLUME_THRESHOLD_USD_EXPONENT + value: "4" initContainers: - name: check-postgres-connection image: public.ecr.aws/docker/library/postgres:15-alpine diff --git a/testenv.sh b/testenv.sh index ccbd25848..4c8cd94dd 100755 --- a/testenv.sh +++ b/testenv.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # +# Name: testenv.sh +# Vesion: 0.1.1 # Description: This script is used to manage the DIA development environment. Helpful on start/stop the platform, and create/remove resources -# Vesion: 0.1.0 SOURCE="${BASH_SOURCE[0]}" while [ -h "$SOURCE" ]; do @@ -80,21 +81,41 @@ function _build_ifnotexist(){ if [ "$arg_single_mode" = true ]; then docker_query="$(docker images -q "$2:latest" 2> /dev/null)" if [[ "${docker_query}" == "" ]]; then - echo "Image $2 is not present, building ..." - docker buildx build -f "build/$1" -t "$2:latest" . + echo "Image $2 is not present, building to local docker env ..." + docker buildx build -f "build/$1" -t "$2:latest" . > /dev/null 2>&1 else echo "Image $2 is present" fi else minikube_docker_query="$(minikube -p "${minikube_profile}" ssh -- "docker images -q $2:latest" 2> /dev/null)" if [[ "${minikube_docker_query}" == "" ]]; then - echo "Image $2 is not present, building ..." - if [[ "$minikube_driver" == "docker" ]]; then - eval "$(minikube -p "${minikube_profile}" docker-env)" - docker buildx build -f "build/$1" -t "$2:latest" . - eval "$(minikube -p "${minikube_profile}" docker-env --unset)" + if [[ -f "testenv-artifacts/$2.tar.gz" ]]; then + echo "Image tarball testenv-artifacts/$2.tar.gz exists. Loading into Minikube..." + if [[ "$minikube_driver" == "docker" ]]; then + eval "$(minikube -p "${minikube_profile}" docker-env)" + docker load -i "testenv-artifacts/$2.tar.gz" + eval "$(minikube -p "${minikube_profile}" docker-env --unset)" + else + minikube -p "${minikube_profile}" image load "testenv-artifacts/$2.tar.gz" + fi + echo "Image $2 loaded successfully" else - minikube -p "${minikube_profile}" image build -f "build/$1" -t "$2:latest" . + echo "Image $2 is not present, building to minikube docker env..." + if [[ "$minikube_driver" == "docker" ]]; then + eval "$(minikube -p "${minikube_profile}" docker-env)" + docker_build_output=$(docker buildx build -f "build/$1" -t "$2:latest" . 2>&1) + docker_build_status=$? + eval "$(minikube -p "${minikube_profile}" docker-env --unset)" + if [ $docker_build_status -ne 0 ]; then + echo "Image $2 build failed:" + echo "$docker_build_output" + exit $docker_build_status + else + echo "Image $2 build succeeded" + fi + else + minikube -p "${minikube_profile}" image build -f "build/$1" -t "$2:latest" . + fi fi else echo "Image $2 is present" @@ -203,14 +224,14 @@ function main() { # Variables local minikube_profile=dia - local minikube_k8s_version=v1.25.7 + local minikube_k8s_version=v1.28 local minikube_hw_cpus=4 local minikube_hw_ram=8g local minikube_hw_disk=50g local minikube_driver=docker local snapshot_docker_registry=https://registry.hub.docker.com/v2/ local snapshot_docker_username=dia_contributor - local snapshot_docker_password=dia_contributor_pw + local snapshot_docker_password=dia_contributor_pw # skipcq: SCT-A000 local snapshot_docker_email=dia_contributor@example.com local data_docker_registry=docker.io local data_docker_username=dia @@ -219,29 +240,28 @@ function main() { # shellcheck disable=SC1091 # TODO: this will break the script? if [ -e testenv.local ]; then - source ./testspace/.testenv.local + source ./testspace/.testenv.local fi if [[ "$arg_cpus" != "" ]]; then minikube_hw_cpus="${arg_cpus}" fi local version_detected version_detected=$(git describe --tags --abbrev=0) - declare -a demos_scraper_cex=("bitfinex" "bittrex" "coinbase" "mexc") - declare -a demos_scraper_dex=("platypus" "orca", "curve") + declare -a demos_scraper_cex=("bitfinex" "bittrex" "coinbase" "mexc" "kraken") + declare -a demos_scraper_dex=("platypus" "orca" "curve") declare -a demos_scraper_liquidity=("platypus" "orca") declare -a demos_scraper_foreign=("yahoofinance") # Command case "${command[0]}" in start) - echo "running start" + echo "Starting cluster ..." if [ "${#command[@]}" -eq 1 ]; then if [ "$arg_verbose_mode" = true ]; then _info; fi if [ "$arg_single_mode" = true ]; then echo "WIP" else if ! _minikube_profile_isrunning "${minikube_profile}"; then - echo "Starting cluster ..." minikube --profile "${minikube_profile}" start \ --kubernetes-version "${minikube_k8s_version}" \ --driver "${minikube_driver}" \ @@ -263,8 +283,8 @@ function main() { fi ;; stop) + echo "Stopping cluster ..." if [ "${#command[@]}" -eq 1 ]; then - echo "Stopping cluster ..." minikube -p "${minikube_profile}" stop else echo "Unknown command" >&2 @@ -272,8 +292,8 @@ function main() { fi ;; delete) + echo "Deleting cluster ..." if [ "${#command[@]}" -eq 1 ]; then - echo "Deleting cluster ..." minikube delete -p "${minikube_profile}" else echo "Unknown command" >&2 @@ -281,12 +301,14 @@ function main() { fi ;; build) + echo "Building images ..." if [ "${#command[@]}" -eq 1 ]; then - echo "Building images ..." _build_ifnotexist build/Dockerfile-DiadataBuild-114-Dev dia.build-114.dev _build_ifnotexist build/Dockerfile-DiadataBuild-117-Dev dia.build-117.dev _build_ifnotexist build/Dockerfile-DiadataBuild-119-Dev dia.build-119.dev _build_ifnotexist build/Dockerfile-DiadataBuild-120-Dev dia.build-120.dev + _build_ifnotexist build/Dockerfile-DiadataBuild-121-Dev dia.build-121.dev + _build_ifnotexist build/Dockerfile-DiadataBuild-122-Dev dia.build-122.dev _build_ifnotexist Dockerfile-filtersBlockService-Dev dia.filtersblockservice.dev _build_ifnotexist Dockerfile-tradesBlockService-Dev dia.tradesblockservice.dev _build_ifnotexist Dockerfile-pairDiscoveryService-Dev dia.pairdiscoveryservice.dev @@ -298,17 +320,22 @@ function main() { if [ "$arg_full_mode" = true ]; then _build_ifnotexist Dockerfile-blockchainservice-Dev dia.blockchainservice.dev fi + echo "Images built with success" else echo "Unknown command" >&2 exit 1 fi ;; install) + echo "Installing services ..." if [ "${#command[@]}" -eq 1 ]; then + echo "Checking requirements before installing services ..." _image_exist dia.build-114.dev || exit 1 _image_exist dia.build-117.dev || exit 1 _image_exist dia.build-119.dev || exit 1 _image_exist dia.build-120.dev || exit 1 + _image_exist dia.build-121.dev || exit 1 + _image_exist dia.build-122.dev || exit 1 _image_exist dia.filtersblockservice.dev || exit 1 _image_exist dia.tradesblockservice.dev || exit 1 _image_exist dia.pairdiscoveryservice.dev || exit 1 @@ -320,10 +347,11 @@ function main() { if [ "$arg_full_mode" = true ]; then _image_exist dia.blockchainservice.dev || exit 1 fi - echo "Installing services ..." + echo "Creating services via kubectl ..." minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/service-filtersblockservice.yaml minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/service-tradesblockservice.yaml minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/data-kafka.yaml + minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/data-kafka-ui.yaml minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/data-redis.yaml minikube -p "${minikube_profile}" kubectl -- create -f deployments/k8s-yaml/data-influx.yaml if [ "$arg_full_mode" = true ]; then @@ -351,6 +379,7 @@ function main() { minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/service-filtersblockservice.yaml || true minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/service-tradesblockservice.yaml || true minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/data-kafka.yaml || true + minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/data-kafka-ui.yaml || true minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/data-redis.yaml || true minikube -p "${minikube_profile}" kubectl -- delete -f deployments/k8s-yaml/data-influx.yaml || true if [ "$arg_full_mode" = true ]; then @@ -385,11 +414,11 @@ function main() { fi # Ask for variables with _EXTRACT and export them - read -p "Enter Postgres Server: " PGHOST_EXTRACT - read -p "Enter Postgres Port: " PGPORT_EXTRACT - read -p "Enter Postgres User: " PGUSER_EXTRACT - read -p "Enter Postgres Password: " PGPASSWORD_EXTRACT - read -p "Enter Postgres Database: " PGDB_EXTRACT + read -r -p "Enter Postgres Server: " PGHOST_EXTRACT + read -r -p "Enter Postgres Port: " PGPORT_EXTRACT + read -r -p "Enter Postgres User: " PGUSER_EXTRACT + read -r -s -p "Enter Postgres Password: " PGPASSWORD_EXTRACT # skipcq: SCT-A000 + read -r -p "Enter Postgres Database: " PGDB_EXTRACT export PGHOST=${PGHOST_EXTRACT} export PGUSER=${PGUSER_EXTRACT} @@ -402,20 +431,20 @@ function main() { unzip -o snapshot.zip # Run the psql commands - psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table blockchain cascade" - psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table asset cascade" - psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table exchange cascade" - psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table exchangepair cascade" - psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table pool cascade" - psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "truncate table poolasset cascade" - psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy asset FROM 'output_assets.csv' DELIMITER ';' CSV" - psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy blockchain FROM 'output_blockchain.csv' DELIMITER ';' CSV" - psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy exchangepair FROM 'output_exchangepair.csv' DELIMITER ';' CSV" - psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy exchange FROM 'output_cex.csv' DELIMITER ';' CSV" - psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy exchange FROM 'output_dex.csv' DELIMITER ';' CSV" + psql --port "${PGPORT}" --username "${PGUSER}" --dbname "${PGDB}" -c "truncate table blockchain cascade" + psql --port "${PGPORT}" --username "${PGUSER}" --dbname "${PGDB}" -c "truncate table asset cascade" + psql --port "${PGPORT}" --username "${PGUSER}" --dbname "${PGDB}" -c "truncate table exchange cascade" + psql --port "${PGPORT}" --username "${PGUSER}" --dbname "${PGDB}" -c "truncate table exchangepair cascade" + psql --port "${PGPORT}" --username "${PGUSER}" --dbname "${PGDB}" -c "truncate table pool cascade" + psql --port "${PGPORT}" --username "${PGUSER}" --dbname "${PGDB}" -c "truncate table poolasset cascade" + psql --port "${PGPORT}" --username "${PGUSER}" --dbname "${PGDB}" -c "\copy asset FROM 'output_assets.csv' DELIMITER ';' CSV" + psql --port "${PGPORT}" --username "${PGUSER}" --dbname "${PGDB}" -c "\copy blockchain FROM 'output_blockchain.csv' DELIMITER ';' CSV" + psql --port "${PGPORT}" --username "${PGUSER}" --dbname "${PGDB}" -c "\copy exchangepair FROM 'output_exchangepair.csv' DELIMITER ';' CSV" + psql --port "${PGPORT}" --username "${PGUSER}" --dbname "${PGDB}" -c "\copy exchange FROM 'output_cex.csv' DELIMITER ';' CSV" + psql --port "${PGPORT}" --username "${PGUSER}" --dbname "${PGDB}" -c "\copy exchange FROM 'output_dex.csv' DELIMITER ';' CSV" # FIXME: this will raise: ERROR: invalid input syntax for type numeric: "2023-09-07 09:05:45.012443" - # psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy poolasset FROM 'output_poolasset.csv' DELIMITER ';' CSV" - psql --port ${PGPORT} --username ${PGUSER} --dbname ${PGDB} -c "\copy pool FROM 'output_pool.csv' DELIMITER ';' CSV" + # psql --port "${PGPORT}" --username "${PGUSER}" --dbname "${PGDB}" -c "\copy poolasset FROM 'output_poolasset.csv' DELIMITER ';' CSV" + psql --port "${PGPORT}" --username "${PGUSER}" --dbname "${PGDB}" -c "\copy pool FROM 'output_pool.csv' DELIMITER ';' CSV" # Delete the output-*.sql and snapshot.zip files