Update test-enviornment build/install command, use pre-built images and add Kafka-UI

.gitignore

@@ -37,4 +37,6 @@ vendor/
 scripts/bifrost/node_modules
 scripts/bifrost/yarn.lock
 
+# Test environment
 .testenv.local
+/testenv-artifacts

build/build/Dockerfile-DiadataBuild-114-Dev

@@ -1,4 +1,4 @@
-FROM public.ecr.aws/docker/library/golang:1.14 as build
+FROM public.ecr.aws/docker/library/golang:1.14 AS build
 
 RUN apt update && apt upgrade -y
 

build/build/Dockerfile-DiadataBuild-117-Dev

@@ -1,4 +1,4 @@
-FROM public.ecr.aws/docker/library/golang:1.17 as build
+FROM public.ecr.aws/docker/library/golang:1.17 AS build
 
 RUN apt update && apt upgrade -y
 

build/build/Dockerfile-DiadataBuild-119-Dev

@@ -1,4 +1,4 @@
-FROM public.ecr.aws/docker/library/golang:1.19 as build
+FROM public.ecr.aws/docker/library/golang:1.19 AS build
 
 RUN apt update && apt upgrade -y
 

build/build/Dockerfile-DiadataBuild-120-Dev

@@ -1,4 +1,4 @@
-FROM public.ecr.aws/docker/library/golang:1.20 as build
+FROM public.ecr.aws/docker/library/golang:1.20 AS build
 
 RUN apt update && apt upgrade -y
 

build/build/Dockerfile-DiadataBuild-121-Dev

@@ -0,0 +1,13 @@
+FROM public.ecr.aws/docker/library/golang:1.21 AS build
+
+RUN apt update && apt upgrade -y
+
+COPY ./config/ /config/
+
+WORKDIR $GOPATH/src/
+COPY ./go.mod ./
+
+ENV GO111MODULE="on"
+RUN go mod download
+
+RUN go get github.com/karalabe/usb@v0.0.0-20210518091819-4ea20957c210

build/build/Dockerfile-DiadataBuild-122-Dev

@@ -0,0 +1,13 @@
+FROM public.ecr.aws/docker/library/golang:1.22 AS build
+
+RUN apt update && apt upgrade -y
+
+COPY ./config/ /config/
+
+WORKDIR $GOPATH/src/
+COPY ./go.mod ./
+
+ENV GO111MODULE="on"
+RUN go mod download
+
+RUN go get github.com/karalabe/usb@v0.0.0-20210518091819-4ea20957c210

deployments/config/pginit.sql

@@ -57,6 +57,22 @@ CREATE TABLE exchange (
     UNIQUE (name)
 );
 
+CREATE TABLE nftexchange (
+    exchange_id UUID DEFAULT gen_random_uuid(),
+    name text NOT NULL,
+    centralized boolean default false,
+    bridge boolean default false,
+    contract text,
+    blockchain text,
+    rest_api text,
+    ws_api text,
+    pairs_api text,
+    watchdog_delay numeric NOT NULL,
+    scraper_active boolean,
+    UNIQUE(exchange_id),
+    UNIQUE (name)
+);
+
 CREATE TABLE pool (
     pool_id UUID DEFAULT gen_random_uuid(),
     exchange text NOT NULL,
@@ -269,9 +285,19 @@ CREATE TABLE exchange_pairs (
 
 
 
+CREATE TABLE plans (
+    plan_id SERIAL PRIMARY KEY,
+    plan_name VARCHAR(50) NOT NULL UNIQUE,
+    plan_description TEXT,
+    plan_price NUMERIC(10, 2) NOT NULL,
+    plan_features TEXT
+);
 
+ALTER TABLE plans ADD COLUMN total_feeds integer default 3;
+ALTER TABLE plans ADD COLUMN total_oracles integer default 3;
 
- CREATE TABLE customers (
+
+CREATE TABLE customers (
     customer_id SERIAL PRIMARY KEY,
     email VARCHAR(255) NOT NULL,
     account_creation_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
@@ -400,18 +426,6 @@ CREATE TABLE loop_payment_responses (
 );
 
 
-CREATE TABLE plans (
-    plan_id SERIAL PRIMARY KEY,
-    plan_name VARCHAR(50) NOT NULL UNIQUE,
-    plan_description TEXT,
-    plan_price NUMERIC(10, 2) NOT NULL,
-    plan_features TEXT
-);
-
-ALTER TABLE plans ADD COLUMN total_feeds integer default 3;
-ALTER TABLE plans ADD COLUMN total_oracles integer default 3;
-
-
 
 
 

deployments/k8s-yaml/data-kafka-ui.yaml

@@ -0,0 +1,47 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kafka-ui
+  labels:
+    app: kafka-ui
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kafka-ui
+  template:
+    metadata:
+      labels:
+        app: kafka-ui
+    spec:
+      containers:
+        - name: kafka-ui
+          image: provectuslabs/kafka-ui:latest
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8080
+              name: http
+          env:
+            - name: KAFKA_CLUSTERS_0_NAME
+              value: local
+            - name: KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS
+              value: data-kafka.default.svc.cluster.local:9094
+            - name: KAFKA_CLUSTERS_0_ZOOKEEPER
+              value: data-kafka.default.svc.cluster.local:2181
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kafka-ui
+  labels:
+    app: kafka-ui
+spec:
+  type: NodePort
+  ports:
+    - port: 8080
+      targetPort: http
+      nodePort: 30080
+      name: http
+  selector:
+    app: kafka-ui

deployments/k8s-yaml/job-prepare.yaml

@@ -39,12 +39,14 @@ spec:
               value: "password"
             - name: POSTGRES_DB
               value: "postgres"
+            - name: REDISURL
+              value: "data-redis.default.svc.cluster.local:6379"
         - name: job-prepare-assetcollectionservice-nonethassets
           image: dia.assetcollectionservice.dev:latest
           imagePullPolicy: Never
           command: ["assetCollectionService"]
           args:
-            ["-source=assetlists", "-secret=non_eth_assets", "-caching=true"]
+            ["-source=assetlists", "-secret=non_eth_assets", "-caching=true"] # skipcq: SCT-A000
           env:
             - name: USE_ENV
               value: "true"
@@ -56,11 +58,13 @@ spec:
               value: "password"
             - name: POSTGRES_DB
               value: "postgres"
+            - name: REDISURL
+              value: "data-redis.default.svc.cluster.local:6379"
         - name: job-prepare-assetcollectionservice-fiatassets
           image: dia.assetcollectionservice.dev:latest
           imagePullPolicy: Never
           command: ["assetCollectionService"]
-          args: ["-source=assetlists", "-secret=fiat_assets", "-caching=true"]
+          args: ["-source=assetlists", "-secret=fiat_assets", "-caching=true"] # skipcq: SCT-A000
           env:
             - name: USE_ENV
               value: "true"
@@ -72,6 +76,8 @@ spec:
               value: "password"
             - name: POSTGRES_DB
               value: "postgres"
+            - name: REDISURL
+              value: "data-redis.default.svc.cluster.local:6379"
       initContainers:
         - name: check-postgres-connection
           image: public.ecr.aws/docker/library/postgres:15-alpine

deployments/k8s-yaml/scraper-cex-coinbase.yaml

@@ -8,7 +8,7 @@ spec:
       image: dia.genericcollector.dev:latest
       imagePullPolicy: Never
       command: ["collector"]
-      args: ["-exchange=CoinBase", "-mode=current", "-pairsfile=true"]
+      args: ["-exchange=CoinBase"] # "-mode=storeTrades|current", "-pairsfile=true|false"
       env:
         - name: USE_ENV
           value: "true"
@@ -30,6 +30,8 @@ spec:
           value: "data-redis.default.svc.cluster.local:6379"
         - name: KAFKAURL
           value: "data-kafka.default.svc.cluster.local:9094"
+        - name: EXEC_MODE
+          value: production
       volumeMounts:
         - name: env-context
           mountPath: /mnt/env-context

deployments/k8s-yaml/scraper-cex-kraken.yaml

@@ -0,0 +1,109 @@
+apiVersion: "v1"
+kind: Pod
+metadata:
+  name: scraper-exchange-kraken
+spec:
+  containers:
+    - name: scraper-exchange-kraken
+      image: dia.genericcollector.dev:latest
+      imagePullPolicy: Never
+      command: ["collector"]
+      args: ["-exchange=Kraken"] # "-mode=storeTrades|current", "-pairsfile=true|false"
+      env:
+        - name: USE_ENV
+          value: "true"
+        - name: POSTGRES_USER
+          value: "postgres"
+        - name: POSTGRES_PASSWORD
+          value: "password"
+        - name: POSTGRES_DB
+          value: "postgres"
+        - name: POSTGRES_HOST
+          value: "data-postgres.default.svc.cluster.local"
+        - name: INFLUXURL
+          value: "http://data-influx.default.svc.cluster.local:8086"
+        - name: INFLUXUSER
+          value: "test"
+        - name: INFLUXPASSWORD
+          value: "testtest"
+        - name: REDISURL
+          value: "data-redis.default.svc.cluster.local:6379"
+        - name: KAFKAURL
+          value: "data-kafka.default.svc.cluster.local:9094"
+        - name: EXEC_MODE
+          value: production
+      volumeMounts:
+        - name: env-context
+          mountPath: /mnt/env-context
+          readOnly: true
+  initContainers:
+    - name: check-postgres-connection
+      image: public.ecr.aws/docker/library/postgres:15-alpine
+      env:
+        - name: PGHOST
+          value: "data-postgres.default.svc.cluster.local"
+        - name: PGUSER
+          value: "postgres"
+        - name: PGPASSWORD
+          value: "password"
+      command:
+        ["sh", "-c", "until pg_isready -h $PGHOST -U $PGUSER; do sleep 1; done"]
+    - name: check-redis-connection
+      image: public.ecr.aws/docker/library/redis:7
+      env:
+        - name: REDISCLI_URL
+          value: "redis://data-redis.default.svc.cluster.local:6379/0"
+      command:
+        ["sh", "-c", "until redis-cli -u $REDISCLI_URL ping; do sleep 1; done"]
+    - name: check-influxdb-connection
+      image: public.ecr.aws/docker/library/influxdb:1.8
+      env:
+        - name: INFLUXDB_URL
+          value: "data-influx.default.svc.cluster.local"
+        - name: INFLUXDB_USERNAME
+          value: "test"
+        - name: INFLUXDB_PASSWORD
+          value: "testtest"
+      command:
+        [
+          "sh",
+          "-c",
+          "until influx -host $INFLUXDB_URL -username $INFLUXDB_USERNAME -password $INFLUXDB_PASSWORD -execute 'SHOW DATABASES'; do sleep 1; done",
+        ]
+    - name: check-kafka-connection
+      image: confluentinc/cp-kafkacat:6.2.1
+      env:
+        - name: KAFKA_BROKER_URL
+          value: "data-kafka.default.svc.cluster.local:9094"
+      command:
+        ["sh", "-c", "until kafkacat -b $KAFKA_BROKER_URL -L; do sleep 1; done"]
+    # - name: scraper-exchange-example-kraken-pairdiscovery
+    #   image: dia.pairdiscoveryservice.dev:latest
+    #   imagePullPolicy: Never
+    #   command: ["pairDiscoveryService"]
+    #   args: ["-exchange=Kraken", "-mode=verification"]
+    #   env:
+    #     - name: USE_ENV
+    #       value: "true"
+    #     - name: POSTGRES_USER
+    #       value: "postgres"
+    #     - name: POSTGRES_PASSWORD
+    #       value: "password"
+    #     - name: POSTGRES_DB
+    #       value: "postgres"
+    #     - name: POSTGRES_HOST
+    #       value: "data-postgres.default.svc.cluster.local"
+    #     - name: INFLUXURL
+    #       value: "http://data-influx.default.svc.cluster.local:8086"
+    #     - name: INFLUXUSER
+    #       value: "test"
+    #     - name: INFLUXPASSWORD
+    #       value: "testtest"
+    #     - name: REDISURL
+    #       value: "data-redis.default.svc.cluster.local:6379"
+    #     - name: KAFKAURL
+    #       value: "data-kafka.default.svc.cluster.local:9094"
+  volumes:
+    - name: env-context
+      hostPath:
+        path: /mnt/diadata

deployments/k8s-yaml/service-filtersblockservice.yaml

@@ -18,7 +18,7 @@ spec:
           image: dia.filtersblockservice.dev:latest
           imagePullPolicy: Never
           command: ["filtersBlockService"]
-          args: ["-replayInflux=false", "-historical=true", "-testing=true"]
+          args: ["-replayInflux=false", "-historical=true", "-testing=true"] # "-testing=true|false"
           env:
             - name: USE_ENV
               value: "true"

deployments/k8s-yaml/service-tradesblockservice.yaml

@@ -18,7 +18,7 @@ spec:
           image: dia.tradesblockservice.dev:latest
           imagePullPolicy: Never
           command: ["tradesBlockService"]
-          args: ["-historical=true", "-testing=true"]
+          args: ["-historical=true", "-testing=true"] # "-historical=true|false", "-testing=true|false"
           env:
             - name: USE_ENV
               value: "true"
@@ -42,6 +42,8 @@ spec:
               value: "data-kafka.default.svc.cluster.local:9094"
             - name: TRADE_VOLUME_THRESHOLD_EXPONENT
               value: "4"
+            - name: TRADE_VOLUME_THRESHOLD_USD_EXPONENT
+              value: "4"
       initContainers:
         - name: check-postgres-connection
           image: public.ecr.aws/docker/library/postgres:15-alpine