From d865737ef8351363ecdf27d152f00598861883d2 Mon Sep 17 00:00:00 2001 From: Samuel Brack Date: Wed, 28 Jan 2026 17:18:25 +0100 Subject: [PATCH 1/2] Delete unused content --- deployments/config/pginit.sql | 425 --------------- deployments/config/pgtriggers.sql | 43 -- .../config/postgres-docker-entrypoint.sh | 356 ------------ deployments/docker-compose.asset-service.yml | 134 ----- deployments/docker-compose.bitcoin-cash.yml | 46 -- deployments/docker-compose.bitcoin.yml | 43 -- deployments/docker-compose.block-scrapers.yml | 42 -- deployments/docker-compose.cardano.yml | 44 -- deployments/docker-compose.dash.yml | 46 -- deployments/docker-compose.dogecoin.yml | 42 -- deployments/docker-compose.eos.yml | 46 -- .../docker-compose.ethereum-classic.yml | 45 -- deployments/docker-compose.ethereum.yml | 25 - .../docker-compose.exchange-scrapers.yml | 510 ------------------ deployments/docker-compose.foreignscraper.yml | 64 --- deployments/docker-compose.graphql.yml | 26 - deployments/docker-compose.http.yml | 77 --- deployments/docker-compose.influxdb-2.yml | 33 -- deployments/docker-compose.influxdb.yml | 27 - .../docker-compose.influxmigration.yml | 43 -- deployments/docker-compose.kafka.yml | 57 -- deployments/docker-compose.litecoin.yml | 45 -- deployments/docker-compose.monero.yml | 36 -- deployments/docker-compose.neo.yml | 44 -- deployments/docker-compose.nftbidsscraper.yml | 41 -- .../docker-compose.nftcollection-service.yml | 49 -- deployments/docker-compose.nftdatascraper.yml | 88 --- .../docker-compose.nftofferscraper.yml | 55 -- .../docker-compose.nfttradesscraper.yml | 83 --- deployments/docker-compose.postgres.yml | 26 - deployments/docker-compose.ratescrapers.yml | 90 ---- deployments/docker-compose.redis.yml | 22 - deployments/docker-compose.ripple.yml | 25 - deployments/docker-compose.services.yml | 174 ------ deployments/docker-compose.stellar.yml | 25 - deployments/docker-compose.stockscrapers.yml | 45 -- deployments/docker-compose.tether.yml | 25 - deployments/docker-compose.tezos.yml | 40 -- deployments/docker-compose.tron.yml | 46 -- deployments/k8s-yaml/cronjob-snapshot.yaml | 118 ---- deployments/k8s-yaml/data-influx.yaml | 55 -- deployments/k8s-yaml/data-kafka.yaml | 72 --- .../k8s-yaml/data-postgres-prepopulated.yaml | 45 -- deployments/k8s-yaml/data-postgres.yaml | 52 -- deployments/k8s-yaml/data-redis.yaml | 36 -- deployments/k8s-yaml/job-ping-influx.yaml | 26 - deployments/k8s-yaml/job-ping-kafka.yaml | 17 - deployments/k8s-yaml/job-ping-postgres.yaml | 21 - deployments/k8s-yaml/job-ping-redis.yaml | 17 - deployments/k8s-yaml/job-prepare.yaml | 129 ----- .../k8s-yaml/scraper-cex-bitfinex.yaml | 107 ---- deployments/k8s-yaml/scraper-cex-bitmart.yaml | 107 ---- deployments/k8s-yaml/scraper-cex-bittrex.yaml | 107 ---- .../k8s-yaml/scraper-cex-coinbase.yaml | 107 ---- deployments/k8s-yaml/scraper-cex-mexc.yaml | 107 ---- deployments/k8s-yaml/scraper-cex.yaml | 107 ---- deployments/k8s-yaml/scraper-dex-curve.yaml | 85 --- deployments/k8s-yaml/scraper-dex-orca.yaml | 83 --- .../k8s-yaml/scraper-dex-platypus.yaml | 81 --- deployments/k8s-yaml/scraper-dex.yaml | 81 --- .../k8s-yaml/service-filtersblockservice.yaml | 95 ---- .../k8s-yaml/service-supplyservice.yaml | 62 --- .../k8s-yaml/service-tradesblockservice.yaml | 97 ---- deployments/local/docker-compose.yml | 91 ---- deployments/local/local.env | 11 - frontend/derivatives/VIX.js | 133 ----- frontend/derivatives/index.html | 24 - frontend/interestRates/SOFR.js | 165 ------ frontend/interestRates/SOFR_old.js | 134 ----- .../interestRates/createCharts/ownChart.html | 32 -- .../interestRates/createCharts/ownChart.js | 166 ------ .../createCharts/ownChart_v2.html | 100 ---- .../interestRates/createCharts/ownChart_v2.js | 166 ------ .../createCharts/ownChart_v3.html | 100 ---- .../interestRates/createCharts/ownChart_v3.js | 262 --------- frontend/interestRates/index.html | 27 - oracle/diaoracle.sol | 47 -- 77 files changed, 6405 deletions(-) delete mode 100644 deployments/config/pginit.sql delete mode 100644 deployments/config/pgtriggers.sql delete mode 100755 deployments/config/postgres-docker-entrypoint.sh delete mode 100644 deployments/docker-compose.asset-service.yml delete mode 100644 deployments/docker-compose.bitcoin-cash.yml delete mode 100644 deployments/docker-compose.bitcoin.yml delete mode 100644 deployments/docker-compose.block-scrapers.yml delete mode 100644 deployments/docker-compose.cardano.yml delete mode 100644 deployments/docker-compose.dash.yml delete mode 100644 deployments/docker-compose.dogecoin.yml delete mode 100644 deployments/docker-compose.eos.yml delete mode 100644 deployments/docker-compose.ethereum-classic.yml delete mode 100644 deployments/docker-compose.ethereum.yml delete mode 100644 deployments/docker-compose.exchange-scrapers.yml delete mode 100644 deployments/docker-compose.foreignscraper.yml delete mode 100644 deployments/docker-compose.graphql.yml delete mode 100644 deployments/docker-compose.http.yml delete mode 100644 deployments/docker-compose.influxdb-2.yml delete mode 100644 deployments/docker-compose.influxdb.yml delete mode 100644 deployments/docker-compose.influxmigration.yml delete mode 100644 deployments/docker-compose.kafka.yml delete mode 100644 deployments/docker-compose.litecoin.yml delete mode 100644 deployments/docker-compose.monero.yml delete mode 100644 deployments/docker-compose.neo.yml delete mode 100644 deployments/docker-compose.nftbidsscraper.yml delete mode 100644 deployments/docker-compose.nftcollection-service.yml delete mode 100644 deployments/docker-compose.nftdatascraper.yml delete mode 100644 deployments/docker-compose.nftofferscraper.yml delete mode 100644 deployments/docker-compose.nfttradesscraper.yml delete mode 100644 deployments/docker-compose.postgres.yml delete mode 100644 deployments/docker-compose.ratescrapers.yml delete mode 100644 deployments/docker-compose.redis.yml delete mode 100644 deployments/docker-compose.ripple.yml delete mode 100644 deployments/docker-compose.services.yml delete mode 100644 deployments/docker-compose.stellar.yml delete mode 100644 deployments/docker-compose.stockscrapers.yml delete mode 100644 deployments/docker-compose.tether.yml delete mode 100644 deployments/docker-compose.tezos.yml delete mode 100644 deployments/docker-compose.tron.yml delete mode 100755 deployments/k8s-yaml/cronjob-snapshot.yaml delete mode 100755 deployments/k8s-yaml/data-influx.yaml delete mode 100755 deployments/k8s-yaml/data-kafka.yaml delete mode 100755 deployments/k8s-yaml/data-postgres-prepopulated.yaml delete mode 100755 deployments/k8s-yaml/data-postgres.yaml delete mode 100755 deployments/k8s-yaml/data-redis.yaml delete mode 100755 deployments/k8s-yaml/job-ping-influx.yaml delete mode 100755 deployments/k8s-yaml/job-ping-kafka.yaml delete mode 100755 deployments/k8s-yaml/job-ping-postgres.yaml delete mode 100755 deployments/k8s-yaml/job-ping-redis.yaml delete mode 100755 deployments/k8s-yaml/job-prepare.yaml delete mode 100755 deployments/k8s-yaml/scraper-cex-bitfinex.yaml delete mode 100755 deployments/k8s-yaml/scraper-cex-bitmart.yaml delete mode 100755 deployments/k8s-yaml/scraper-cex-bittrex.yaml delete mode 100755 deployments/k8s-yaml/scraper-cex-coinbase.yaml delete mode 100755 deployments/k8s-yaml/scraper-cex-mexc.yaml delete mode 100755 deployments/k8s-yaml/scraper-cex.yaml delete mode 100755 deployments/k8s-yaml/scraper-dex-curve.yaml delete mode 100755 deployments/k8s-yaml/scraper-dex-orca.yaml delete mode 100755 deployments/k8s-yaml/scraper-dex-platypus.yaml delete mode 100755 deployments/k8s-yaml/scraper-dex.yaml delete mode 100755 deployments/k8s-yaml/service-filtersblockservice.yaml delete mode 100755 deployments/k8s-yaml/service-supplyservice.yaml delete mode 100755 deployments/k8s-yaml/service-tradesblockservice.yaml delete mode 100644 deployments/local/docker-compose.yml delete mode 100644 deployments/local/local.env delete mode 100644 frontend/derivatives/VIX.js delete mode 100644 frontend/derivatives/index.html delete mode 100644 frontend/interestRates/SOFR.js delete mode 100644 frontend/interestRates/SOFR_old.js delete mode 100644 frontend/interestRates/createCharts/ownChart.html delete mode 100644 frontend/interestRates/createCharts/ownChart.js delete mode 100644 frontend/interestRates/createCharts/ownChart_v2.html delete mode 100644 frontend/interestRates/createCharts/ownChart_v2.js delete mode 100644 frontend/interestRates/createCharts/ownChart_v3.html delete mode 100644 frontend/interestRates/createCharts/ownChart_v3.js delete mode 100644 frontend/interestRates/index.html delete mode 100644 oracle/diaoracle.sol diff --git a/deployments/config/pginit.sql b/deployments/config/pginit.sql deleted file mode 100644 index 677471a59..000000000 --- a/deployments/config/pginit.sql +++ /dev/null @@ -1,425 +0,0 @@ -CREATE EXTENSION "pgcrypto"; - - --- Table asset is the single source of truth for all assets handled at DIA. --- If a field is not case sensitive (such as address for Ethereum) it should --- be all lowercase for consistency reasons. --- Otherwise it must be as defined in the underlying contract. -CREATE TABLE asset ( - asset_id UUID DEFAULT gen_random_uuid(), - symbol text NOT NULL, - name text NOT NULL, - decimals text, - blockchain text, - address text NOT NULL, - UNIQUE (asset_id), - UNIQUE (address, blockchain) -); - --- Table exchangepair holds all trading pairs for the pair scrapers. --- The format has to be the same as emitted by the exchange's API in order --- for the pair scrapers to be able to scrape trading data from the API. -CREATE TABLE exchangepair ( - exchangepair_id UUID DEFAULT gen_random_uuid(), - symbol text NOT NULL, - foreignname text NOT NULL, - exchange text NOT NULL, - UNIQUE (foreignname, exchange), - -- These fields reference asset table and should be verified by pairdiscoveryservice. - -- Only trades with verified pairs are processed further and thereby enter price calculation. - verified boolean default false, - id_quotetoken UUID REFERENCES asset(asset_id), - id_basetoken UUID REFERENCES asset(asset_id) -); - -CREATE TABLE exchangesymbol ( - exchangesymbol_id UUID DEFAULT gen_random_uuid(), - symbol text NOT NULL, - exchange text NOT NULL, - UNIQUE (symbol,exchange), - verified boolean default false, - asset_id UUID REFERENCES asset(asset_id) -); - -CREATE TABLE exchange ( - exchange_id UUID DEFAULT gen_random_uuid(), - name text NOT NULL, - centralized boolean default false, - bridge boolean default false, - contract text, - blockchain text, - rest_api text, - ws_api text, - pairs_api text, - watchdog_delay numeric NOT NULL, - scraper_active boolean, - UNIQUE(exchange_id), - UNIQUE (name) -); - -CREATE TABLE pool ( - pool_id UUID DEFAULT gen_random_uuid(), - exchange text NOT NULL, - blockchain text NOT NULL, - address text NOT NULL, - UNIQUE (pool_id), - UNIQUE (blockchain,address) -); - -CREATE TABLE poolasset ( - poolasset_id UUID DEFAULT gen_random_uuid(), - pool_id UUID REFERENCES pool(pool_id) NOT NULL, - asset_id UUID REFERENCES asset(asset_id) NOT NULL, - liquidity numeric, - liquidity_usd numeric, - time_stamp timestamp, - token_index integer, - UNIQUE (poolasset_id), - UNIQUE(pool_id,asset_id) -); - -CREATE TABLE scraper_cronjob_state ( - scraper_cronjob_state_id UUID DEFAULT gen_random_uuid(), - scraper text NOT NULL, - index_type text NOT NULL, - index_value numeric, - UNIQUE(scraper_cronjob_state_id), - UNIQUE(scraper,index_type) -); - -CREATE TABLE chainconfig ( - chain_config_id UUID DEFAULT gen_random_uuid(), - rpcurl text NOT NULL, - wsurl text NOT NULL, - chainID text NOT NULL, - UNIQUE (chainID) -); - --- blockchain table stores all blockchains available in our databases -CREATE TABLE blockchain ( - blockchain_id UUID DEFAULT gen_random_uuid(), - name text NOT NULL, - genesisdate numeric, - nativetoken_id UUID REFERENCES asset(asset_id), - verificationmechanism text, - chain_id text, - UNIQUE(blockchain_id), - UNIQUE(name) -); - -CREATE TABLE assetvolume ( - asset_id UUID primary key, - volume decimal, - time_stamp timestamp -); - --- polling table stores data - required for HTTP polling -CREATE TABLE polling ( - polling_id UUID DEFAULT gen_random_uuid(), - blockchain text, - contract_address text NOT NULL, - page numeric DEFAULT 1, - UNIQUE(blockchain, contract_address) -); - --- historicalquotation collects USD quotes with lower frequency --- for a selection of assets. -CREATE TABLE historicalquotation ( - historicalquotation_id UUID DEFAULT gen_random_uuid(), - asset_id UUID REFERENCES asset(asset_id) NOT NULL, - price numeric, - quote_time timestamp, - source text, - UNIQUE(asset_id,quote_time,source), - UNIQUE(historicalquotation_id) -); - -CREATE TABLE IF NOT EXISTS scrapers ( - name character varying(255) NOT NULL, - conf json, - state json, - CONSTRAINT pk_scrapers PRIMARY KEY(name) -); - -CREATE TABLE blockdata ( - blockdata_id UUID DEFAULT gen_random_uuid(), - blockchain text NOT NULL, - block_number numeric NOT NULL, - block_data jsonb, - UNIQUE(blockchain, block_number), - UNIQUE(blockdata_id) -); - -CREATE TABLE assetpriceident ( - priceident_id UUID DEFAULT gen_random_uuid(), - asset_id UUID REFERENCES asset(asset_id), - group_id numeric NOT NULL, - rank_in_group numeric NOT NULL, - UNIQUE(asset_id), - UNIQUE(group_id, rank_in_group) -); - - -CREATE TABLE oracleconfig ( - id uuid DEFAULT gen_random_uuid(), - address text NOT NULL, - feeder_id text NOT NULL, - owner text NOT NULL, - symbols text NOT NULL, - chainid text NOT NULL, - active boolean DEFAULT true, - frequency text, - sleepseconds text, - deviationpermille text, - blockchainnode text DEFAULT ''::text, - feeder_address text, - mandatory_frequency text, - deleted boolean DEFAULT false, - createddate timestamp without time zone DEFAULT now() NOT NULL, - lastupdate timestamp without time zone, - creation_block bigint, - creation_block_time timestamp without time zone DEFAULT '1970-01-01 00:00:00'::timestamp without time zone, - feedselection text, - expired boolean DEFAULT false, - expired_time timestamp without time zone DEFAULT '1970-01-01 00:00:00'::timestamp without time zone -); - -ALTER TABLE oracleconfig ADD COLUMN name VARCHAR(255); -ALTER TABLE oracleconfig ADD COLUMN draft boolean DEFAULT true; -ALTER TABLE oracleconfig ADD COLUMN customer_id int ; -ALTER TABLE oracleconfig ADD COLUMN billable boolean DEFAULT false ; - - - - -ALTER TABLE oracleconfig ADD COLUMN ecosystem boolean DEFAULT false; - - - -ALTER TABLE oracleconfig -ADD CONSTRAINT unique_customer_chainid_address -UNIQUE (customer_id, chainid, address); - - - -ALTER TABLE oracleconfig -ADD CONSTRAINT unique_feeder_id UNIQUE (feeder_id); - - --- CREATE TABLE oracleconfig ( --- id UUID DEFAULT gen_random_uuid(), --- address text NOT NULL, --- feeder_id text NOT NULL, --- owner text NOT NULL, --- symbols text NOT NULL, --- feeder_address text NOT NULL, --- chainID text NOT NULL, --- active boolean default true, --- deleted boolean default false, --- frequency text , --- sleepseconds text, --- deviationpermille text, --- blockchainnode text, --- mandatory_frequency text, --- createddate TIMESTAMP NOT NULL DEFAULT NOW(), --- lastupdate TIMESTAMP NOT NULL, --- UNIQUE (id), --- UNIQUE (feeder_id) --- ); - --- ALTER TABLE oracleconfig ADD COLUMN creation_block_time TIMESTAMP DEFAULT 'epoch'::timestamp; --- ALTER TABLE oracleconfig ADD COLUMN feedSelection TEXT ; --- ALTER TABLE oracleconfig ADD COLUMN expired boolean default false ; --- ALTER TABLE oracleconfig ADD COLUMN expired_time TIMESTAMP DEFAULT 'epoch'::timestamp; - - - - -CREATE TABLE feederresource ( - id SERIAL PRIMARY KEY, - owner text NOT NULL, - total numeric NOT NULL, - UNIQUE (id), - UNIQUE (owner) -); - -CREATE TABLE asset_list ( - id SERIAL PRIMARY KEY, - asset_name VARCHAR(255) NOT NULL, - custom_name VARCHAR(255), - symbol VARCHAR(50), - methodology TEXT, - list_name TEXT - -); - -CREATE TABLE exchange_list ( - id SERIAL PRIMARY KEY, - name VARCHAR(255) NOT NULL, - asset_id INT REFERENCES asset_list(id) ON DELETE CASCADE -); - -CREATE TABLE exchange_pairs ( - id SERIAL PRIMARY KEY, - exchange_id INT REFERENCES exchange_list(id) ON DELETE CASCADE, - pair VARCHAR(255) NOT NULL -); - - - - - - - - CREATE TABLE customers ( - customer_id SERIAL PRIMARY KEY, - email VARCHAR(255) NOT NULL, - account_creation_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - customer_plan INTEGER REFERENCES plans(plan_id) ON DELETE SET NULL, - deployed_oracles INTEGER DEFAULT 0, - payment_status VARCHAR(50), - last_payment TIMESTAMP, - payment_source VARCHAR(255), - number_of_data_feeds INTEGER DEFAULT 0, - active BOOLEAN DEFAULT TRUE -); - - - -ALTER TABLE customers ADD COLUMN name VARCHAR(255); -ALTER TABLE customers ADD COLUMN payer_address text; - - - -CREATE TABLE wallet_public_keys ( - key_id SERIAL PRIMARY KEY, - customer_id INTEGER REFERENCES customers(customer_id) ON DELETE CASCADE, - public_key TEXT NOT NULL, - access_level VARCHAR(50) NOT NULL DEFAULT 'read_write', - UNIQUE (public_key), - CONSTRAINT check_access_level CHECK (access_level IN ('read', 'read_write')) - -); - - - -ALTER TABLE wallet_public_keys ADD COLUMN username VARCHAR(255) UNIQUE; - -CREATE TABLE wallet_public_keys_temp ( - key_id SERIAL PRIMARY KEY, - customer_id INTEGER REFERENCES customers(customer_id) ON DELETE CASCADE, - public_key TEXT NOT NULL, - access_level VARCHAR(50) NOT NULL DEFAULT 'read_write', - username VARCHAR(255) - ); - - ALTER TABLE wallet_public_keys_temp add invitor TEXT; - - ALTER TABLE wallet_public_keys_temp -ADD CONSTRAINT unique_customer_public_key UNIQUE (customer_id, public_key); - - -CREATE TABLE transfer_created ( - event VARCHAR(50), - transaction VARCHAR(50), - network_id INT, - network_name VARCHAR(50), - contract_address VARCHAR(42), - email VARCHAR(255), - company TEXT, - parent VARCHAR(50), - id UUID, - invoice_id VARCHAR(50), - bill_date TIMESTAMP, - to_address VARCHAR(42), - from_address VARCHAR(42), - token_symbol VARCHAR(10), - token_address VARCHAR(42), - payment_type VARCHAR(50), - usd BOOLEAN, - amount NUMERIC(10, 2), - item VARCHAR(255), - item_id INT, - source VARCHAR(50), - batch_id UUID, - transfer_id UUID, - ref_id VARCHAR(255), - agreement_id UUID -); - -CREATE TABLE loop_payment_transfer_processed ( - event VARCHAR(255) NOT NULL, - transaction VARCHAR(255) NOT NULL, - network_id INTEGER NOT NULL, - network_name VARCHAR(255) NOT NULL, - contract_address VARCHAR(255) NOT NULL, - email VARCHAR(255), - company VARCHAR(255), - parent VARCHAR(255), - transfer_id VARCHAR(255) NOT NULL, - success BOOLEAN NOT NULL, - payment_token_address VARCHAR(255), - payment_token_symbol VARCHAR(255), - end_user VARCHAR(255), - reason VARCHAR(255), - invoice_id VARCHAR(255), - amount_paid DOUBLE PRECISION, - agreement_id VARCHAR(255), - ref_id VARCHAR(255), - batch_id VARCHAR(255), - usd_amount VARCHAR(255) -); - -CREATE TABLE loop_payment_responses ( - id SERIAL PRIMARY KEY, - event TEXT, - transaction TEXT, - network_id INT, - network_name TEXT, - contract_address TEXT, - email TEXT, - company TEXT, - parent TEXT, - subscriber TEXT, - item TEXT, - item_id TEXT, - agreement_id TEXT, - agreement_amount TEXT, - frequency_number INT, - frequency_unit TEXT, - add_on_agreements TEXT, - add_on_items TEXT, - add_on_item_ids TEXT, - add_on_total_amount TEXT, - payment_token_symbol TEXT, - payment_token_address TEXT, - event_date INT, - ref_id TEXT, - invoice_id TEXT, - metadata JSONB DEFAULT '{}'::jsonb -); - - -CREATE TABLE plans ( - plan_id SERIAL PRIMARY KEY, - plan_name VARCHAR(50) NOT NULL UNIQUE, - plan_description TEXT, - plan_price NUMERIC(10, 2) NOT NULL, - plan_features TEXT -); - -ALTER TABLE plans ADD COLUMN total_feeds integer default 3; -ALTER TABLE plans ADD COLUMN total_oracles integer default 3; - - - - - -INSERT INTO "plans"("plan_id","plan_name","plan_description","plan_price","plan_features","total_feeds") -VALUES -(1,E'Plan 2',E'default',0,E'desc',10); - -INSERT INTO "plans"("plan_id","plan_name","plan_description","plan_price","plan_features","total_feeds") -VALUES -(2,E'Plan 1',E'default',0,E'desc',3); - diff --git a/deployments/config/pgtriggers.sql b/deployments/config/pgtriggers.sql deleted file mode 100644 index c478a3cc6..000000000 --- a/deployments/config/pgtriggers.sql +++ /dev/null @@ -1,43 +0,0 @@ -CREATE OR REPLACE FUNCTION ensure_mandatory_read_write() -RETURNS TRIGGER AS $$ -BEGIN - IF TG_OP = 'DELETE' AND OLD.access_level = 'read_write' THEN - IF NOT EXISTS ( - SELECT 1 - FROM wallet_public_keys - WHERE customer_id = OLD.customer_id - AND access_level = 'read_write' - AND key_id != OLD.key_id - ) THEN - RAISE EXCEPTION 'Customer % must have at least one read_write access.', OLD.customer_id; - END IF; - END IF; - - IF TG_OP = 'UPDATE' AND OLD.access_level = 'read_write' AND NEW.access_level != 'read_write' THEN - IF NOT EXISTS ( - SELECT 1 - FROM wallet_public_keys - WHERE customer_id = OLD.customer_id - AND access_level = 'read_write' - AND key_id != OLD.key_id - ) THEN - RAISE EXCEPTION 'Customer % must have at least one read_write access.', OLD.customer_id; - END IF; - END IF; - - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - - -CREATE TRIGGER check_delete_read_write -BEFORE DELETE ON wallet_public_keys -FOR EACH ROW -EXECUTE FUNCTION ensure_mandatory_read_write(); - - - -CREATE TRIGGER check_update_read_write -BEFORE UPDATE ON wallet_public_keys -FOR EACH ROW -EXECUTE FUNCTION ensure_mandatory_read_write(); \ No newline at end of file diff --git a/deployments/config/postgres-docker-entrypoint.sh b/deployments/config/postgres-docker-entrypoint.sh deleted file mode 100755 index beaf7421d..000000000 --- a/deployments/config/postgres-docker-entrypoint.sh +++ /dev/null @@ -1,356 +0,0 @@ -#!/usr/bin/env bash -echo "Starting entrypoint script (${BASH_SOURCE[0]}) ..." -echo "Current user: $(whoami)" -echo "Current working directory: $(pwd)" -echo; echo "Injected scripts /docker-entrypoint-initdb.d/:" -ls -la /docker-entrypoint-initdb.d/ -echo; echo "Injected volumes (/mnt/):" -ls -la /mnt/ -set -Eeo pipefail -# TODO swap to -Eeuo pipefail above (after handling all potentially-unset variables) - -# usage: file_env VAR [DEFAULT] -# ie: file_env 'XYZ_DB_PASSWORD' 'example' -# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of -# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) -file_env() { - local var="$1" - local fileVar="${var}_FILE" - local def="${2:-}" - if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then - printf >&2 'error: both %s and %s are set (but are exclusive)\n' "$var" "$fileVar" - exit 1 - fi - local val="$def" - if [ "${!var:-}" ]; then - val="${!var}" - elif [ "${!fileVar:-}" ]; then - val="$(< "${!fileVar}")" - fi - export "$var"="$val" - unset "$fileVar" -} - -# check to see if this file is being run or sourced from another script -_is_sourced() { - # https://unix.stackexchange.com/a/215279 - [ "${#FUNCNAME[@]}" -ge 2 ] \ - && [ "${FUNCNAME[0]}" = '_is_sourced' ] \ - && [ "${FUNCNAME[1]}" = 'source' ] -} - -# used to create initial postgres directories and if run as root, ensure ownership to the "postgres" user -docker_create_db_directories() { - local user; user="$(id -u)" - - mkdir -p "$PGDATA" - # ignore failure since there are cases where we can't chmod (and PostgreSQL might fail later anyhow - it's picky about permissions of this directory) - chmod 00700 "$PGDATA" || : - - # ignore failure since it will be fine when using the image provided directory; see also https://github.com/docker-library/postgres/pull/289 - mkdir -p /var/run/postgresql || : - chmod 03775 /var/run/postgresql || : - - # Create the transaction log directory before initdb is run so the directory is owned by the correct user - if [ -n "${POSTGRES_INITDB_WALDIR:-}" ]; then - mkdir -p "$POSTGRES_INITDB_WALDIR" - if [ "$user" = '0' ]; then - find "$POSTGRES_INITDB_WALDIR" \! -user postgres -exec chown postgres '{}' + - fi - chmod 700 "$POSTGRES_INITDB_WALDIR" - fi - - # allow the container to be started with `--user` - if [ "$user" = '0' ]; then - find "$PGDATA" \! -user postgres -exec chown postgres '{}' + - find /var/run/postgresql \! -user postgres -exec chown postgres '{}' + - fi -} - -# initialize empty PGDATA directory with new database via 'initdb' -# arguments to `initdb` can be passed via POSTGRES_INITDB_ARGS or as arguments to this function -# `initdb` automatically creates the "postgres", "template0", and "template1" dbnames -# this is also where the database user is created, specified by `POSTGRES_USER` env -docker_init_database_dir() { - # "initdb" is particular about the current user existing in "/etc/passwd", so we use "nss_wrapper" to fake that if necessary - # see https://github.com/docker-library/postgres/pull/253, https://github.com/docker-library/postgres/issues/359, https://cwrap.org/nss_wrapper.html - local uid; uid="$(id -u)" - if ! getent passwd "$uid" &> /dev/null; then - # see if we can find a suitable "libnss_wrapper.so" (https://salsa.debian.org/sssd-team/nss-wrapper/-/commit/b9925a653a54e24d09d9b498a2d913729f7abb15) - local wrapper - for wrapper in {/usr,}/lib{/*,}/libnss_wrapper.so; do - if [ -s "$wrapper" ]; then - NSS_WRAPPER_PASSWD="$(mktemp)" - NSS_WRAPPER_GROUP="$(mktemp)" - export LD_PRELOAD="$wrapper" NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP - local gid; gid="$(id -g)" - printf 'postgres:x:%s:%s:PostgreSQL:%s:/bin/false\n' "$uid" "$gid" "$PGDATA" > "$NSS_WRAPPER_PASSWD" - printf 'postgres:x:%s:\n' "$gid" > "$NSS_WRAPPER_GROUP" - break - fi - done - fi - - if [ -n "${POSTGRES_INITDB_WALDIR:-}" ]; then - set -- --waldir "$POSTGRES_INITDB_WALDIR" "$@" - fi - - # --pwfile refuses to handle a properly-empty file (hence the "\n"): https://github.com/docker-library/postgres/issues/1025 - eval 'initdb --username="$POSTGRES_USER" --pwfile=<(printf "%s\n" "$POSTGRES_PASSWORD") '"$POSTGRES_INITDB_ARGS"' "$@"' - - # unset/cleanup "nss_wrapper" bits - if [[ "${LD_PRELOAD:-}" == */libnss_wrapper.so ]]; then - rm -f "$NSS_WRAPPER_PASSWD" "$NSS_WRAPPER_GROUP" - unset LD_PRELOAD NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP - fi -} - -# print large warning if POSTGRES_PASSWORD is long -# error if both POSTGRES_PASSWORD is empty and POSTGRES_HOST_AUTH_METHOD is not 'trust' -# print large warning if POSTGRES_HOST_AUTH_METHOD is set to 'trust' -# assumes database is not set up, ie: [ -z "$DATABASE_ALREADY_EXISTS" ] -docker_verify_minimum_env() { - # check password first so we can output the warning before postgres - # messes it up - if [ "${#POSTGRES_PASSWORD}" -ge 100 ]; then - cat >&2 <<-'EOWARN' - - WARNING: The supplied POSTGRES_PASSWORD is 100+ characters. - - This will not work if used via PGPASSWORD with "psql". - - https://www.postgresql.org/message-id/flat/E1Rqxp2-0004Qt-PL%40wrigleys.postgresql.org (BUG #6412) - https://github.com/docker-library/postgres/issues/507 - - EOWARN - fi - if [ -z "$POSTGRES_PASSWORD" ] && [ 'trust' != "$POSTGRES_HOST_AUTH_METHOD" ]; then - # The - option suppresses leading tabs but *not* spaces. :) - cat >&2 <<-'EOE' - Error: Database is uninitialized and superuser password is not specified. - You must specify POSTGRES_PASSWORD to a non-empty value for the - superuser. For example, "-e POSTGRES_PASSWORD=password" on "docker run". - - You may also use "POSTGRES_HOST_AUTH_METHOD=trust" to allow all - connections without a password. This is *not* recommended. - - See PostgreSQL documentation about "trust": - https://www.postgresql.org/docs/current/auth-trust.html - EOE - exit 1 - fi - if [ 'trust' = "$POSTGRES_HOST_AUTH_METHOD" ]; then - cat >&2 <<-'EOWARN' - ******************************************************************************** - WARNING: POSTGRES_HOST_AUTH_METHOD has been set to "trust". This will allow - anyone with access to the Postgres port to access your database without - a password, even if POSTGRES_PASSWORD is set. See PostgreSQL - documentation about "trust": - https://www.postgresql.org/docs/current/auth-trust.html - In Docker's default configuration, this is effectively any other - container on the same system. - - It is not recommended to use POSTGRES_HOST_AUTH_METHOD=trust. Replace - it with "-e POSTGRES_PASSWORD=password" instead to set a password in - "docker run". - ******************************************************************************** - EOWARN - fi -} - -# usage: docker_process_init_files [file [file [...]]] -# ie: docker_process_init_files /always-initdb.d/* -# process initializer files, based on file extensions and permissions -docker_process_init_files() { - # psql here for backwards compatibility "${psql[@]}" - psql=( docker_process_sql ) - - printf '\n' - local f - for f; do - case "$f" in - *.sh) - # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 - # https://github.com/docker-library/postgres/pull/452 - if [ -x "$f" ]; then - printf '%s: running %s\n' "$0" "$f" - "$f" - else - printf '%s: sourcing %s\n' "$0" "$f" - . "$f" - fi - ;; - *.sql) printf '%s: running %s\n' "$0" "$f"; docker_process_sql -f "$f"; printf '\n' ;; - *.sql.gz) printf '%s: running %s\n' "$0" "$f"; gunzip -c "$f" | docker_process_sql; printf '\n' ;; - *.sql.xz) printf '%s: running %s\n' "$0" "$f"; xzcat "$f" | docker_process_sql; printf '\n' ;; - *.sql.zst) printf '%s: running %s\n' "$0" "$f"; zstd -dc "$f" | docker_process_sql; printf '\n' ;; - *) printf '%s: ignoring %s\n' "$0" "$f" ;; - esac - printf '\n' - done -} - -# Execute sql script, passed via stdin (or -f flag of pqsl) -# usage: docker_process_sql [psql-cli-args] -# ie: docker_process_sql --dbname=mydb <<<'INSERT ...' -# ie: docker_process_sql -f my-file.sql -# ie: docker_process_sql > "$PGDATA/pg_hba.conf" -} - -# start socket-only postgresql server for setting up or running scripts -# all arguments will be passed along as arguments to `postgres` (via pg_ctl) -docker_temp_server_start() { - if [ "$1" = 'postgres' ]; then - shift - fi - - # internal start of server in order to allow setup using psql client - # does not listen on external TCP/IP and waits until start finishes - set -- "$@" -c listen_addresses='' -p "${PGPORT:-5432}" - - PGUSER="${PGUSER:-$POSTGRES_USER}" \ - pg_ctl -D "$PGDATA" \ - -o "$(printf '%q ' "$@")" \ - -w start -} - -# stop postgresql server after done setting up user and running scripts -docker_temp_server_stop() { - PGUSER="${PGUSER:-postgres}" \ - pg_ctl -D "$PGDATA" -m fast -w stop -} - -# check arguments for an option that would cause postgres to stop -# return true if there is one -_pg_want_help() { - local arg - for arg; do - case "$arg" in - # postgres --help | grep 'then exit' - # leaving out -C on purpose since it always fails and is unhelpful: - # postgres: could not access the server configuration file "/var/lib/postgresql/data/postgresql.conf": No such file or directory - -'?'|--help|--describe-config|-V|--version) - return 0 - ;; - esac - done - return 1 -} - -_main() { - # if first arg looks like a flag, assume we want to run postgres server - if [ "${1:0:1}" = '-' ]; then - set -- postgres "$@" - fi - - if [ "$1" = 'postgres' ] && ! _pg_want_help "$@"; then - docker_setup_env - # setup data directories and permissions (when run as root) - docker_create_db_directories - if [ "$(id -u)" = '0' ]; then - # then restart script as postgres user - exec su-exec postgres "$BASH_SOURCE" "$@" - fi - - # only run initialization on an empty data directory - if [ -z "$DATABASE_ALREADY_EXISTS" ]; then - docker_verify_minimum_env - - # check dir permissions to reduce likelihood of half-initialized database - ls /docker-entrypoint-initdb.d/ > /dev/null - - docker_init_database_dir - pg_setup_hba_conf "$@" - - # PGPASSWORD is required for psql when authentication is required for 'local' connections via pg_hba.conf and is otherwise harmless - # e.g. when '--auth=md5' or '--auth-local=md5' is used in POSTGRES_INITDB_ARGS - export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}" - docker_temp_server_start "$@" - - docker_setup_db - docker_process_init_files /docker-entrypoint-initdb.d/* - - docker_temp_server_stop - unset PGPASSWORD - - cat <<-'EOM' - - PostgreSQL init process complete; ready for start up. - - EOM - else - cat <<-'EOM' - - PostgreSQL Database directory appears to contain a database; Skipping initialization - - EOM - fi - fi -} - -if ! _is_sourced; then - _main "$@" -fi \ No newline at end of file diff --git a/deployments/docker-compose.asset-service.yml b/deployments/docker-compose.asset-service.yml deleted file mode 100644 index 4b057e522..000000000 --- a/deployments/docker-compose.asset-service.yml +++ /dev/null @@ -1,134 +0,0 @@ -version: '3.2' -services: - - genericassetcollector: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-assetCollectionService - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericassetcollector:latest - restart: "no" - networks: - - redis-network - - postgres-network - environment: - - EXEC_MODE=production - secrets: - - postgres_credentials - logging: - options: - max-size: "50m" - - - uniswapassetcollector: - depends_on: [genericassetcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericassetcollector:latest - command: /bin/assetCollectionService -source=Uniswap - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - networks: - - redis-network - - postgres-network - secrets: - - postgres_credentials - - sushiswapassetcollector: - depends_on: [genericassetcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericassetcollector:latest - command: /bin/assetCollectionService -source=SushiSwap - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - networks: - - redis-network - - postgres-network - secrets: - - postgres_credentials - - pancakeswapassetcollector: - depends_on: [genericassetcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericassetcollector:latest - command: /bin/assetCollectionService -source=PanCakeSwap - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - networks: - - redis-network - - postgres-network - secrets: - - postgres_credentials - - ethassetcollector: - depends_on: [genericassetcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericassetcollector:latest - command: /bin/assetCollectionService -source=assetlists -secret=eth_assets - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - networks: - - redis-network - - postgres-network - secrets: - - postgres_credentials - - nonethassetcollector: - depends_on: [genericassetcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericassetcollector:latest - command: /bin/assetCollectionService -source=assetlists -secret=non_eth_assets - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - networks: - - redis-network - - postgres-network - secrets: - - postgres_credentials - - fiatassetcollector: - depends_on: [genericassetcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericassetcollector:latest - command: /bin/assetCollectionService -source=assetlists -secret=fiat_assets - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - networks: - - redis-network - - postgres-network - secrets: - - postgres_credentials - -secrets: - api_binance: - file: ../secrets/api_binance.json - api_bitfinex: - file: ../secrets/api_bitfinex.json - api_kraken: - file: ../secrets/api_kraken.json - postgres_credentials: - file: ../secrets/postgres_credentials.txt - -networks: - kafka-network: - external: - name: kafka_kafka-network - redis-network: - external: - name: redis_redis-network - influxdb-network: - external: - name: influxdb_influxdb-network - postgres-network: - external: - name: postgres_postgres-network diff --git a/deployments/docker-compose.bitcoin-cash.yml b/deployments/docker-compose.bitcoin-cash.yml deleted file mode 100644 index d2774e0bb..000000000 --- a/deployments/docker-compose.bitcoin-cash.yml +++ /dev/null @@ -1,46 +0,0 @@ -version: '3.2' - -services: - bitcoin-cash: - image: - uphold/bitcoin-abc - volumes: - - /home/srv/bitcoin-cash:/var/lib/bitcoin-cash - command: -printtoconsole -prune=550 -rpcallowip=::/0 -disablewallet -rpcuser=mysecretrpcdiauser -rpcpassword=njTcaNX74sSf46_TXacMVlyPMJjuv9i03bqBgj9KQ8E= - environment: - - BITCOIN_ABC_DATA=/var/lib/bitcoin-cash - logging: - options: - max-size: "50m" - networks: - - scrapers-network - deploy: - mode: global - restart_policy: - delay: 2s - - bch: - build: - context: $GOPATH - dockerfile: $GOPATH/src/github.com/diadata-org/diadata/build/Dockerfile-bch - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_bch - depends_on: - - bitcoin-cash - networks: - - scrapers-network - logging: - options: - max-size: "50m" - secrets: - - api_diadata - - -secrets: - api_diadata: - file: $GOPATH/src/github.com/diadata-org/diadata/secrets/api_diadata.json - -volumes: - bitcoin: - -networks: - scrapers-network: diff --git a/deployments/docker-compose.bitcoin.yml b/deployments/docker-compose.bitcoin.yml deleted file mode 100644 index 5e4c25a8a..000000000 --- a/deployments/docker-compose.bitcoin.yml +++ /dev/null @@ -1,43 +0,0 @@ -version: '3.2' - -services: - bitcoin: - image: - kylemanna/bitcoind - user: "1000" - volumes: - - /home/srv/bitcoin:/bitcoin - command: btc_oneshot -printtoconsole -prune=550 -rpcallowip=::/0 -disablewallet -rpcpassword=mysecretrpcdiapassword -rpcuser=mysecretrpcdiauser - logging: - options: - max-size: "50m" - networks: - - scrapers-network - deploy: - mode: global - restart_policy: - delay: 2s - window: 20s - - btc: - build: - context: $GOPATH - dockerfile: $GOPATH/src/github.com/diadata-org/diadata/build/Dockerfile-btc - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_btc:latest - networks: - - scrapers-network - logging: - options: - max-size: "50m" - secrets: - - api_diadata - -secrets: - api_diadata: - file: $GOPATH/src/github.com/diadata-org/diadata/secrets/api_diadata.json - -volumes: - bitcoin: - -networks: - scrapers-network: diff --git a/deployments/docker-compose.block-scrapers.yml b/deployments/docker-compose.block-scrapers.yml deleted file mode 100644 index da79b6945..000000000 --- a/deployments/docker-compose.block-scrapers.yml +++ /dev/null @@ -1,42 +0,0 @@ -version: '3.2' -services: - - genericblockscraper: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-genericBlockScraper - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericblockscraper:latest - restart: "no" - networks: - - postgres-network - secrets: - - postgres_credentials - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - - ethereumscraper: - depends_on: [genericblockscraper] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericblockscraper:latest - command: /bin/blockscraper -blockchain=Ethereum - networks: - - postgres-network - secrets: - - postgres_credentials - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - - -secrets: - postgres_credentials: - file: ../secrets/postgres_credentials.txt - -networks: - postgres-network: - external: - name: postgres_postgres-network diff --git a/deployments/docker-compose.cardano.yml b/deployments/docker-compose.cardano.yml deleted file mode 100644 index 6ae24ec2a..000000000 --- a/deployments/docker-compose.cardano.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: '3.2' - -services: - cardano: - image: - diadata/cardano-explorer-docker - volumes: - - /home/srv/cardano:/home/cardano/cardano-sl/state-explorer-mainnet - logging: - options: - max-size: "50m" - networks: - - scrapers-network - deploy: - mode: global - resources: - limits: - memory: 6G - restart_policy: - delay: 2s - window: 20s - - ada: - build: - context: $GOPATH - dockerfile: $GOPATH/src/github.com/diadata-org/diadata/build/Dockerfile-ada - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_ada - networks: - - scrapers-network - logging: - options: - max-size: "50m" - secrets: - - api_diadata - -secrets: - api_diadata: - file: $GOPATH/src/github.com/diadata-org/diadata/secrets/api_diadata.json - -volumes: - cardano: - -networks: - scrapers-network: diff --git a/deployments/docker-compose.dash.yml b/deployments/docker-compose.dash.yml deleted file mode 100644 index d9bedc64f..000000000 --- a/deployments/docker-compose.dash.yml +++ /dev/null @@ -1,46 +0,0 @@ -version: '3.2' - -services: - dashd: - image: - dashpay/dashd - ports: - - "9998" - volumes: - - /home/srv/dash:/dash - command: dash_oneshot -printtoconsole -server -rpcuser=dashrpc -rpcpassword=dash_rpc_521d43b -rpcallowip=::/0 - logging: - options: - max-size: "50m" - networks: - - scrapers-network - deploy: - mode: global - restart_policy: - delay: 2s - window: 20s - - dash: - build: - context: $GOPATH - dockerfile: $GOPATH/src/github.com/diadata-org/diadata/build/Dockerfile-dash - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_dash - networks: - - scrapers-network - logging: - options: - max-size: "50m" - secrets: - - api_diadata - depends_on: - - dashd - -secrets: - api_diadata: - file: $GOPATH/src/github.com/diadata-org/diadata/secrets/api_diadata.json - -volumes: - bitcoin: - -networks: - scrapers-network: \ No newline at end of file diff --git a/deployments/docker-compose.dogecoin.yml b/deployments/docker-compose.dogecoin.yml deleted file mode 100644 index 5b14065b3..000000000 --- a/deployments/docker-compose.dogecoin.yml +++ /dev/null @@ -1,42 +0,0 @@ -version: '3.2' - -services: - dogecoin: - image: - joschua011/dogecoind-docker - volumes: - - /home/srv/dogecoin:/dogecoin - command: dogecoind -rpcuser=dogediauser -rpcpassword=dogediapassword -printtoconsole -server -rpcssl=0 -rpcport=22555 -rpcallowip=::/0 - logging: - options: - max-size: "50m" - networks: - - scrapers-network - deploy: - mode: global - restart_policy: - delay: 2s - window: 20s - - doge: - build: - context: $GOPATH - dockerfile: $GOPATH/src/github.com/diadata-org/diadata/build/Dockerfile-doge - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_doge - networks: - - scrapers-network - logging: - options: - max-size: "50m" - secrets: - - api_diadata - -secrets: - api_diadata: - file: $GOPATH/src/github.com/diadata-org/diadata/secrets/api_diadata.json - -volumes: - bitcoin: - -networks: - scrapers-network: \ No newline at end of file diff --git a/deployments/docker-compose.eos.yml b/deployments/docker-compose.eos.yml deleted file mode 100644 index 786e47ac1..000000000 --- a/deployments/docker-compose.eos.yml +++ /dev/null @@ -1,46 +0,0 @@ -version: '3.2' - -services: - eosnode: - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_eosnode - build: - context: $GOPATH - dockerfile: $GOPATH/src/github.com/diadata-org/diadata/build/Dockerfile-eosnode - volumes: - - eosio:/opt/eosio - logging: - options: - max-size: "50m" - cap_add: - - IPC_LOCK - stop_grace_period: 10m - networks: - - scrapers-network - - eos: - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_eos - depends_on: - - eosnode - build: - context: $GOPATH - dockerfile: $GOPATH/src/github.com/diadata-org/diadata/build/Dockerfile-eos - networks: - - scrapers-network - logging: - options: - max-size: "50m" - networks: - - scrapers-network - secrets: - - api_diadata - -secrets: - api_diadata: - file: $GOPATH/src/github.com/diadata-org/diadata/secrets/api_diadata.json - -volumes: - eosio: - external: true - -networks: - scrapers-network: diff --git a/deployments/docker-compose.ethereum-classic.yml b/deployments/docker-compose.ethereum-classic.yml deleted file mode 100644 index 6f7eb8bb6..000000000 --- a/deployments/docker-compose.ethereum-classic.yml +++ /dev/null @@ -1,45 +0,0 @@ -version: '3.2' - -services: - gethc: - image: - unscipreneur/ethereum-classic-geth - hostname: etcnode - volumes: - - /home/srv/gethc:/root/.ethereum-classic - command: "--name geth-classic --rpc --cache=1024 --rpcaddr=0.0.0.0 --rpcport=8545" - ports: - - 8545 - - 30303 - logging: - options: - max-size: "50m" - networks: - - scrapers-network - deploy: - mode: global - restart_policy: - delay: 2s - window: 20s - - etc: - build: - context: $GOPATH - dockerfile: $GOPATH/src/github.com/diadata-org/diadata/build/Dockerfile-etc - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_etc - networks: - - scrapers-network - depends_on: - - gethc - logging: - options: - max-size: "50m" - secrets: - - api_diadata - -secrets: - api_diadata: - file: $GOPATH/src/github.com/diadata-org/diadata/secrets/api_diadata.json - -networks: - scrapers-network: diff --git a/deployments/docker-compose.ethereum.yml b/deployments/docker-compose.ethereum.yml deleted file mode 100644 index f8feabaeb..000000000 --- a/deployments/docker-compose.ethereum.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: '3.2' - -services: - - geth: - image: - ethereum/client-go:v1.10.13 - volumes: - - /home/srv/geth:/root/.ethereum - command: --http.vhosts geth --http --http.addr "0.0.0.0" --cache 1024 --syncmode "fast" --ws --ws.addr "0.0.0.0" --ws.origins "*" --ws.port 8546 --http.api admin,db,eth,debug,miner,net,shh,txpool,personal,web3 --txpool.globalqueue 4096 --rpc.allow-unprotected-txs=true --txlookuplimit 0 - ports: - - mode: host - target: 8545 - published: 8545 - - mode: host - target: 8546 - published: 8546 - logging: - options: - max-size: "50m" - deploy: - mode: global - restart_policy: - delay: 2s - window: 20s diff --git a/deployments/docker-compose.exchange-scrapers.yml b/deployments/docker-compose.exchange-scrapers.yml deleted file mode 100644 index 1b87c017f..000000000 --- a/deployments/docker-compose.exchange-scrapers.yml +++ /dev/null @@ -1,510 +0,0 @@ -version: '3.2' -services: - - ecb: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-ecb - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_ecb:latest - networks: - - redis-network - - influxdb-network - - postgres-network - secrets: - - postgres_credentials - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - dfyncollector: - depends_on: [genericcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - command: /bin/collector -exchange=DFYN - networks: - - kafka-network - - redis-network - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - bitmaxcollector: - depends_on: [genericcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - command: /bin/collector -exchange=Bitmax - networks: - - kafka-network - - redis-network - - postgres-network - secrets: - - postgres_credentials - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - crexcollector: - depends_on: [genericcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - command: /bin/collector -exchange=CREX24 - networks: - - kafka-network - - redis-network - - postgres-network - secrets: - - postgres_credentials - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - hitbtccollector: - depends_on: [genericcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - command: /bin/collector -exchange=HitBTC -onePairPerSymbol - networks: - - kafka-network - - redis-network - - postgres-network - secrets: - - postgres_credentials - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - # gateiocollector: - # depends_on: [genericcollector] - # image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - # command: /bin/collector -exchange=GateIO - # networks: - # - kafka-network - # - redis-network - # logging: - # options: - # max-size: "50m" - # environment: - # - EXEC_MODE=production - - # bancorcollector: - # depends_on: [genericcollector] - # image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - # command: /bin/collector -exchange=Bancor - # networks: - # - kafka-network - # - redis-network - # logging: - # options: - # max-size: "50m" - # environment: - # - EXEC_MODE=production - - - loopringcollector: - depends_on: [genericcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - command: /bin/collector -exchange=Loopring - networks: - - kafka-network - - redis-network - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - uniswapcollector: - depends_on: [genericcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - command: /bin/collector -exchange=Uniswap - networks: - - kafka-network - - redis-network - - postgres-network - secrets: - - postgres_credentials - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - uniswapv3collector: - depends_on: [ genericcollector ] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - command: /bin/collector -exchange=UniswapV3 - networks: - - kafka-network - - redis-network - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - sushiswapcollector: - depends_on: [genericcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - command: /bin/collector -exchange=SushiSwap - networks: - - kafka-network - - redis-network - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - # pancakeswapcollector: - # depends_on: [genericcollector] - # image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - # command: /bin/collector -exchange=PanCakeSwap - # networks: - # - kafka-network - # - redis-network - # logging: - # options: - # max-size: "50m" - # environment: - # - EXEC_MODE=production - - - # gnosiscollector: - # depends_on: [genericcollector] - # image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - # command: /bin/collector -exchange=Gnosis - # networks: - # - kafka-network - # - redis-network - # logging: - # options: - # max-size: "50m" - # environment: - # - EXEC_MODE=production - - # curveficollector: - # depends_on: [genericcollector] - # image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - # command: /bin/collector -exchange=Curvefi - # networks: - # - kafka-network - # - redis-network - # logging: - # options: - # max-size: "50m" - # environment: - # - EXEC_MODE=production - - kucoincollector: - depends_on: [genericcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - command: /bin/collector -exchange=KuCoin - networks: - - kafka-network - - redis-network - - postgres-network - secrets: - - postgres_credentials - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - # makercollector: - # depends_on: [genericcollector] - # image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - # command: /bin/collector -exchange=Maker - # networks: - # - kafka-network - # - redis-network - # logging: - # options: - # max-size: "50m" - # environment: - # - EXEC_MODE=production - - genericcollector: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-genericCollector - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - restart: "no" - networks: - - kafka-network - - redis-network - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - # simexcollector: - # depends_on: [genericcollector] - # image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - # command: /bin/collector -exchange=Simex - # networks: - # - kafka-network - # - redis-network - # logging: - # options: - # max-size: "50m" - # environment: - # - EXEC_MODE=production - - stexcollector: - depends_on: [genericcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - command: /bin/collector -exchange=STEX - networks: - - kafka-network - - redis-network - - postgres-network - secrets: - - postgres_credentials - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - bittrexcollector: - depends_on: [genericcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - command: /bin/collector -exchange=Bittrex - networks: - - kafka-network - - redis-network - - postgres-network - secrets: - - postgres_credentials - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - # balancercollector: - # depends_on: [genericcollector] - # image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - # command: /bin/collector -exchange=Balancer - # networks: - # - kafka-network - # - redis-network - # logging: - # options: - # max-size: "50m" - # environment: - # - EXEC_MODE=production - - # dforcecollector: - # depends_on: [genericcollector] - # image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - # command: /bin/collector -exchange=Dforce - # networks: - # - kafka-network - # - redis-network - # logging: - # options: - # max-size: "50m" - # environment: - # - EXEC_MODE=production - - # lbankcollector: - # depends_on: [genericcollector] - # image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - # command: /bin/collector -exchange=LBank - # networks: - # - kafka-network - # - redis-network - # logging: - # options: - # max-size: "50m" - # environment: - # - EXEC_MODE=production - - okexcollector: - depends_on: [genericcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - command: /bin/collector -exchange=OKEx -onePairPerSymbol - networks: - - kafka-network - - redis-network - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - # huobicollector: - # depends_on: [genericcollector] - # image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - # command: /bin/collector -exchange=Huobi - # networks: - # - kafka-network - # - redis-network - # logging: - # options: - # max-size: "50m" - # environment: - # - EXEC_MODE=production - - coinbasecollector: - depends_on: [genericcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - command: /bin/collector -exchange=CoinBase - networks: - - kafka-network - - redis-network - - postgres-network - secrets: - - postgres_credentials - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - # binancecollector: - # depends_on: [genericcollector] - # image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - # command: /bin/collector -exchange=Binance - # networks: - # - kafka-network - # - redis-network - # - postgres-network - # secrets: - # - postgres_credentials - # - api_binance - # logging: - # options: - # max-size: "50m" - # environment: - # - EXEC_MODE=production - - # bitfinexcollector: - # depends_on: [genericcollector] - # image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - # command: /bin/collector -exchange=Bitfinex - # networks: - # - kafka-network - # - redis-network - # logging: - # options: - # max-size: "50m" - # secrets: - # - api_bitfinex - # environment: - # - EXEC_MODE=production - - krakencollector: - depends_on: [genericcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - command: /bin/collector -exchange=Kraken - networks: - - kafka-network - - redis-network - - postgres-network - secrets: - - postgres_credentials - - api_kraken - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - zbcollector: - depends_on: [genericcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - command: /bin/collector -exchange=ZB - networks: - - kafka-network - - redis-network - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - quoinecollector: - depends_on: [genericcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - command: /bin/collector -exchange=Quoine - networks: - - kafka-network - - redis-network - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - bitbaycollector: - depends_on: [genericcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - command: /bin/collector -exchange=BitBay - networks: - - kafka-network - - redis-network - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - # zeroxcollector: - # depends_on: [genericcollector] - # image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - # command: /bin/collector -exchange=0x - # networks: - # - kafka-network - # - redis-network - # logging: - # options: - # max-size: "50m" - # environment: - # - EXEC_MODE=production - - # kybercollector: - # depends_on: [genericcollector] - # image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericcollector:latest - # command: /bin/collector -exchange=Kyber - # networks: - # - kafka-network - # - redis-network - # logging: - # options: - # max-size: "50m" - # environment: - # - EXEC_MODE=production - -secrets: - api_binance: - file: ../secrets/api_binance.json - api_bitfinex: - file: ../secrets/api_bitfinex.json - api_kraken: - file: ../secrets/api_kraken.json - postgres_credentials: - file: ../secrets/postgres_credentials.txt - -networks: - kafka-network: - external: - name: kafka_kafka-network - redis-network: - external: - name: redis_redis-network - influxdb-network: - external: - name: influxdb_influxdb-network - postgres-network: - external: - name: postgres_postgres-network diff --git a/deployments/docker-compose.foreignscraper.yml b/deployments/docker-compose.foreignscraper.yml deleted file mode 100644 index ac8147619..000000000 --- a/deployments/docker-compose.foreignscraper.yml +++ /dev/null @@ -1,64 +0,0 @@ -version: '3.2' -services: - - coingeckoscraper: - depends_on: [genericforeignscraper] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericforeignscraper:latest - command: /bin/foreignscraper -foreignType Coingecko - networks: - - kafka-network - - influxdb-network - - redis-network - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - - coinmarketcapscraper: - depends_on: [genericforeignscraper] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericforeignscraper:latest - command: /bin/foreignscraper -foreignType CoinMarketCap - networks: - - kafka-network - - influxdb-network - - redis-network - environment: - - EXEC_MODE=production - - CMC_API_KEY=${CMC_API_KEY} - secrets: - - Coinmarketcap-API.key - logging: - options: - max-size: "50m" - - genericforeignscraper: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-genericForeignScraper - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericforeignscraper:latest - restart: "no" - networks: - - kafka-network - - influxdb-network - - redis-network - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - -secrets: - Coinmarketcap-API.key: - file: $GOPATH/src/github.com/diadata-org/diadata/secrets/Coinmarketcap-API.key - -networks: - kafka-network: - external: - name: kafka_kafka-network - redis-network: - external: - name: redis_redis-network - influxdb-network: - external: - name: influxdb_influxdb-network diff --git a/deployments/docker-compose.graphql.yml b/deployments/docker-compose.graphql.yml deleted file mode 100644 index 214d7eebf..000000000 --- a/deployments/docker-compose.graphql.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: "3.2" -services: - graphqlserver: - environment: - - EXEC_MODE=production - ports: - - "1111:1111" - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-graphqlServer - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_graphqlserver:latest - networks: - - redis-network - - influxdb-network - - logging: - options: - max-size: "50m" - -networks: - redis-network: - external: - name: redis_redis-network - influxdb-network: - external: - name: influxdb_influxdb-network diff --git a/deployments/docker-compose.http.yml b/deployments/docker-compose.http.yml deleted file mode 100644 index 2f2082ba9..000000000 --- a/deployments/docker-compose.http.yml +++ /dev/null @@ -1,77 +0,0 @@ -version: '3.2' -services: - - restserver: - environment: - - EXEC_MODE=production - ports: - - "8080:8080" - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-restServer - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_restserver:latest - networks: - - kafka-network - - redis-network - - influxdb-network - - postgres-network - secrets: - - api_diadata - - postgres_credentials - logging: - options: - max-size: "50m" - volumes: - - /home/srv/restServer/charts:/charts - - nginx: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-nginx - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_nginx:latest - networks: - - kafka-network - ports: - - "443:443" - - "80:80" - secrets: - - api_diadata_ssl_certificate - - api_diadata_ssl_certificate_key - - d2_diadata_ssl_certificate - - d2_diadata_ssl_certificate_key - - htpassword - logging: - options: - max-size: "50m" - depends_on: - - restserver - -secrets: - api_diadata_ssl_certificate: - file: ../secrets/live/api.diadata.org/fullchain.pem - api_diadata_ssl_certificate_key: - file: ../secrets/live/api.diadata.org/privkey.pem - d2_diadata_ssl_certificate: - file: ../secrets/live/d2.diadata.org/fullchain.pem - d2_diadata_ssl_certificate_key: - file: ../secrets/live/d2.diadata.org/privkey.pem - api_diadata: - file: ../secrets/api_diadata.json - htpassword: - file: ../secrets/htpassword.nginx - postgres_credentials: - file: ../secrets/postgres_credentials.txt - -networks: - kafka-network: - external: - name: kafka_kafka-network - redis-network: - external: - name: redis_redis-network - influxdb-network: - external: - name: influxdb_influxdb-network - postgres-network: - external: - name: postgres_postgres-network \ No newline at end of file diff --git a/deployments/docker-compose.influxdb-2.yml b/deployments/docker-compose.influxdb-2.yml deleted file mode 100644 index 29fcc82b0..000000000 --- a/deployments/docker-compose.influxdb-2.yml +++ /dev/null @@ -1,33 +0,0 @@ -version: '3.2' -services: - - influxdb2: - environment: - - EXEC_MODE=production - - image: influxdb:1.8.3 - volumes: - - /home/srv/influxdb-2:/var/lib/influxdb - networks: - - influxdb2-network - environment: - INFLUXDB_HTTP_MAX_BODY_SIZE: 0 - INFLUXDB_MONITOR_STORE_ENABLED: "FALSE" - # Default is 1g - INFLUXDB_DATA_CACHE_MAX_MEMORY_SIZE: 4g - # Default is 1m - INFLUXDB_DATA_MAX_INDEX_LOG_FILE_SIZE: 8m - logging: - options: - max-size: "50m" - - deploy: - resources: - limits: - memory: 100G - - -networks: - influxdb2-network: - driver: overlay - attachable: true diff --git a/deployments/docker-compose.influxdb.yml b/deployments/docker-compose.influxdb.yml deleted file mode 100644 index ac59d5e06..000000000 --- a/deployments/docker-compose.influxdb.yml +++ /dev/null @@ -1,27 +0,0 @@ -#alias influxcli="docker exec -it `docker ps | grep influx | cut -f1 -d\ ` influx" - -version: '3.2' -services: - - influxdb: -# ports: -# - "8086:8086" - environment: - - EXEC_MODE=production - - image: influxdb:1.8.3 - volumes: - - /home/srv/influx-backup:/var/lib/influxdb - networks: - - influxdb-network - environment: - INFLUXDB_HTTP_MAX_BODY_SIZE: 0 - INFLUXDB_MONITOR_STORE_ENABLED: "FALSE" - logging: - options: - max-size: "50m" - -networks: - influxdb-network: - driver: overlay - attachable: true diff --git a/deployments/docker-compose.influxmigration.yml b/deployments/docker-compose.influxmigration.yml deleted file mode 100644 index a60f3226a..000000000 --- a/deployments/docker-compose.influxmigration.yml +++ /dev/null @@ -1,43 +0,0 @@ -version: '3.2' -services: - - influxmigration: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-influxMigration - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_influxmigration:latest - command: /bin/influxMigration - networks: - - influxdb-network - - influxdb2-network - - postgres-network - logging: - options: - max-size: "50m" - secrets: - - postgres_credentials - volumes: - - /home/srv/config:/config - environment: - - EXEC_MODE=production - - INFLUX_READ_URL=http://influxdb:8086 - - INFLUX_WRITE_URL=http://influxdb-2:8086 - - INFLUX_TABLE_ORIGIN=trades - - INFLUX_TABLE_DESTINATION=tradesTmp - - TIME_FINAL=1636618800 - - INFLUX_MIGRATION_TESTMODE=false - -secrets: - postgres_credentials: - file: ../secrets/postgres_credentials.txt - -networks: - postgres-network: - external: - name: postgres_postgres-network - influxdb-network: - external: - name: influxdb_influxdb-network - influxdb2-network: - external: - name: influxdb-2_influxdb2-network \ No newline at end of file diff --git a/deployments/docker-compose.kafka.yml b/deployments/docker-compose.kafka.yml deleted file mode 100644 index 1afe005d3..000000000 --- a/deployments/docker-compose.kafka.yml +++ /dev/null @@ -1,57 +0,0 @@ -# add labels to machines: -# docker node update xxx --label-add kafka=1 -# docker node update xxx --label-add zoo=1 - -version: '3.2' -services: - - zookeeper: - image: wurstmeister/zookeeper -# ports: -# - "2181:2181" - volumes: -# TODO figure out how to have this without version number - - /home/srv/zoo:/opt/zookeeper-3.4.9/data - networks: - - kafka-network - deploy: - mode: global - restart_policy: - delay: 2s - window: 20s - kafka0: - image: wurstmeister/kafka:latest -# ports: -# - "9094:9094" - networks: - - kafka-network - deploy: - mode: global - environment: - HOSTNAME_COMMAND: "docker info | grep ^Name: | cut -d' ' -f 2" - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT - # KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://_{HOSTNAME_COMMAND}:9094 - KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://kafka0:9094 - KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094 - KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE - KAFKA_LOG_DIRS: /kafka/kafka-logs-1 - # KAFKA_LOG_RETENTION_BYTES: -1 - # KAFKA_LOG_RETENTION_HOURS: -1 - KAFKA_DELETE_TOPIC_ENABLE: "true" - KAFKA_MESSAGE_MAX_BYTES: 50000000 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - /home/srv/kafka:/kafka - deploy: - mode: global - restart_policy: - delay: 2s - window: 20s - depends_on: - - zookeeper - -networks: - kafka-network: - driver: overlay - attachable: true diff --git a/deployments/docker-compose.litecoin.yml b/deployments/docker-compose.litecoin.yml deleted file mode 100644 index 176bbc8d0..000000000 --- a/deployments/docker-compose.litecoin.yml +++ /dev/null @@ -1,45 +0,0 @@ -version: '3.2' - -services: - litecoin: - image: - uphold/litecoin-core - volumes: - - /home/srv/litecoin:/var/lib/litecoin - command: -printtoconsole -prune=550 -rpcallowip=::/0 -disablewallet -rpcuser=mysecretrpcdiauser -rpcpassword=njTcaNX74sSf46_TXacMVlyPMJjuv9i03bqBgj9KQ8E= - environment: - - LITECOIN_DATA=/var/lib/litecoin - logging: - options: - max-size: "50m" - networks: - - scrapers-network - deploy: - mode: global - restart_policy: - delay: 2s - window: 20s - - ltc: - build: - context: $GOPATH - dockerfile: $GOPATH/src/github.com/diadata-org/diadata/build/Dockerfile-ltc - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_ltc - networks: - - scrapers-network - logging: - options: - max-size: "50m" - secrets: - - api_diadata - - -secrets: - api_diadata: - file: $GOPATH/src/github.com/diadata-org/diadata/secrets/api_diadata.json - -volumes: - bitcoin: - -networks: - scrapers-network: \ No newline at end of file diff --git a/deployments/docker-compose.monero.yml b/deployments/docker-compose.monero.yml deleted file mode 100644 index 95343217a..000000000 --- a/deployments/docker-compose.monero.yml +++ /dev/null @@ -1,36 +0,0 @@ -version: '3.2' - -services: - monero: - image: - ${DOCKER_HUB_LOGIN}/${STACKNAME}_monero - build: - context: $GOPATH - dockerfile: $GOPATH/src/github.com/diadata-org/diadata/build/Dockerfile-monero - volumes: - - /home/srv/monero:/root/.bitmonero - networks: - - scrapers-network - - xmr: - build: - context: $GOPATH - dockerfile: $GOPATH/src/github.com/diadata-org/diadata/build/Dockerfile-xmr - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_xmr - networks: - - scrapers-network - logging: - options: - max-size: "50m" - secrets: - - api_diadata - -secrets: - api_diadata: - file: $GOPATH/src/github.com/diadata-org/diadata/secrets/api_diadata.json - -volumes: - bitcoin: - -networks: - scrapers-network: \ No newline at end of file diff --git a/deployments/docker-compose.neo.yml b/deployments/docker-compose.neo.yml deleted file mode 100644 index 9c789e26f..000000000 --- a/deployments/docker-compose.neo.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: '3.2' - -services: - neonode: - build: - context: $GOPATH - dockerfile: $GOPATH/src/github.com/diadata-org/diadata/build/Dockerfile-neonode - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_neonode - tty: true - volumes: - - /home/srv/neo:/app/Chain_00746E41 - logging: - options: - max-size: "50m" - networks: - - scrapers-network - deploy: - mode: global - restart_policy: - delay: 2s - window: 20s - - neo: - build: - context: $GOPATH - dockerfile: $GOPATH/src/github.com/diadata-org/diadata/build/Dockerfile-neo - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_neo - networks: - - scrapers-network - logging: - options: - max-size: "50m" - secrets: - - api_diadata - -secrets: - api_diadata: - file: $GOPATH/src/github.com/diadata-org/diadata/secrets/api_diadata.json - -volumes: - bitcoin: - -networks: - scrapers-network: diff --git a/deployments/docker-compose.nftbidsscraper.yml b/deployments/docker-compose.nftbidsscraper.yml deleted file mode 100644 index d89b262f9..000000000 --- a/deployments/docker-compose.nftbidsscraper.yml +++ /dev/null @@ -1,41 +0,0 @@ -version: '3.2' -services: - - genericnftbidsscraper: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-genericNFTBidsScraper - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericnftbidsscraper:latest - restart: "no" - networks: - - postgres-network - secrets: - - postgres_credentials - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - - cryptopunkscraper: - depends_on: [genericnftbidsscraper] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericnftbidsscraper:latest - command: /bin/nftBid-scrapers -nftclass=CryptoPunks - networks: - - postgres-network - secrets: - - postgres_credentials - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - -secrets: - postgres_credentials: - file: ../secrets/postgres_credentials.txt - -networks: - postgres-network: - external: - name: postgres_postgres-network \ No newline at end of file diff --git a/deployments/docker-compose.nftcollection-service.yml b/deployments/docker-compose.nftcollection-service.yml deleted file mode 100644 index 84b41cd78..000000000 --- a/deployments/docker-compose.nftcollection-service.yml +++ /dev/null @@ -1,49 +0,0 @@ -version: '3.2' -services: - - genericnftcollector: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-nftCollectionService - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericnftcollector:latest - restart: "no" - networks: - - redis-network - - postgres-network - environment: - - EXEC_MODE=production - secrets: - - postgres_credentials - logging: - options: - max-size: "50m" - - openseanftcollector: - depends_on: [genericnftcollector] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericnftcollector:latest - command: /bin/nftCollectionService -source=Opensea - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - networks: - - redis-network - - postgres-network - secrets: - - postgres_credentials - - Opensea-API.key - -secrets: - postgres_credentials: - file: ../secrets/postgres_credentials.txt - Opensea-API.key: - file: $GOPATH/src/github.com/diadata-org/diadata/secrets/Opensea-API.key - -networks: - redis-network: - external: - name: redis_redis-network - postgres-network: - external: - name: postgres_postgres-network \ No newline at end of file diff --git a/deployments/docker-compose.nftdatascraper.yml b/deployments/docker-compose.nftdatascraper.yml deleted file mode 100644 index c7d04e91e..000000000 --- a/deployments/docker-compose.nftdatascraper.yml +++ /dev/null @@ -1,88 +0,0 @@ -version: '3.2' -services: - - genericnftdatascraper: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-genericNFTDataScraper - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericnftdatascraper:latest - restart: "no" - networks: - - postgres-network - secrets: - - postgres_credentials - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - - sorarescraper: - depends_on: [genericnftdatascraper] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericnftdatascraper:latest - command: /bin/nftDatascraper -nftclass=Sorare - networks: - - postgres-network - secrets: - - postgres_credentials - - Opensea-API.key - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - - # cryptopunkscraper: - # depends_on: [genericnftdatascraper] - # image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericnftdatascraper:latest - # command: /bin/nftDatascraper -nftclass=CryptoPunks - # networks: - # - postgres-network - # secrets: - # - postgres_credentials - # - Opensea-API.key - # environment: - # - EXEC_MODE=production - # logging: - # options: - # max-size: "50m" - - cryptokittiescraper: - depends_on: [genericnftdatascraper] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericnftdatascraper:latest - command: /bin/nftDatascraper -nftclass=CryptoKitties - networks: - - postgres-network - secrets: - - postgres_credentials - - Opensea-API.key - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - - topshotscraper: - depends_on: [genericnftdatascraper] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericnftdatascraper:latest - command: /bin/nftDatascraper -nftclass=Topshot - networks: - - postgres-network - secrets: - - postgres_credentials - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - -secrets: - postgres_credentials: - file: ../secrets/postgres_credentials.txt - Opensea-API.key: - file: $GOPATH/src/github.com/diadata-org/diadata/secrets/Opensea-API.key - -networks: - postgres-network: - external: - name: postgres_postgres-network \ No newline at end of file diff --git a/deployments/docker-compose.nftofferscraper.yml b/deployments/docker-compose.nftofferscraper.yml deleted file mode 100644 index 223247db3..000000000 --- a/deployments/docker-compose.nftofferscraper.yml +++ /dev/null @@ -1,55 +0,0 @@ -version: '3.2' -services: - - genericnftofferscraper: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-genericNFTOfferScraper - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericnftofferscraper:latest - restart: "no" - networks: - - postgres-network - secrets: - - postgres_credentials - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - - cryptokittiesscraper: - depends_on: [genericnftofferscraper] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericnftofferscraper:latest - command: /bin/nftOffer-scrapers -nftclass=CryptoKitties - networks: - - postgres-network - secrets: - - postgres_credentials - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - - cryptopunksscraper: - depends_on: [genericnftofferscraper] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericnftofferscraper:latest - command: /bin/nftOffer-scrapers -nftclass=CryptoPunks - networks: - - postgres-network - secrets: - - postgres_credentials - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - -secrets: - postgres_credentials: - file: ../secrets/postgres_credentials.txt - -networks: - postgres-network: - external: - name: postgres_postgres-network \ No newline at end of file diff --git a/deployments/docker-compose.nfttradesscraper.yml b/deployments/docker-compose.nfttradesscraper.yml deleted file mode 100644 index 6d53676e1..000000000 --- a/deployments/docker-compose.nfttradesscraper.yml +++ /dev/null @@ -1,83 +0,0 @@ -version: '3.2' -services: - - genericnfttradesscraper: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-genericNFTTradesScraper - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericnfttradesscraper:latest - restart: "no" - networks: - - postgres-network - secrets: - - postgres_credentials - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - - openseascraper: - depends_on: [genericnfttradesscraper] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericnfttradesscraper:latest - command: /bin/nftTrade-scrapers -nftclass=Opensea - networks: - - postgres-network - secrets: - - postgres_credentials - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - - cryptopunksscraper: - depends_on: [genericnfttradesscraper] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericnfttradesscraper:latest - command: /bin/nftTrade-scrapers -nftclass=CryptoPunks - networks: - - postgres-network - secrets: - - postgres_credentials - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - - cryptokittiesscraper: - depends_on: [genericnfttradesscraper] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericnfttradesscraper:latest - command: /bin/nftTrade-scrapers -nftclass=CryptoKitties - networks: - - postgres-network - secrets: - - postgres_credentials - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - - topshotscraper: - depends_on: [genericnfttradesscraper] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericnfttradesscraper:latest - command: /bin/nftTrade-scrapers -nftclass=Topshot - networks: - - postgres-network - secrets: - - postgres_credentials - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - -secrets: - postgres_credentials: - file: ../secrets/postgres_credentials.txt - -networks: - postgres-network: - external: - name: postgres_postgres-network \ No newline at end of file diff --git a/deployments/docker-compose.postgres.yml b/deployments/docker-compose.postgres.yml deleted file mode 100644 index ad09bbb17..000000000 --- a/deployments/docker-compose.postgres.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: '3.2' -services: - - postgres: - image: postgres - restart: always - ports: - - "27017:27017" - - "5432:5432" - environment: - EXEC_MODE: production - env_file: - - ../secrets/postgres.env - networks: - - postgres-network - volumes: - - /home/srv/config/pginit.sql:/docker-entrypoint-initdb.d/init.sql - - /home/srv/postgres:/var/lib/postgresql/data - logging: - options: - max-size: "50m" - -networks: - postgres-network: - driver: overlay - attachable: true diff --git a/deployments/docker-compose.ratescrapers.yml b/deployments/docker-compose.ratescrapers.yml deleted file mode 100644 index 4a7f9c645..000000000 --- a/deployments/docker-compose.ratescrapers.yml +++ /dev/null @@ -1,90 +0,0 @@ -version: '3.2' -services: - - sofr-scraper: - depends_on: [ratescraper] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_ratescraper:latest - command: /bin/ratescrapers -type SOFR - networks: - - influxdb-network - - redis-network - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - safr-scraper: - depends_on: [ratescraper] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_ratescraper:latest - command: /bin/ratescrapers -type SAFR - networks: - - influxdb-network - - redis-network - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - safrAvgs-scraper: - depends_on: [ratescraper] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_ratescraper:latest - command: /bin/ratescrapers -type SAFR-AVGS - networks: - - influxdb-network - - redis-network - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - ester-scraper: - depends_on: [ratescraper] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_ratescraper:latest - command: /bin/ratescrapers -type ESTER - networks: - - influxdb-network - - redis-network - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - sonia-scraper: - depends_on: [ratescraper] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_ratescraper:latest - command: /bin/ratescrapers -type SONIA - networks: - - influxdb-network - - redis-network - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - - ratescraper: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-ratescraper - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_ratescraper:latest - restart: "no" - networks: - - influxdb-network - - redis-network - logging: - options: - max-size: "50m" - environment: - - EXEC_MODE=production - -networks: - influxdb-network: - external: - name: influxdb_influxdb-network - redis-network: - external: - name: redis_redis-network diff --git a/deployments/docker-compose.redis.yml b/deployments/docker-compose.redis.yml deleted file mode 100644 index 788f2c632..000000000 --- a/deployments/docker-compose.redis.yml +++ /dev/null @@ -1,22 +0,0 @@ -# docker exec -it `docker ps | grep redis-server | cut -f1 -d\ ` redis-cli - -version: '3.2' -services: - - redis: - environment: - - EXEC_MODE=production - image: redis - volumes: - - /home/srv/redis:/data - networks: - - redis-network - entrypoint: redis-server --appendonly yes - logging: - options: - max-size: "50m" - -networks: - redis-network: - driver: overlay - attachable: true \ No newline at end of file diff --git a/deployments/docker-compose.ripple.yml b/deployments/docker-compose.ripple.yml deleted file mode 100644 index d96fb795d..000000000 --- a/deployments/docker-compose.ripple.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: '3.2' - -services: - xrp: - build: - context: $GOPATH - dockerfile: $GOPATH/src/github.com/diadata-org/diadata/build/Dockerfile-xrp - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_xrp - networks: - - scrapers-network - logging: - options: - max-size: "50m" - secrets: - - api_diadata - -secrets: - api_diadata: - file: $GOPATH/src/github.com/diadata-org/diadata/secrets/api_diadata.json - -volumes: - bitcoin: - -networks: - scrapers-network: \ No newline at end of file diff --git a/deployments/docker-compose.services.yml b/deployments/docker-compose.services.yml deleted file mode 100644 index 2cdc4f312..000000000 --- a/deployments/docker-compose.services.yml +++ /dev/null @@ -1,174 +0,0 @@ -version: '3.2' -services: - - tradesblockservice: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-tradesBlockService - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_tradesblockservice:latest - networks: - - kafka-network - - redis-network - - influxdb-network - - postgres-network - logging: - options: - max-size: "50m" - volumes: - - /home/srv/config:/config - environment: - - EXEC_MODE=production - - filtersblockservice: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-filtersBlockService - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_filtersblockservice:latest - networks: - - kafka-network - - redis-network - - influxdb-network - - postgres-network - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - - graphservice: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-graphService - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_graphservice:latest - networks: - - redis-network - - influxdb-network - volumes: - - /home/srv/restServer/charts:/charts - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - - # itinservice: - # build: - # context: ../../../.. - # dockerfile: github.com/diadata-org/diadata/build/Dockerfile-itinService - # image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_itinservice:latest - # networks: - # - redis-network - # - influxdb-network - # environment: - # - EXEC_MODE=production - # logging: - # options: - # max-size: "50m" - - indexcalculationservice: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-indexCalculationService - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_indexcalculationservice:latest - networks: - - redis-network - - influxdb-network - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - - cviservice: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-cviService - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_cviservice:latest - networks: - - redis-network - - influxdb-network - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - - ethcviservice: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-cviServiceEth - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_ethcviservice:latest - networks: - - redis-network - - influxdb-network - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - - blockchainservice: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-blockchainservice - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_blockchainservice:latest - networks: - - postgres-network - environment: - - EXEC_MODE=production - secrets: - - postgres_credentials - logging: - options: - max-size: "50m" - - pairdiscoveryservice: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-pairDiscoveryService - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_pairdiscoveryservice:latest - networks: - - redis-network - - postgres-network - environment: - - EXEC_MODE=production - secrets: - - postgres_credentials - logging: - options: - max-size: "50m" - - supplyservice: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-supplyService - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_supplyservice:latest - networks: - - redis-network - - influxdb-network - - postgres-network - secrets: - - postgres_credentials - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - -networks: - kafka-network: - external: - name: kafka_kafka-network - redis-network: - external: - name: redis_redis-network - influxdb-network: - external: - name: influxdb_influxdb-network - postgres-network: - external: - name: postgres_postgres-network - -secrets: - postgres_credentials: - file: ../secrets/postgres_credentials.txt \ No newline at end of file diff --git a/deployments/docker-compose.stellar.yml b/deployments/docker-compose.stellar.yml deleted file mode 100644 index 440c35fb4..000000000 --- a/deployments/docker-compose.stellar.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: '3.2' - -services: - xlm: - build: - context: $GOPATH - dockerfile: $GOPATH/src/github.com/diadata-org/diadata/build/Dockerfile-xlm - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_xlm - networks: - - scrapers-network - logging: - options: - max-size: "50m" - secrets: - - api_diadata - -secrets: - api_diadata: - file: $GOPATH/src/github.com/diadata-org/diadata/secrets/api_diadata.json - -volumes: - bitcoin: - -networks: - scrapers-network: \ No newline at end of file diff --git a/deployments/docker-compose.stockscrapers.yml b/deployments/docker-compose.stockscrapers.yml deleted file mode 100644 index 59ed42223..000000000 --- a/deployments/docker-compose.stockscrapers.yml +++ /dev/null @@ -1,45 +0,0 @@ -version: '3.2' -services: - - finagescraper: - depends_on: [genericstockscraper] - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericstockscraper:latest - command: /bin/stock-scrapers -source=Finage - networks: - - influxdb-network - - redis-network - environment: - - EXEC_MODE=production - secrets: - - api_finage - logging: - options: - max-size: "50m" - - - genericstockscraper: - build: - context: ../../../.. - dockerfile: github.com/diadata-org/diadata/build/Dockerfile-genericStockScraper - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_genericstockscraper:latest - restart: "no" - networks: - - influxdb-network - - redis-network - environment: - - EXEC_MODE=production - logging: - options: - max-size: "50m" - -networks: - influxdb-network: - external: - name: influxdb_influxdb-network - redis-network: - external: - name: redis_redis-network - -secrets: - api_finage: - file: $GOPATH/src/github.com/diadata-org/diadata/secrets/api_finage \ No newline at end of file diff --git a/deployments/docker-compose.tether.yml b/deployments/docker-compose.tether.yml deleted file mode 100644 index a475d81fb..000000000 --- a/deployments/docker-compose.tether.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: '3.2' - -services: - usdt: - build: - context: $GOPATH - dockerfile: $GOPATH/src/github.com/diadata-org/diadata/build/Dockerfile-usdt - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_usdt - networks: - - scrapers-network - logging: - options: - max-size: "50m" - secrets: - - api_diadata - -secrets: - api_diadata: - file: $GOPATH/src/github.com/diadata-org/diadata/secrets/api_diadata.json - -volumes: - bitcoin: - -networks: - scrapers-network: \ No newline at end of file diff --git a/deployments/docker-compose.tezos.yml b/deployments/docker-compose.tezos.yml deleted file mode 100644 index 1e09bd475..000000000 --- a/deployments/docker-compose.tezos.yml +++ /dev/null @@ -1,40 +0,0 @@ -version: "3.2" - -volumes: - node_data: - client_data: - -services: - tezosnode: - image: tezos/tezos:mainnet - hostname: tezosnode - command: tezos-node --net-addr :9732 - ports: - - "9732" - - "8732" - volumes: - - node_data:/home/srv/tezos/node - - client_data:/home/srv/tezos/client - restart: on-failure - networks: - - scrapers-network - - xtz: - build: - context: ../../../.. - dockerfile: $GOPATH/src/github.com/diadata-org/diadata/build/Dockerfile-xtz - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_xtz - networks: - - scrapers-network - logging: - options: - max-size: "50m" - secrets: - - api_diadata - -networks: - scrapers-network: - -secrets: - api_diadata: - file: $GOPATH/src/github.com/diadata-org/diadata/secrets/api_diadata.json \ No newline at end of file diff --git a/deployments/docker-compose.tron.yml b/deployments/docker-compose.tron.yml deleted file mode 100644 index 098350b33..000000000 --- a/deployments/docker-compose.tron.yml +++ /dev/null @@ -1,46 +0,0 @@ -version: '3.2' - -services: - tron: - build: - context: $GOPATH - dockerfile: $GOPATH/src/github.com/diadata-org/diadata/build/Dockerfile-tron - image: - ${DOCKER_HUB_LOGIN}/${STACKNAME}_tron - working_dir: /java-tron/build/libs - command: bash -c "java -jar FullNode.jar --witness" - logging: - options: - max-size: "50m" - networks: - - scrapers-network - deploy: - mode: global - restart_policy: - delay: 2s - window: 20s - - trx: - build: - context: $GOPATH - dockerfile: $GOPATH/src/github.com/diadata-org/diadata/build/Dockerfile-trx - image: ${DOCKER_HUB_LOGIN}/${STACKNAME}_trx - depends_on: - - tron - networks: - - scrapers-network - logging: - options: - max-size: "50m" - secrets: - - api_diadata - -secrets: - api_diadata: - file: $GOPATH/src/github.com/diadata-org/diadata/secrets/api_diadata.json - -volumes: - bitcoin: - -networks: - scrapers-network: diff --git a/deployments/k8s-yaml/cronjob-snapshot.yaml b/deployments/k8s-yaml/cronjob-snapshot.yaml deleted file mode 100755 index 23899a0c6..000000000 --- a/deployments/k8s-yaml/cronjob-snapshot.yaml +++ /dev/null @@ -1,118 +0,0 @@ -apiVersion: batch/v1 -kind: CronJob -metadata: - name: cronjob-postgresdump - labels: - app: cronjob-postgresdump -spec: - schedule: "0 0 * * *" - jobTemplate: - spec: - template: - spec: - restartPolicy: Never - containers: - - name: cronjob-postgresdump-kaniko - image: gcr.io/kaniko-project/executor:v1.6.0 - imagePullPolicy: IfNotPresent - args: - - "--dockerfile=build/Dockerfile-postgrescronjob-Dev" - - "--context=dir:///kaniko-context" - - "--destination=registry.hub.docker.com/alex1a/diadata.postgres:latest" - - "--verbosity=debug" - - "--registry-mirror=public.ecr.aws" - volumeMounts: - - name: docker-config - mountPath: /kaniko/.docker - - name: env-context - mountPath: /kaniko-context - readOnly: true - - name: env-context - mountPath: /env-workdir/00-pginitscript.sql - subPath: ./deployments/config/pginit.sql - - name: env-workdir - mountPath: /env-workdir/01-pginitscript.sql - subPath: ./pg_dump.sql - initContainers: - - name: check-postgres-connection - image: public.ecr.aws/docker/library/postgres:15-alpine - env: - - name: PGHOST - value: "data-postgres.default.svc.cluster.local" - - name: PGUSER - value: "postgres" - - name: PGDB - value: "postgres" - - name: PGPASSWORD - value: "password" - command: - [ - "sh", - "-c", - "until pg_isready -h $PGHOST -U $PGUSER; do sleep 1; done", - ] - - name: cronjob-postgresdump-extract - image: public.ecr.aws/docker/library/postgres:15-alpine - command: - [ - "/mnt/env-context/deployments/config/postgres-docker-entrypoint.sh", - ] - args: ["postgres"] - imagePullPolicy: IfNotPresent - env: - - name: POSTGRES_USER - value: postgres - - name: POSTGRES_PASSWORD - value: password - - name: POSTGRES_DB - value: postgres - - name: PGHOST_EXTRACT - value: "data-postgres.default.svc.cluster.local" - - name: PGUSER_EXTRACT - value: "postgres" - - name: PGDB_EXTRACT - value: "postgres" - - name: PGPASSWORD_EXTRACT - value: "password" - - name: PGHOST_TEMP - value: "127.0.0.1" - - name: PGUSER_TEMP - value: "postgres" - - name: PGDB_TEMP - value: "postgres" - - name: PGPASSWORD_TEMP - value: "password" - volumeMounts: - - name: postgres-schemma-volume - mountPath: /docker-entrypoint-initdb.d/00-pginitscript.sql - subPath: pginit.sql - readOnly: true - - name: postgres-crondump-volume - mountPath: /docker-entrypoint-initdb.d/01-pginitscript.sh - subPath: dump.sh - readOnly: true - - name: env-context - mountPath: /mnt/env-context - readOnly: true - - name: env-workdir - mountPath: /mnt/env-workdir - volumes: - - name: docker-config - projected: - sources: - - secret: - name: regcred - items: - - key: .dockerconfigjson - path: config.json - - name: postgres-schemma-volume - configMap: - name: postgres-schemma - - name: postgres-crondump-volume - configMap: - name: postgres-crondump - - name: env-context - hostPath: - path: /mnt/diadata - - name: env-workdir - emptyDir: {} diff --git a/deployments/k8s-yaml/data-influx.yaml b/deployments/k8s-yaml/data-influx.yaml deleted file mode 100755 index a266fd5de..000000000 --- a/deployments/k8s-yaml/data-influx.yaml +++ /dev/null @@ -1,55 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: data-influx - labels: - app: data-influx -spec: - selector: - matchLabels: - app: data-influx - template: - metadata: - labels: - app: data-influx - spec: - containers: - - name: data-influx - image: public.ecr.aws/docker/library/influxdb:1.8 - imagePullPolicy: IfNotPresent - ports: - - name: influx-server - containerPort: 8086 - env: - - name: INFLUXDB_HTTP_MAX_BODY_SIZE - value: "0" - - name: INFLUXDB_DATA_CACHE_MAX_MEMORY_SIZE - value: "4g" - - name: INFLUXDB_DATA_MAX_INDEX_LOG_FILE_SIZE - value: "8m" - - name: DOCKER_INFLUXDB_INIT_MODE - value: "setup" - - name: DOCKER_INFLUXDB_INIT_USERNAME - value: "test" - - name: DOCKER_INFLUXDB_INIT_PASSWORD - value: "testtest" - - name: DOCKER_INFLUXDB_INIT_ORG - value: "test_org" - - name: DOCKER_INFLUXDB_INIT_BUCKET - value: "test_bucket" - - name: DOCKER_INFLUXDB_INIT_ADMIN_TOKEN - value: "test_token" ---- -apiVersion: v1 -kind: Service -metadata: - name: data-influx - labels: - app: data-influx -spec: - ports: - - port: 8086 - targetPort: influx-server - selector: - app: data-influx - type: NodePort diff --git a/deployments/k8s-yaml/data-kafka.yaml b/deployments/k8s-yaml/data-kafka.yaml deleted file mode 100755 index 99d1fbb1d..000000000 --- a/deployments/k8s-yaml/data-kafka.yaml +++ /dev/null @@ -1,72 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: data-kafka - labels: - app: data-kafka -spec: - selector: - matchLabels: - app: data-kafka - template: - metadata: - labels: - app: data-kafka - spec: - containers: - - name: zookeeper - image: wurstmeister/zookeeper:latest - imagePullPolicy: IfNotPresent - ports: - - name: zookeep-server - containerPort: 2181 - env: - - name: ZOOKEEPER_CLIENT_PORT - value: "2181" - - name: ZOOKEEPER_TICK_TIME - value: "2000" - - name: kafka - image: wurstmeister/kafka:latest - imagePullPolicy: IfNotPresent - ports: - - name: kafka-server - containerPort: 9094 - env: - - name: HOSTNAME_COMMAND - value: "docker info | grep ^Name: | cut -d' ' -f 2" - - name: KAFKA_PORT - value: "9092" - - name: KAFKA_ZOOKEEPER_CONNECT - value: "localhost:2181" - - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP - value: "INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT" - - name: KAFKA_ADVERTISED_LISTENERS - value: "INSIDE://:9092,OUTSIDE://data-kafka.default.svc.cluster.local:9094" - - name: KAFKA_LISTENERS - value: "INSIDE://:9092,OUTSIDE://:9094" - - name: KAFKA_INTER_BROKER_LISTENER_NAME - value: "INSIDE" - - name: KAFKA_LOG_DIRS - value: "/kafka/kafka-logs-1" - - name: KAFKA_DELETE_TOPIC_ENABLE - value: "true" - - name: KAFKA_MESSAGE_MAX_BYTES - value: "50000000" ---- -apiVersion: v1 -kind: Service -metadata: - name: data-kafka - labels: - app: data-kafka -spec: - ports: - - port: 9094 - targetPort: kafka-server - name: kafka-server - - port: 2181 - targetPort: zookeep-server - name: zookeep-server - selector: - app: data-kafka - type: NodePort diff --git a/deployments/k8s-yaml/data-postgres-prepopulated.yaml b/deployments/k8s-yaml/data-postgres-prepopulated.yaml deleted file mode 100755 index 61052b859..000000000 --- a/deployments/k8s-yaml/data-postgres-prepopulated.yaml +++ /dev/null @@ -1,45 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: data-postgres - labels: - app: data-postgres -spec: - selector: - matchLabels: - app: data-postgres - template: - metadata: - labels: - app: data-postgres - spec: - imagePullSecrets: - - name: regcred-read - containers: - - name: data-postgres - image: diadata/diadata.postgres:latest - imagePullPolicy: Always - ports: - - name: postgres-server - containerPort: 5432 - env: - - name: POSTGRES_USER - value: postgres - - name: POSTGRES_DB - value: postgres - - name: POSTGRES_PASSWORD - value: password ---- -apiVersion: v1 -kind: Service -metadata: - name: data-postgres - labels: - app: data-postgres -spec: - ports: - - port: 5432 - targetPort: postgres-server - selector: - app: data-postgres - type: NodePort diff --git a/deployments/k8s-yaml/data-postgres.yaml b/deployments/k8s-yaml/data-postgres.yaml deleted file mode 100755 index e1adadfa4..000000000 --- a/deployments/k8s-yaml/data-postgres.yaml +++ /dev/null @@ -1,52 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: data-postgres - labels: - app: data-postgres -spec: - selector: - matchLabels: - app: data-postgres - template: - metadata: - labels: - app: data-postgres - spec: - containers: - - name: data-postgres - image: public.ecr.aws/docker/library/postgres:latest - imagePullPolicy: IfNotPresent - ports: - - name: postgres-server - containerPort: 5432 - env: - - name: POSTGRES_USER - value: postgres - - name: POSTGRES_DB - value: postgres - - name: POSTGRES_PASSWORD - value: password - volumeMounts: - - name: postgres-schemma-volume - mountPath: /docker-entrypoint-initdb.d/pginit.sql - subPath: pginit.sql - readOnly: true - volumes: - - name: postgres-schemma-volume - configMap: - name: postgres-schemma ---- -apiVersion: v1 -kind: Service -metadata: - name: data-postgres - labels: - app: data-postgres -spec: - ports: - - port: 5432 - targetPort: postgres-server - selector: - app: data-postgres - type: NodePort diff --git a/deployments/k8s-yaml/data-redis.yaml b/deployments/k8s-yaml/data-redis.yaml deleted file mode 100755 index 2831df278..000000000 --- a/deployments/k8s-yaml/data-redis.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: data-redis - labels: - app: data-redis -spec: - selector: - matchLabels: - app: data-redis - template: - metadata: - labels: - app: data-redis - spec: - containers: - - name: data-redis - image: public.ecr.aws/docker/library/redis:7 - imagePullPolicy: IfNotPresent - ports: - - name: redis-server - containerPort: 6379 ---- -apiVersion: v1 -kind: Service -metadata: - name: data-redis - labels: - app: data-redis -spec: - ports: - - port: 6379 - targetPort: redis-server - selector: - app: data-redis - type: NodePort diff --git a/deployments/k8s-yaml/job-ping-influx.yaml b/deployments/k8s-yaml/job-ping-influx.yaml deleted file mode 100755 index 573277697..000000000 --- a/deployments/k8s-yaml/job-ping-influx.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: ping-influx -spec: - ttlSecondsAfterFinished: 10 - template: - spec: - containers: - - name: ping-influx - image: public.ecr.aws/docker/library/influxdb:1.8 - env: - - name: INFLUXDB_URL - value: "data-influx.default.svc.cluster.local" - - name: INFLUXDB_USERNAME - value: "test" - - name: INFLUXDB_PASSWORD - value: "testtest" - command: - [ - "sh", - "-c", - "influx -host $INFLUXDB_URL -username $INFLUXDB_USERNAME -password $INFLUXDB_PASSWORD -execute 'SHOW DATABASES'", - ] - restartPolicy: Never - backoffLimit: 0 diff --git a/deployments/k8s-yaml/job-ping-kafka.yaml b/deployments/k8s-yaml/job-ping-kafka.yaml deleted file mode 100755 index 0275a39cd..000000000 --- a/deployments/k8s-yaml/job-ping-kafka.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: ping-kafka -spec: - ttlSecondsAfterFinished: 10 - template: - spec: - containers: - - name: ping-kafka - image: confluentinc/cp-kafkacat:6.2.1 - env: - - name: KAFKA_BROKER_URL - value: "data-kafka.default.svc.cluster.local:9094" - command: ["sh", "-c", "kafkacat -b $KAFKA_BROKER_URL -L"] - restartPolicy: Never - backoffLimit: 0 diff --git a/deployments/k8s-yaml/job-ping-postgres.yaml b/deployments/k8s-yaml/job-ping-postgres.yaml deleted file mode 100755 index 63470820b..000000000 --- a/deployments/k8s-yaml/job-ping-postgres.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: ping-postgres -spec: - ttlSecondsAfterFinished: 10 - template: - spec: - containers: - - name: ping-postgres - image: public.ecr.aws/docker/library/postgres:15-alpine - env: - - name: PGHOST - value: "data-postgres.default.svc.cluster.local" - - name: PGUSER - value: "postgres" - - name: PGPASSWORD - value: "password" - command: ["sh", "-c", "pg_isready -h $PGHOST -U $PGUSER"] - restartPolicy: Never - backoffLimit: 0 diff --git a/deployments/k8s-yaml/job-ping-redis.yaml b/deployments/k8s-yaml/job-ping-redis.yaml deleted file mode 100755 index 7bda82756..000000000 --- a/deployments/k8s-yaml/job-ping-redis.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: ping-redis -spec: - ttlSecondsAfterFinished: 10 - template: - spec: - containers: - - name: ping-redis - image: public.ecr.aws/docker/library/redis:7 - env: - - name: REDISCLI_URL - value: "redis://data-redis.default.svc.cluster.local:6379/0" - command: ["sh", "-c", "redis-cli -u $REDISCLI_URL ping"] - restartPolicy: Never - backoffLimit: 0 diff --git a/deployments/k8s-yaml/job-prepare.yaml b/deployments/k8s-yaml/job-prepare.yaml deleted file mode 100755 index e2ef3db68..000000000 --- a/deployments/k8s-yaml/job-prepare.yaml +++ /dev/null @@ -1,129 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: job-prepare -spec: - ttlSecondsAfterFinished: 10 - template: - spec: - containers: - - name: job-prepare-blockchainservice - image: dia.blockchainservice.dev:latest - imagePullPolicy: IfNotPresent - command: ["blockchainservice"] - args: [] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: job-prepare-assetcollectionservice-ethassets - image: dia.assetcollectionservice.dev:latest - imagePullPolicy: Never - command: ["assetCollectionService"] - args: ["-source=assetlists", "-secret=eth_assets", "-caching=true"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: job-prepare-assetcollectionservice-nonethassets - image: dia.assetcollectionservice.dev:latest - imagePullPolicy: Never - command: ["assetCollectionService"] - args: - ["-source=assetlists", "-secret=non_eth_assets", "-caching=true"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: job-prepare-assetcollectionservice-fiatassets - image: dia.assetcollectionservice.dev:latest - imagePullPolicy: Never - command: ["assetCollectionService"] - args: ["-source=assetlists", "-secret=fiat_assets", "-caching=true"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - initContainers: - - name: check-postgres-connection - image: public.ecr.aws/docker/library/postgres:15-alpine - env: - - name: PGHOST - value: "data-postgres.default.svc.cluster.local" - - name: PGUSER - value: "postgres" - - name: PGPASSWORD - value: "password" - command: - [ - "sh", - "-c", - "until pg_isready -h $PGHOST -U $PGUSER; do sleep 1; done", - ] - - name: check-redis-connection - image: public.ecr.aws/docker/library/redis:7 - env: - - name: REDISCLI_URL - value: "redis://data-redis.default.svc.cluster.local:6379/0" - command: - [ - "sh", - "-c", - "until redis-cli -u $REDISCLI_URL ping; do sleep 1; done", - ] - - name: check-influxdb-connection - image: public.ecr.aws/docker/library/influxdb:1.8 - env: - - name: INFLUXDB_URL - value: "data-influx.default.svc.cluster.local" - - name: INFLUXDB_USERNAME - value: "test" - - name: INFLUXDB_PASSWORD - value: "testtest" - command: - [ - "sh", - "-c", - "until influx -host $INFLUXDB_URL -username $INFLUXDB_USERNAME -password $INFLUXDB_PASSWORD -execute 'SHOW DATABASES'; do sleep 1; done", - ] - - name: check-kafka-connection - image: confluentinc/cp-kafkacat:6.2.1 - env: - - name: KAFKA_BROKER_URL - value: "data-kafka.default.svc.cluster.local:9094" - command: - [ - "sh", - "-c", - "until kafkacat -b $KAFKA_BROKER_URL -L; do sleep 1; done", - ] - restartPolicy: Never - backoffLimit: 0 diff --git a/deployments/k8s-yaml/scraper-cex-bitfinex.yaml b/deployments/k8s-yaml/scraper-cex-bitfinex.yaml deleted file mode 100755 index 20bb43ee3..000000000 --- a/deployments/k8s-yaml/scraper-cex-bitfinex.yaml +++ /dev/null @@ -1,107 +0,0 @@ -apiVersion: "v1" -kind: Pod -metadata: - name: scraper-exchange-bitfinex -spec: - containers: - - name: scraper-exchange-bitfinex - image: dia.genericcollector.dev:latest - imagePullPolicy: Never - command: ["collector"] - args: ["-exchange=Bitfinex", "-mode=current", "-pairsfile=true"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: INFLUXURL - value: "http://data-influx.default.svc.cluster.local:8086" - - name: INFLUXUSER - value: "test" - - name: INFLUXPASSWORD - value: "testtest" - - name: REDISURL - value: "data-redis.default.svc.cluster.local:6379" - - name: KAFKAURL - value: "data-kafka.default.svc.cluster.local:9094" - volumeMounts: - - name: env-context - mountPath: /mnt/env-context - readOnly: true - initContainers: - - name: check-postgres-connection - image: public.ecr.aws/docker/library/postgres:15-alpine - env: - - name: PGHOST - value: "data-postgres.default.svc.cluster.local" - - name: PGUSER - value: "postgres" - - name: PGPASSWORD - value: "password" - command: - ["sh", "-c", "until pg_isready -h $PGHOST -U $PGUSER; do sleep 1; done"] - - name: check-redis-connection - image: public.ecr.aws/docker/library/redis:7 - env: - - name: REDISCLI_URL - value: "redis://data-redis.default.svc.cluster.local:6379/0" - command: - ["sh", "-c", "until redis-cli -u $REDISCLI_URL ping; do sleep 1; done"] - - name: check-influxdb-connection - image: public.ecr.aws/docker/library/influxdb:1.8 - env: - - name: INFLUXDB_URL - value: "data-influx.default.svc.cluster.local" - - name: INFLUXDB_USERNAME - value: "test" - - name: INFLUXDB_PASSWORD - value: "testtest" - command: - [ - "sh", - "-c", - "until influx -host $INFLUXDB_URL -username $INFLUXDB_USERNAME -password $INFLUXDB_PASSWORD -execute 'SHOW DATABASES'; do sleep 1; done", - ] - - name: check-kafka-connection - image: confluentinc/cp-kafkacat:6.2.1 - env: - - name: KAFKA_BROKER_URL - value: "data-kafka.default.svc.cluster.local:9094" - command: - ["sh", "-c", "until kafkacat -b $KAFKA_BROKER_URL -L; do sleep 1; done"] - - name: scraper-exchange-bitfinex-pairdiscovery - image: dia.pairdiscoveryservice.dev:latest - imagePullPolicy: Never - command: ["pairDiscoveryService"] - args: ["-exchange=Bitfinex", "-mode=verification"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: INFLUXURL - value: "http://data-influx.default.svc.cluster.local:8086" - - name: INFLUXUSER - value: "test" - - name: INFLUXPASSWORD - value: "testtest" - - name: REDISURL - value: "data-redis.default.svc.cluster.local:6379" - - name: KAFKAURL - value: "data-kafka.default.svc.cluster.local:9094" - volumes: - - name: env-context - hostPath: - path: /mnt/diadata diff --git a/deployments/k8s-yaml/scraper-cex-bitmart.yaml b/deployments/k8s-yaml/scraper-cex-bitmart.yaml deleted file mode 100755 index 6d7f2b71b..000000000 --- a/deployments/k8s-yaml/scraper-cex-bitmart.yaml +++ /dev/null @@ -1,107 +0,0 @@ -apiVersion: "v1" -kind: Pod -metadata: - name: scraper-exchange-bitmart -spec: - containers: - - name: scraper-exchange-bitmart - image: dia.genericcollector.dev:latest - imagePullPolicy: Never - command: ["collector"] - args: ["-exchange=BitMart", "-mode=current", "-pairsfile=true"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: INFLUXURL - value: "http://data-influx.default.svc.cluster.local:8086" - - name: INFLUXUSER - value: "test" - - name: INFLUXPASSWORD - value: "testtest" - - name: REDISURL - value: "data-redis.default.svc.cluster.local:6379" - - name: KAFKAURL - value: "data-kafka.default.svc.cluster.local:9094" - volumeMounts: - - name: env-context - mountPath: /mnt/env-context - readOnly: true - initContainers: - - name: check-postgres-connection - image: public.ecr.aws/docker/library/postgres:15-alpine - env: - - name: PGHOST - value: "data-postgres.default.svc.cluster.local" - - name: PGUSER - value: "postgres" - - name: PGPASSWORD - value: "password" - command: - ["sh", "-c", "until pg_isready -h $PGHOST -U $PGUSER; do sleep 1; done"] - - name: check-redis-connection - image: public.ecr.aws/docker/library/redis:7 - env: - - name: REDISCLI_URL - value: "redis://data-redis.default.svc.cluster.local:6379/0" - command: - ["sh", "-c", "until redis-cli -u $REDISCLI_URL ping; do sleep 1; done"] - - name: check-influxdb-connection - image: public.ecr.aws/docker/library/influxdb:1.8 - env: - - name: INFLUXDB_URL - value: "data-influx.default.svc.cluster.local" - - name: INFLUXDB_USERNAME - value: "test" - - name: INFLUXDB_PASSWORD - value: "testtest" - command: - [ - "sh", - "-c", - "until influx -host $INFLUXDB_URL -username $INFLUXDB_USERNAME -password $INFLUXDB_PASSWORD -execute 'SHOW DATABASES'; do sleep 1; done", - ] - - name: check-kafka-connection - image: confluentinc/cp-kafkacat:6.2.1 - env: - - name: KAFKA_BROKER_URL - value: "data-kafka.default.svc.cluster.local:9094" - command: - ["sh", "-c", "until kafkacat -b $KAFKA_BROKER_URL -L; do sleep 1; done"] - - name: scraper-exchange-bitmart-pairdiscovery - image: dia.pairdiscoveryservice.dev:latest - imagePullPolicy: Never - command: ["pairDiscoveryService"] - args: ["-exchange=BitMart", "-mode=verification"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: INFLUXURL - value: "http://data-influx.default.svc.cluster.local:8086" - - name: INFLUXUSER - value: "test" - - name: INFLUXPASSWORD - value: "testtest" - - name: REDISURL - value: "data-redis.default.svc.cluster.local:6379" - - name: KAFKAURL - value: "data-kafka.default.svc.cluster.local:9094" - volumes: - - name: env-context - hostPath: - path: /mnt/diadata diff --git a/deployments/k8s-yaml/scraper-cex-bittrex.yaml b/deployments/k8s-yaml/scraper-cex-bittrex.yaml deleted file mode 100755 index aa9954668..000000000 --- a/deployments/k8s-yaml/scraper-cex-bittrex.yaml +++ /dev/null @@ -1,107 +0,0 @@ -apiVersion: "v1" -kind: Pod -metadata: - name: scraper-exchange-bittrex -spec: - containers: - - name: scraper-exchange-bittrex - image: dia.genericcollector.dev:latest - imagePullPolicy: Never - command: ["collector"] - args: ["-exchange=Bittrex", "-mode=current", "-pairsfile=true"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: INFLUXURL - value: "http://data-influx.default.svc.cluster.local:8086" - - name: INFLUXUSER - value: "test" - - name: INFLUXPASSWORD - value: "testtest" - - name: REDISURL - value: "data-redis.default.svc.cluster.local:6379" - - name: KAFKAURL - value: "data-kafka.default.svc.cluster.local:9094" - volumeMounts: - - name: env-context - mountPath: /mnt/env-context - readOnly: true - initContainers: - - name: check-postgres-connection - image: public.ecr.aws/docker/library/postgres:15-alpine - env: - - name: PGHOST - value: "data-postgres.default.svc.cluster.local" - - name: PGUSER - value: "postgres" - - name: PGPASSWORD - value: "password" - command: - ["sh", "-c", "until pg_isready -h $PGHOST -U $PGUSER; do sleep 1; done"] - - name: check-redis-connection - image: public.ecr.aws/docker/library/redis:7 - env: - - name: REDISCLI_URL - value: "redis://data-redis.default.svc.cluster.local:6379/0" - command: - ["sh", "-c", "until redis-cli -u $REDISCLI_URL ping; do sleep 1; done"] - - name: check-influxdb-connection - image: public.ecr.aws/docker/library/influxdb:1.8 - env: - - name: INFLUXDB_URL - value: "data-influx.default.svc.cluster.local" - - name: INFLUXDB_USERNAME - value: "test" - - name: INFLUXDB_PASSWORD - value: "testtest" - command: - [ - "sh", - "-c", - "until influx -host $INFLUXDB_URL -username $INFLUXDB_USERNAME -password $INFLUXDB_PASSWORD -execute 'SHOW DATABASES'; do sleep 1; done", - ] - - name: check-kafka-connection - image: confluentinc/cp-kafkacat:6.2.1 - env: - - name: KAFKA_BROKER_URL - value: "data-kafka.default.svc.cluster.local:9094" - command: - ["sh", "-c", "until kafkacat -b $KAFKA_BROKER_URL -L; do sleep 1; done"] - - name: scraper-exchange-bittrex-pairdiscovery - image: dia.pairdiscoveryservice.dev:latest - imagePullPolicy: Never - command: ["pairDiscoveryService"] - args: ["-exchange=Bittrex", "-mode=verification"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: INFLUXURL - value: "http://data-influx.default.svc.cluster.local:8086" - - name: INFLUXUSER - value: "test" - - name: INFLUXPASSWORD - value: "testtest" - - name: REDISURL - value: "data-redis.default.svc.cluster.local:6379" - - name: KAFKAURL - value: "data-kafka.default.svc.cluster.local:9094" - volumes: - - name: env-context - hostPath: - path: /mnt/diadata diff --git a/deployments/k8s-yaml/scraper-cex-coinbase.yaml b/deployments/k8s-yaml/scraper-cex-coinbase.yaml deleted file mode 100755 index 5223b28c9..000000000 --- a/deployments/k8s-yaml/scraper-cex-coinbase.yaml +++ /dev/null @@ -1,107 +0,0 @@ -apiVersion: "v1" -kind: Pod -metadata: - name: scraper-exchange-coinbase -spec: - containers: - - name: scraper-exchange-coinbase - image: dia.genericcollector.dev:latest - imagePullPolicy: Never - command: ["collector"] - args: ["-exchange=CoinBase", "-mode=current", "-pairsfile=true"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: INFLUXURL - value: "http://data-influx.default.svc.cluster.local:8086" - - name: INFLUXUSER - value: "test" - - name: INFLUXPASSWORD - value: "testtest" - - name: REDISURL - value: "data-redis.default.svc.cluster.local:6379" - - name: KAFKAURL - value: "data-kafka.default.svc.cluster.local:9094" - volumeMounts: - - name: env-context - mountPath: /mnt/env-context - readOnly: true - initContainers: - - name: check-postgres-connection - image: public.ecr.aws/docker/library/postgres:15-alpine - env: - - name: PGHOST - value: "data-postgres.default.svc.cluster.local" - - name: PGUSER - value: "postgres" - - name: PGPASSWORD - value: "password" - command: - ["sh", "-c", "until pg_isready -h $PGHOST -U $PGUSER; do sleep 1; done"] - - name: check-redis-connection - image: public.ecr.aws/docker/library/redis:7 - env: - - name: REDISCLI_URL - value: "redis://data-redis.default.svc.cluster.local:6379/0" - command: - ["sh", "-c", "until redis-cli -u $REDISCLI_URL ping; do sleep 1; done"] - - name: check-influxdb-connection - image: public.ecr.aws/docker/library/influxdb:1.8 - env: - - name: INFLUXDB_URL - value: "data-influx.default.svc.cluster.local" - - name: INFLUXDB_USERNAME - value: "test" - - name: INFLUXDB_PASSWORD - value: "testtest" - command: - [ - "sh", - "-c", - "until influx -host $INFLUXDB_URL -username $INFLUXDB_USERNAME -password $INFLUXDB_PASSWORD -execute 'SHOW DATABASES'; do sleep 1; done", - ] - - name: check-kafka-connection - image: confluentinc/cp-kafkacat:6.2.1 - env: - - name: KAFKA_BROKER_URL - value: "data-kafka.default.svc.cluster.local:9094" - command: - ["sh", "-c", "until kafkacat -b $KAFKA_BROKER_URL -L; do sleep 1; done"] - - name: scraper-exchange-example-coinbase-pairdiscovery - image: dia.pairdiscoveryservice.dev:latest - imagePullPolicy: Never - command: ["pairDiscoveryService"] - args: ["-exchange=CoinBase", "-mode=verification"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: INFLUXURL - value: "http://data-influx.default.svc.cluster.local:8086" - - name: INFLUXUSER - value: "test" - - name: INFLUXPASSWORD - value: "testtest" - - name: REDISURL - value: "data-redis.default.svc.cluster.local:6379" - - name: KAFKAURL - value: "data-kafka.default.svc.cluster.local:9094" - volumes: - - name: env-context - hostPath: - path: /mnt/diadata diff --git a/deployments/k8s-yaml/scraper-cex-mexc.yaml b/deployments/k8s-yaml/scraper-cex-mexc.yaml deleted file mode 100755 index b85ed8194..000000000 --- a/deployments/k8s-yaml/scraper-cex-mexc.yaml +++ /dev/null @@ -1,107 +0,0 @@ -apiVersion: "v1" -kind: Pod -metadata: - name: scraper-exchange-mexc -spec: - containers: - - name: scraper-exchange-mexc - image: dia.genericcollector.dev:latest - imagePullPolicy: Never - command: ["collector"] - args: ["-exchange=MEXC", "-mode=current", "-pairsfile=true"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: INFLUXURL - value: "http://data-influx.default.svc.cluster.local:8086" - - name: INFLUXUSER - value: "test" - - name: INFLUXPASSWORD - value: "testtest" - - name: REDISURL - value: "data-redis.default.svc.cluster.local:6379" - - name: KAFKAURL - value: "data-kafka.default.svc.cluster.local:9094" - volumeMounts: - - name: env-context - mountPath: /mnt/env-context - readOnly: true - initContainers: - - name: check-postgres-connection - image: public.ecr.aws/docker/library/postgres:15-alpine - env: - - name: PGHOST - value: "data-postgres.default.svc.cluster.local" - - name: PGUSER - value: "postgres" - - name: PGPASSWORD - value: "password" - command: - ["sh", "-c", "until pg_isready -h $PGHOST -U $PGUSER; do sleep 1; done"] - - name: check-redis-connection - image: public.ecr.aws/docker/library/redis:7 - env: - - name: REDISCLI_URL - value: "redis://data-redis.default.svc.cluster.local:6379/0" - command: - ["sh", "-c", "until redis-cli -u $REDISCLI_URL ping; do sleep 1; done"] - - name: check-influxdb-connection - image: public.ecr.aws/docker/library/influxdb:1.8 - env: - - name: INFLUXDB_URL - value: "data-influx.default.svc.cluster.local" - - name: INFLUXDB_USERNAME - value: "test" - - name: INFLUXDB_PASSWORD - value: "testtest" - command: - [ - "sh", - "-c", - "until influx -host $INFLUXDB_URL -username $INFLUXDB_USERNAME -password $INFLUXDB_PASSWORD -execute 'SHOW DATABASES'; do sleep 1; done", - ] - - name: check-kafka-connection - image: confluentinc/cp-kafkacat:6.2.1 - env: - - name: KAFKA_BROKER_URL - value: "data-kafka.default.svc.cluster.local:9094" - command: - ["sh", "-c", "until kafkacat -b $KAFKA_BROKER_URL -L; do sleep 1; done"] - - name: scraper-exchange-mexc-pairdiscovery - image: dia.pairdiscoveryservice.dev:latest - imagePullPolicy: Never - command: ["pairDiscoveryService"] - args: ["-exchange=MEXC", "-mode=verification"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: INFLUXURL - value: "http://data-influx.default.svc.cluster.local:8086" - - name: INFLUXUSER - value: "test" - - name: INFLUXPASSWORD - value: "testtest" - - name: REDISURL - value: "data-redis.default.svc.cluster.local:6379" - - name: KAFKAURL - value: "data-kafka.default.svc.cluster.local:9094" - volumes: - - name: env-context - hostPath: - path: /mnt/diadata diff --git a/deployments/k8s-yaml/scraper-cex.yaml b/deployments/k8s-yaml/scraper-cex.yaml deleted file mode 100755 index 00988bd90..000000000 --- a/deployments/k8s-yaml/scraper-cex.yaml +++ /dev/null @@ -1,107 +0,0 @@ -apiVersion: "v1" -kind: Pod -metadata: - name: scraper-exchange-cex -spec: - containers: - - name: scraper-exchange-cex - image: dia.genericcollector.dev:latest - imagePullPolicy: Never - command: ["tail"] - args: ["-f", "/dev/null"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: INFLUXURL - value: "http://data-influx.default.svc.cluster.local:8086" - - name: INFLUXUSER - value: "test" - - name: INFLUXPASSWORD - value: "testtest" - - name: REDISURL - value: "data-redis.default.svc.cluster.local:6379" - - name: KAFKAURL - value: "data-kafka.default.svc.cluster.local:9094" - volumeMounts: - - name: env-context - mountPath: /mnt/env-context - readOnly: true - - name: scraper-exchange-cex-pairdiscovery - image: dia.pairdiscoveryservice.dev:latest - imagePullPolicy: Never - command: ["tail"] - args: ["-f", "/dev/null"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: INFLUXURL - value: "http://data-influx.default.svc.cluster.local:8086" - - name: INFLUXUSER - value: "test" - - name: INFLUXPASSWORD - value: "testtest" - - name: REDISURL - value: "data-redis.default.svc.cluster.local:6379" - - name: KAFKAURL - value: "data-kafka.default.svc.cluster.local:9094" - initContainers: - - name: check-postgres-connection - image: public.ecr.aws/docker/library/postgres:15-alpine - env: - - name: PGHOST - value: "data-postgres.default.svc.cluster.local" - - name: PGUSER - value: "postgres" - - name: PGPASSWORD - value: "password" - command: - ["sh", "-c", "until pg_isready -h $PGHOST -U $PGUSER; do sleep 1; done"] - - name: check-redis-connection - image: public.ecr.aws/docker/library/redis:7 - env: - - name: REDISCLI_URL - value: "redis://data-redis.default.svc.cluster.local:6379/0" - command: - ["sh", "-c", "until redis-cli -u $REDISCLI_URL ping; do sleep 1; done"] - - name: check-influxdb-connection - image: public.ecr.aws/docker/library/influxdb:1.8 - env: - - name: INFLUXDB_URL - value: "data-influx.default.svc.cluster.local" - - name: INFLUXDB_USERNAME - value: "test" - - name: INFLUXDB_PASSWORD - value: "testtest" - command: - [ - "sh", - "-c", - "until influx -host $INFLUXDB_URL -username $INFLUXDB_USERNAME -password $INFLUXDB_PASSWORD -execute 'SHOW DATABASES'; do sleep 1; done", - ] - - name: check-kafka-connection - image: confluentinc/cp-kafkacat:6.2.1 - env: - - name: KAFKA_BROKER_URL - value: "data-kafka.default.svc.cluster.local:9094" - command: - ["sh", "-c", "until kafkacat -b $KAFKA_BROKER_URL -L; do sleep 1; done"] - volumes: - - name: env-context - hostPath: - path: /mnt/diadata diff --git a/deployments/k8s-yaml/scraper-dex-curve.yaml b/deployments/k8s-yaml/scraper-dex-curve.yaml deleted file mode 100755 index e220c998e..000000000 --- a/deployments/k8s-yaml/scraper-dex-curve.yaml +++ /dev/null @@ -1,85 +0,0 @@ -apiVersion: "v1" -kind: Pod -metadata: - name: scraper-exchange-curve -spec: - containers: - - name: scraper-exchange-curve - image: dia.genericcollector.dev:latest - imagePullPolicy: Never - command: ["collector"] - args: ["-exchange=Curvefi", "-mode=current", "-pairsfile=true"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: INFLUXURL - value: "http://data-influx.default.svc.cluster.local:8086" - - name: INFLUXUSER - value: "test" - - name: INFLUXPASSWORD - value: "testtest" - - name: REDISURL - value: "data-redis.default.svc.cluster.local:6379" - - name: KAFKAURL - value: "data-kafka.default.svc.cluster.local:9094" - - name: ETHEREUM_URI_REST - value: "https://mainnet.infura.io/v3/fc2e181454424779b1f09e1f47304bc3" - - name: ETHEREUM_URI_WS - value: "wss://mainnet.infura.io/ws/v3/fc2e181454424779b1f09e1f47304bc3" - volumeMounts: - - name: env-context - mountPath: /mnt/env-context - readOnly: true - initContainers: - - name: check-postgres-connection - image: public.ecr.aws/docker/library/postgres:15-alpine - env: - - name: PGHOST - value: "data-postgres.default.svc.cluster.local" - - name: PGUSER - value: "postgres" - - name: PGPASSWORD - value: "password" - command: - ["sh", "-c", "until pg_isready -h $PGHOST -U $PGUSER; do sleep 1; done"] - - name: check-redis-connection - image: public.ecr.aws/docker/library/redis:7 - env: - - name: REDISCLI_URL - value: "redis://data-redis.default.svc.cluster.local:6379/0" - command: - ["sh", "-c", "until redis-cli -u $REDISCLI_URL ping; do sleep 1; done"] - - name: check-influxdb-connection - image: public.ecr.aws/docker/library/influxdb:1.8 - env: - - name: INFLUXDB_URL - value: "data-influx.default.svc.cluster.local" - - name: INFLUXDB_USERNAME - value: "test" - - name: INFLUXDB_PASSWORD - value: "testtest" - command: - [ - "sh", - "-c", - "until influx -host $INFLUXDB_URL -username $INFLUXDB_USERNAME -password $INFLUXDB_PASSWORD -execute 'SHOW DATABASES'; do sleep 1; done", - ] - - name: check-kafka-connection - image: confluentinc/cp-kafkacat:6.2.1 - env: - - name: KAFKA_BROKER_URL - value: "data-kafka.default.svc.cluster.local:9094" - command: - ["sh", "-c", "until kafkacat -b $KAFKA_BROKER_URL -L; do sleep 1; done"] - volumes: - - name: env-context - hostPath: - path: /mnt/diadata diff --git a/deployments/k8s-yaml/scraper-dex-orca.yaml b/deployments/k8s-yaml/scraper-dex-orca.yaml deleted file mode 100755 index 06fed3d26..000000000 --- a/deployments/k8s-yaml/scraper-dex-orca.yaml +++ /dev/null @@ -1,83 +0,0 @@ -apiVersion: "v1" -kind: Pod -metadata: - name: scraper-exchange-orca -spec: - containers: - - name: scraper-exchange-orca - image: dia.genericcollector.dev:latest - imagePullPolicy: Never - command: ["collector"] - args: ["-exchange=Orca", "-mode=current", "-pairsfile=true"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: INFLUXURL - value: "http://data-influx.default.svc.cluster.local:8086" - - name: INFLUXUSER - value: "test" - - name: INFLUXPASSWORD - value: "testtest" - - name: REDISURL - value: "data-redis.default.svc.cluster.local:6379" - - name: KAFKAURL - value: "data-kafka.default.svc.cluster.local:9094" - - name: SOLANA_URI_REST - value: "https://try-rpc.mainnet.solana.blockdaemon.tech/" - volumeMounts: - - name: env-context - mountPath: /mnt/env-context - readOnly: true - initContainers: - - name: check-postgres-connection - image: public.ecr.aws/docker/library/postgres:15-alpine - env: - - name: PGHOST - value: "data-postgres.default.svc.cluster.local" - - name: PGUSER - value: "postgres" - - name: PGPASSWORD - value: "password" - command: - ["sh", "-c", "until pg_isready -h $PGHOST -U $PGUSER; do sleep 1; done"] - - name: check-redis-connection - image: public.ecr.aws/docker/library/redis:7 - env: - - name: REDISCLI_URL - value: "redis://data-redis.default.svc.cluster.local:6379/0" - command: - ["sh", "-c", "until redis-cli -u $REDISCLI_URL ping; do sleep 1; done"] - - name: check-influxdb-connection - image: public.ecr.aws/docker/library/influxdb:1.8 - env: - - name: INFLUXDB_URL - value: "data-influx.default.svc.cluster.local" - - name: INFLUXDB_USERNAME - value: "test" - - name: INFLUXDB_PASSWORD - value: "testtest" - command: - [ - "sh", - "-c", - "until influx -host $INFLUXDB_URL -username $INFLUXDB_USERNAME -password $INFLUXDB_PASSWORD -execute 'SHOW DATABASES'; do sleep 1; done", - ] - - name: check-kafka-connection - image: confluentinc/cp-kafkacat:6.2.1 - env: - - name: KAFKA_BROKER_URL - value: "data-kafka.default.svc.cluster.local:9094" - command: - ["sh", "-c", "until kafkacat -b $KAFKA_BROKER_URL -L; do sleep 1; done"] - volumes: - - name: env-context - hostPath: - path: /mnt/diadata diff --git a/deployments/k8s-yaml/scraper-dex-platypus.yaml b/deployments/k8s-yaml/scraper-dex-platypus.yaml deleted file mode 100755 index 3c8b8feb0..000000000 --- a/deployments/k8s-yaml/scraper-dex-platypus.yaml +++ /dev/null @@ -1,81 +0,0 @@ -apiVersion: "v1" -kind: Pod -metadata: - name: scraper-exchange-platypus -spec: - containers: - - name: scraper-exchange-platypus - image: dia.genericcollector.dev:latest - imagePullPolicy: Never - command: ["collector"] - args: ["-exchange=PlatypusFinance", "-mode=current", "-pairsfile=true"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: INFLUXURL - value: "http://data-influx.default.svc.cluster.local:8086" - - name: INFLUXUSER - value: "test" - - name: INFLUXPASSWORD - value: "testtest" - - name: REDISURL - value: "data-redis.default.svc.cluster.local:6379" - - name: KAFKAURL - value: "data-kafka.default.svc.cluster.local:9094" - volumeMounts: - - name: env-context - mountPath: /mnt/env-context - readOnly: true - initContainers: - - name: check-postgres-connection - image: public.ecr.aws/docker/library/postgres:15-alpine - env: - - name: PGHOST - value: "data-postgres.default.svc.cluster.local" - - name: PGUSER - value: "postgres" - - name: PGPASSWORD - value: "password" - command: - ["sh", "-c", "until pg_isready -h $PGHOST -U $PGUSER; do sleep 1; done"] - - name: check-redis-connection - image: public.ecr.aws/docker/library/redis:7 - env: - - name: REDISCLI_URL - value: "redis://data-redis.default.svc.cluster.local:6379/0" - command: - ["sh", "-c", "until redis-cli -u $REDISCLI_URL ping; do sleep 1; done"] - - name: check-influxdb-connection - image: public.ecr.aws/docker/library/influxdb:1.8 - env: - - name: INFLUXDB_URL - value: "data-influx.default.svc.cluster.local" - - name: INFLUXDB_USERNAME - value: "test" - - name: INFLUXDB_PASSWORD - value: "testtest" - command: - [ - "sh", - "-c", - "until influx -host $INFLUXDB_URL -username $INFLUXDB_USERNAME -password $INFLUXDB_PASSWORD -execute 'SHOW DATABASES'; do sleep 1; done", - ] - - name: check-kafka-connection - image: confluentinc/cp-kafkacat:6.2.1 - env: - - name: KAFKA_BROKER_URL - value: "data-kafka.default.svc.cluster.local:9094" - command: - ["sh", "-c", "until kafkacat -b $KAFKA_BROKER_URL -L; do sleep 1; done"] - volumes: - - name: env-context - hostPath: - path: /mnt/diadata diff --git a/deployments/k8s-yaml/scraper-dex.yaml b/deployments/k8s-yaml/scraper-dex.yaml deleted file mode 100755 index f0d020b50..000000000 --- a/deployments/k8s-yaml/scraper-dex.yaml +++ /dev/null @@ -1,81 +0,0 @@ -apiVersion: "v1" -kind: Pod -metadata: - name: scraper-exchange-dex -spec: - containers: - - name: scraper-exchange-dex - image: dia.genericcollector.dev:latest - imagePullPolicy: Never - command: ["tail"] - args: ["-f", "/dev/null"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: INFLUXURL - value: "http://data-influx.default.svc.cluster.local:8086" - - name: INFLUXUSER - value: "test" - - name: INFLUXPASSWORD - value: "testtest" - - name: REDISURL - value: "data-redis.default.svc.cluster.local:6379" - - name: KAFKAURL - value: "data-kafka.default.svc.cluster.local:9094" - volumeMounts: - - name: env-context - mountPath: /mnt/env-context - readOnly: true - initContainers: - - name: check-postgres-connection - image: public.ecr.aws/docker/library/postgres:15-alpine - env: - - name: PGHOST - value: "data-postgres.default.svc.cluster.local" - - name: PGUSER - value: "postgres" - - name: PGPASSWORD - value: "password" - command: - ["sh", "-c", "until pg_isready -h $PGHOST -U $PGUSER; do sleep 1; done"] - - name: check-redis-connection - image: public.ecr.aws/docker/library/redis:7 - env: - - name: REDISCLI_URL - value: "redis://data-redis.default.svc.cluster.local:6379/0" - command: - ["sh", "-c", "until redis-cli -u $REDISCLI_URL ping; do sleep 1; done"] - - name: check-influxdb-connection - image: public.ecr.aws/docker/library/influxdb:1.8 - env: - - name: INFLUXDB_URL - value: "data-influx.default.svc.cluster.local" - - name: INFLUXDB_USERNAME - value: "test" - - name: INFLUXDB_PASSWORD - value: "testtest" - command: - [ - "sh", - "-c", - "until influx -host $INFLUXDB_URL -username $INFLUXDB_USERNAME -password $INFLUXDB_PASSWORD -execute 'SHOW DATABASES'; do sleep 1; done", - ] - - name: check-kafka-connection - image: confluentinc/cp-kafkacat:6.2.1 - env: - - name: KAFKA_BROKER_URL - value: "data-kafka.default.svc.cluster.local:9094" - command: - ["sh", "-c", "until kafkacat -b $KAFKA_BROKER_URL -L; do sleep 1; done"] - volumes: - - name: env-context - hostPath: - path: /mnt/diadata diff --git a/deployments/k8s-yaml/service-filtersblockservice.yaml b/deployments/k8s-yaml/service-filtersblockservice.yaml deleted file mode 100755 index a571035da..000000000 --- a/deployments/k8s-yaml/service-filtersblockservice.yaml +++ /dev/null @@ -1,95 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: service-filtersblockservice - labels: - app: service-filtersblockservice -spec: - selector: - matchLabels: - app: service-filtersblockservice - template: - metadata: - labels: - app: service-filtersblockservice - spec: - containers: - - name: service-filtersblockservice - image: dia.filtersblockservice.dev:latest - imagePullPolicy: Never - command: ["filtersBlockService"] - args: ["-replayInflux=false", "-historical=true", "-testing=true"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: INFLUXURL - value: "http://data-influx.default.svc.cluster.local:8086" - - name: INFLUXUSER - value: "test" - - name: INFLUXPASSWORD - value: "testtest" - - name: REDISURL - value: "data-redis.default.svc.cluster.local:6379" - - name: KAFKAURL - value: "data-kafka.default.svc.cluster.local:9094" - initContainers: - - name: check-postgres-connection - image: public.ecr.aws/docker/library/postgres:15-alpine - env: - - name: PGHOST - value: "data-postgres.default.svc.cluster.local" - - name: PGUSER - value: "postgres" - - name: PGPASSWORD - value: "password" - command: - [ - "sh", - "-c", - "until pg_isready -h $PGHOST -U $PGUSER; do sleep 1; done", - ] - - name: check-redis-connection - image: public.ecr.aws/docker/library/redis:7 - env: - - name: REDISCLI_URL - value: "redis://data-redis.default.svc.cluster.local:6379/0" - command: - [ - "sh", - "-c", - "until redis-cli -u $REDISCLI_URL ping; do sleep 1; done", - ] - - name: check-influxdb-connection - image: public.ecr.aws/docker/library/influxdb:1.8 - env: - - name: INFLUXDB_URL - value: "data-influx.default.svc.cluster.local" - - name: INFLUXDB_USERNAME - value: "test" - - name: INFLUXDB_PASSWORD - value: "testtest" - command: - [ - "sh", - "-c", - "until influx -host $INFLUXDB_URL -username $INFLUXDB_USERNAME -password $INFLUXDB_PASSWORD -execute 'SHOW DATABASES'; do sleep 1; done", - ] - - name: check-kafka-connection - image: confluentinc/cp-kafkacat:6.2.1 - env: - - name: KAFKA_BROKER_URL - value: "data-kafka.default.svc.cluster.local:9094" - command: - [ - "sh", - "-c", - "until kafkacat -b $KAFKA_BROKER_URL -L; do sleep 1; done", - ] diff --git a/deployments/k8s-yaml/service-supplyservice.yaml b/deployments/k8s-yaml/service-supplyservice.yaml deleted file mode 100755 index 6c3532843..000000000 --- a/deployments/k8s-yaml/service-supplyservice.yaml +++ /dev/null @@ -1,62 +0,0 @@ -apiVersion: "v1" -kind: Pod -metadata: - name: "diadata-clusterdev-service-supplyservice" -spec: - containers: - - name: clusterdev-service-supplyservice - image: dia-service-supplyservice:0.1 - command: ["supplyService"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_USER - value: "diadata_psql_user" - - name: POSTGRES_PASSWORD - value: "diadata_psql_pass" - - name: POSTGRES_DB - value: "diadata_psql_db" - - name: POSTGRES_HOST - value: "clusterdev-db-postgres-service.default.svc.cluster.local" - - name: INFLUXURL - value: "http://clusterdev-db-influx-service.default.svc.cluster.local:8086" - - name: INFLUXUSER - value: "diadata_user" - - name: INFLUXPASSWORD - value: "diadata_pw" - - name: REDISURL - value: "clusterdev-db-redis-service.default.svc.cluster.local:6379" - initContainers: - - name: check-postgres-connection - image: public.ecr.aws/docker/library/postgres:latest - env: - - name: PGHOST - value: "clusterdev-db-postgres-service.default.svc.cluster.local" - - name: PGUSER - value: "diadata_psql_user" - - name: PGPASSWORD - value: "diadata_psql_pass" - command: - ["sh", "-c", "until pg_isready -h $PGHOST -U $PGUSER; do sleep 1; done"] - - name: check-redis-connection - image: public.ecr.aws/docker/library/redis:latest - env: - - name: REDISCLI_URL - value: "redis://clusterdev-db-redis-service.default.svc.cluster.local:6379/0" - command: - ["sh", "-c", "until redis-cli -u $REDISCLI_URL ping; do sleep 1; done"] - - name: check-influxdb-connection - image: public.ecr.aws/docker/library/influxdb:1.8 - env: - - name: INFLUXDB_URL - value: "clusterdev-db-influx-service.default.svc.cluster.local" - - name: INFLUXDB_USERNAME - value: "diadata_user" - - name: INFLUXDB_PASSWORD - value: "diadata_pw" - command: - [ - "sh", - "-c", - "until influx -host $INFLUXDB_URL -username $INFLUXDB_USERNAME -password $INFLUXDB_PASSWORD -execute 'SHOW DATABASES'; do sleep 1; done", - ] diff --git a/deployments/k8s-yaml/service-tradesblockservice.yaml b/deployments/k8s-yaml/service-tradesblockservice.yaml deleted file mode 100755 index 5cd88bc69..000000000 --- a/deployments/k8s-yaml/service-tradesblockservice.yaml +++ /dev/null @@ -1,97 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: service-tradesblockservice - labels: - app: service-tradesblockservice -spec: - selector: - matchLabels: - app: service-tradesblockservice - template: - metadata: - labels: - app: service-tradesblockservice - spec: - containers: - - name: service-tradesblockservice - image: dia.tradesblockservice.dev:latest - imagePullPolicy: Never - command: ["tradesBlockService"] - args: ["-historical=true", "-testing=true"] - env: - - name: USE_ENV - value: "true" - - name: POSTGRES_HOST - value: "data-postgres.default.svc.cluster.local" - - name: POSTGRES_USER - value: "postgres" - - name: POSTGRES_PASSWORD - value: "password" - - name: POSTGRES_DB - value: "postgres" - - name: INFLUXURL - value: "http://data-influx.default.svc.cluster.local:8086" - - name: INFLUXUSER - value: "test" - - name: INFLUXPASSWORD - value: "testtest" - - name: REDISURL - value: "data-redis.default.svc.cluster.local:6379" - - name: KAFKAURL - value: "data-kafka.default.svc.cluster.local:9094" - - name: TRADE_VOLUME_THRESHOLD_EXPONENT - value: "4" - initContainers: - - name: check-postgres-connection - image: public.ecr.aws/docker/library/postgres:15-alpine - env: - - name: PGHOST - value: "data-postgres.default.svc.cluster.local" - - name: PGUSER - value: "postgres" - - name: PGPASSWORD - value: "password" - command: - [ - "sh", - "-c", - "until pg_isready -h $PGHOST -U $PGUSER; do sleep 1; done", - ] - - name: check-redis-connection - image: public.ecr.aws/docker/library/redis:7 - env: - - name: REDISCLI_URL - value: "redis://data-redis.default.svc.cluster.local:6379/0" - command: - [ - "sh", - "-c", - "until redis-cli -u $REDISCLI_URL ping; do sleep 1; done", - ] - - name: check-influxdb-connection - image: public.ecr.aws/docker/library/influxdb:1.8 - env: - - name: INFLUXDB_URL - value: "data-influx.default.svc.cluster.local" - - name: INFLUXDB_USERNAME - value: "test" - - name: INFLUXDB_PASSWORD - value: "testtest" - command: - [ - "sh", - "-c", - "until influx -host $INFLUXDB_URL -username $INFLUXDB_USERNAME -password $INFLUXDB_PASSWORD -execute 'SHOW DATABASES'; do sleep 1; done", - ] - - name: check-kafka-connection - image: confluentinc/cp-kafkacat:6.2.1 - env: - - name: KAFKA_BROKER_URL - value: "data-kafka.default.svc.cluster.local:9094" - command: - [ - "sh", - "-c", - "until kafkacat -b $KAFKA_BROKER_URL -L; do sleep 1; done", - ] diff --git a/deployments/local/docker-compose.yml b/deployments/local/docker-compose.yml deleted file mode 100644 index 30e5a11d1..000000000 --- a/deployments/local/docker-compose.yml +++ /dev/null @@ -1,91 +0,0 @@ -version: "3.3" -services: - redis: - image: redis - ports: - - "6379:6379" - influxdb: - image: influxdb:1.8.3 - ports: - - "8086:8086" - environment: - INFLUXDB_HTTP_AUTH_ENABLED: "false" - volumes: - - influxdb-data:/var/lib/influxdb - networks: - - influxdb-network -# influxdb_cli: -# links: -# - influxdb -# image: influxdb:1.8.3 -# entrypoint: influx setup --bucket test_bucket -t test_token -o test_org --username test --password testtest --host http://127.0.0.1:8087 -f -# # restart: on-failure:20 -# depends_on: -# - influxdb - zookeeper: - image: wurstmeister/zookeeper - ports: - - "2181:2181" - volumes: - # TODO figure out how to have this without version number - - /home/srv/zoo:/opt/zookeeper-3.4.9/data - networks: - - kafka-network - deploy: - mode: global - restart_policy: - delay: 2s - window: 20s - kafka0: - image: wurstmeister/kafka:latest - ports: - - "9094:9094" - networks: - - kafka-network - environment: - HOSTNAME_COMMAND: "docker info | grep ^Name: | cut -d' ' -f 2" - ALLOW_PLAINTEXT_LISTENER: 'yes' - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT - # KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://_{HOSTNAME_COMMAND}:9094 - KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://kafka0:9094 - KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094 - KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE - KAFKA_LOG_DIRS: /kafka/kafka-logs-1 - # KAFKA_LOG_RETENTION_BYTES: -1 - # KAFKA_LOG_RETENTION_HOURS: -1 - KAFKA_DELETE_TOPIC_ENABLE: "true" - KAFKA_MESSAGE_MAX_BYTES: 500000000 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - /home/srv/kafka:/kafka - deploy: - mode: global - restart_policy: - delay: 2s - window: 20s - depends_on: - - zookeeper - postgres: - image: postgres - restart: always - ports: - - "27017:27017" - - "5432:5432" - environment: - POSTGRES_PASSWORD: password - volumes: - - ../../config/pginit.sql:/docker-entrypoint-initdb.d/init.sql -networks: - kafka-network: - driver: overlay - attachable: true - influxdb-network: - driver: overlay - attachable: true -volumes: - kafka_data: - driver: local - influxdb-data: - driver: local - diff --git a/deployments/local/local.env b/deployments/local/local.env deleted file mode 100644 index cb37244bb..000000000 --- a/deployments/local/local.env +++ /dev/null @@ -1,11 +0,0 @@ -export USE_ENV=true -export INFLUXURL=http://localhost:8086 -export INFLUXUSER=test -export INFLUXPASSWORD=testtest -export POSTGRES_USER=postgres -export POSTGRES_PASSWORD=password -export POSTGRES_HOST=localhost -export POSTGRES_DB=postgres -export REDISURL=localhost:6379 -export KAFKAURL=localhost:9094 -export KAFKA_TOPIC_SUFFIX=dia \ No newline at end of file diff --git a/frontend/derivatives/VIX.js b/frontend/derivatives/VIX.js deleted file mode 100644 index 391c7335f..000000000 --- a/frontend/derivatives/VIX.js +++ /dev/null @@ -1,133 +0,0 @@ -var chartCVI; - -IndexInfo = { - name: 'Crypto Volatility Index', - symbol: 'CVI', - timeInterval: 30000, - timeUpdate: 10000 -} - -let now = Math.round(Date.now()/1000); -let beforeNow = (now - IndexInfo.timeInterval).toString(); -now = now.toString() - -getApi = { - prefill: 'https://api.diadata.org/v1/cviIndex?starttime=1&endtime=' + beforeNow, - update: 'https://api.diadata.org/v1/cviIndex?starttime=' + beforeNow + '&endtime=' + now, -} - -// ------------------------------------------------------------------------------------------------ -// Prefill with data -// ------------------------------------------------------------------------------------------------ - -function getHistoric(url, callback) { - var request = new XMLHttpRequest() - request.open('GET', url, true) - - request.onload = function() { - var data = JSON.parse(this.response) - if(this.status == 200) { - if (typeof callback === "function") { - callback(data) - } - } else if(this.status == 404) { - console.log('Not found error') - } - } - request.onerror = function() { - console.log('Request error.') - } - request.send() -} - - -// Fill chart with data upon loading screen -getHistoric(getApi.prefill, function(obj) { - console.log("prefill"); - prefillArray = [] - for(i = 0; i < obj.length; i++) { - prefillArray.push([Date.parse(obj[i].Timestamp), obj[i].Value]); - } - chartCVI.series[0].setData(prefillArray); -}); - - -// ------------------------------------------------------------------------------------------------ -// As soon as chart is loaded update asynchonously -// ------------------------------------------------------------------------------------------------ - -function requestData() { - let now = Math.round(Date.now()/1000); - let beforeNow = (now - IndexInfo.timeInterval).toString(); - $.ajax({ - cache: true, - url: 'https://api.diadata.org/v1/cviIndex?starttime=' + beforeNow + '&endtime=' + now, - type: 'GET', - - success: function(points) { - if(points.length > 1) { - var point = points[points.length-1]; - } else { - var point = points; - } - var date = Date.parse(point.Timestamp); - console.log(point) - - L = chartCVI.series[0].xData.length; - if(L == 0) { - chartCVI.series[0].addPoint([date, point.Value]); - console.log("Initial fill: " + point.Timestamp) - } else if(L > 0 && date != chartCVI.series[0].xData[L-1]){ - chartCVI.series[0].addPoint([date, point.Value]); - console.log("Updated point at: " + point.Timestamp) - } - - setTimeout(requestData, IndexInfo.timeUpdate); - }, - }); -} - -// ------------------------------------------------------------------------------------------------ -// Draw chart -// ------------------------------------------------------------------------------------------------ - -document.addEventListener('DOMContentLoaded', function() { - - chartCVI = Highcharts.stockChart('container', { - chart: { - type: 'spline', - events: { - load: requestData, - } - }, - credits: { - text: 'DIAdata', - href: 'https://diadata.org' - }, - title: { - text: IndexInfo.name, - }, - xAxis: { - tickPixelInterval: 150, - maxZoom: 20 * 1000, - title: { - margin: 10, - } - }, - yAxis: { - minPadding: 0.2, - maxPadding: 0.2, - title: { - text: 'Index value', - margin: 80 - } - }, - series: [ - { - name: IndexInfo.symbol, - data: [] - }, - ] - }); -}); - diff --git a/frontend/derivatives/index.html b/frontend/derivatives/index.html deleted file mode 100644 index d42ed6c79..000000000 --- a/frontend/derivatives/index.html +++ /dev/null @@ -1,24 +0,0 @@ - - - - - - - - - - - - - - - - - Volatility Index - - - -
- - - diff --git a/frontend/interestRates/SOFR.js b/frontend/interestRates/SOFR.js deleted file mode 100644 index 641625f05..000000000 --- a/frontend/interestRates/SOFR.js +++ /dev/null @@ -1,165 +0,0 @@ -// ------------------------------------------------------------------------------------------------ -// General variables and functions -// ------------------------------------------------------------------------------------------------ - -let today = new Date().toISOString().slice(0, 10); -let yesterday = new Date(Date.now() - 864e5).toISOString().slice(0,10); - -// getHistoric fetches historic data from our API with address @url -function getHistoric(url, callback) { - - // Instantiate request object - var request = new XMLHttpRequest() - request.open('GET', url, true) - - // Load data in GET request - request.onload = function() { - var data = JSON.parse(this.response) - if(this.status == 200) { - if (typeof callback === "function") { - callback(data) - } - } else if(this.status == 404) { - console.log('Not found error') - } - } - request.onerror = function() { - console.log('Request error.') - } - request.send() -} - -// requestData(rate) asynchrononously loads new data points into the chart. -function requestData(rate) { - $.ajax({ - cache: true, - url: rate.urlActual, - type: 'GET', - - // If GET request is successful, add data to the chart - success: function(point) { - - // convert time (string) to Unix time for plotting - var date = Date.parse(point.EffectiveDate); - - // Append a data point to the chart's series if the timestamp is new - L = window['chart' + rate.symbol].series[0].xData.length; - - if(L == 0) { - window['chart' + rate.symbol].series[0].addPoint([date, point.Value]); - } else if(L > 0 && date != window['chart' + rate.symbol].series[0].xData[L-1]){ - window['chart' + rate.symbol].series[0].addPoint([date, point.Value]); - } - console.log("Updated " + rate.symbol); - window['chart' + rate.symbol].redraw(); - setTimeout(function(){requestData.call(this, rate)}, rate.timeUpdate); - }, - }); -} - -function makechart(rate) { - window['chart' + rate.symbol] = Highcharts.stockChart(rate.container, { - rangeSelector: { - buttonTheme: { - width: 20, - }, - inputBoxWidth: 75, - }, - chart: { - type: 'spline', - events: { - load: function(){ - requestData.call(this, rate) - } - } - }, - credits: { - text: 'DIADATA', - href: 'https://diadata.org' - }, - title: { - text: rate.name, - style: { - fontSize: '20px', - }, - }, - xAxis: { - tickPixelInterval: 150, - maxZoom: 20 * 1000, - title: { - margin: 10, - } - }, - yAxis: { - minPadding: 0.2, - maxPadding: 0.2, - title: { - text: 'Index value', - margin: 80 - } - }, - series: [ - { - name: rate.symbol, - data: [] - }, - ] - }); -} - - - -// ------------------------------------------------------------------------------------------------ -// Rate specific information -// ------------------------------------------------------------------------------------------------ - -InfoSOFR = { - name: 'Secured Overnight Financing Rate', - symbol: 'SOFR', - container: 'containerSOFR', - timeUpdate: 10000, - urlHistoric: 'https://api.diadata.org/v1/interestrate/SOFR?dateInit=2018-11-01&dateFinal=' + yesterday, - urlActual: 'https://api.diadata.org/v1/interestrate/SOFR/' + today, -} - - -getHistoric(InfoSOFR.urlHistoric, function(obj) { - // Each entry of obj corresponds to rate information at a specific timestamp. - prefillArray = [] - for(i = 0; i < obj.length; i++) { - prefillArray.push([Date.parse(obj[i].EffectiveDate), obj[i].Value]); - } - // Sort array by date ... - prefillArray.sort() - // ... and fill the chart. - window['chart' + InfoSOFR.symbol].series[0].setData(prefillArray) - // Redraw for correct display of boxes - window['chart' + InfoSOFR.symbol].redraw(); -}); - -document.addEventListener('DOMContentLoaded', makechart(InfoSOFR)); - - - - -InfoESTR = { - name: 'Euro Short-Term Rate', - symbol: '€STR', - container: 'containerESTR', - timeUpdate: 10000, - urlHistoric: 'https://api.diadata.org/v1/interestrate/ESTER?dateInit=2018-10-01&dateFinal=' + yesterday, - urlActual: 'https://api.diadata.org/v1/interestrate/ESTER/' + today, -} - - -getHistoric(InfoESTR.urlHistoric, function(obj) { - prefillArray = [] - for(i = 0; i < obj.length; i++) { - prefillArray.push([Date.parse(obj[i].EffectiveDate), obj[i].Value]); - } - prefillArray.sort() - window['chart' + InfoESTR.symbol].series[0].setData(prefillArray) - window['chart' + InfoESTR.symbol].redraw(); -}); - -document.addEventListener('DOMContentLoaded', makechart(InfoESTR)); \ No newline at end of file diff --git a/frontend/interestRates/SOFR_old.js b/frontend/interestRates/SOFR_old.js deleted file mode 100644 index a181ea319..000000000 --- a/frontend/interestRates/SOFR_old.js +++ /dev/null @@ -1,134 +0,0 @@ -// ------------------------------------------------------------------------------------------------ -// General variables and functions -// ------------------------------------------------------------------------------------------------ - -let today = new Date().toISOString().slice(0, 10); -let yesterday = new Date(Date.now() - 864e5).toISOString().slice(0,10); - -function getHistoric(url, callback) { - // getHistoric fetches historic data from the API - - // Instantiate request object - var request = new XMLHttpRequest() - request.open('GET', url, true) - - // Load data in GET request - request.onload = function() { - var data = JSON.parse(this.response) - if(this.status == 200) { - if (typeof callback === "function") { - callback(data) - } - } else if(this.status == 404) { - console.log('Not found error') - } - } - request.onerror = function() { - console.log('Request error.') - } - request.send() -} - -function requestData(rate) { - console.log("my object is: ", rate) - $.ajax({ - cache: true, - url: rate.urlActual, - type: 'GET', - - // If GET request is successful, add data to the chart - success: function(point) { - - // convert time (string) to Unix time for plotting - var date = Date.parse(point.EffectiveDate); - console.log("point is: ", point); - - // Append a data point to the chart's series if the timestamp is new - // L = chartSOFR.series[0].xData.length; - L = chartSOFR.series[0].xData.length; - - if(L == 0) { - chartSOFR.series[0].addPoint([date, point.Value]); - console.log("Initial fill: " + point.EffectiveDate); - } else if(L > 0 && date != window['chart' + rate.symbol].series[0].xData[L-1]){ - chartSOFR.series[0].addPoint([date, point.Value]); - console.log("Updated point at: " + point.EffectiveDate); - } - - // Check for new data after @timeUpdate milliseconds - setTimeout(function(){requestData.call(this, rate)}, rate.timeUpdate); - }, - }); -} - - -// ------------------------------------------------------------------------------------------------ -// Rate specific information -// ------------------------------------------------------------------------------------------------ - -InfoSOFR = { - name: 'Secured Overnight Financing Rate', - symbol: 'SOFR', - timeUpdate: 10000, - urlHistoric: 'https://api.diadata.org/v1/interestrate/SOFR?dateInit=2018-11-01&dateFinal=' + yesterday, - urlActual: 'https://api.diadata.org/v1/interestrate/SOFR/' + today, -} - - -getHistoric(InfoSOFR.urlHistoric, function(obj) { - // Each entry of obj corresponds to rate information at a specific timestamp. - prefillArray = [] - for(i = 0; i < obj.length; i++) { - prefillArray.push([Date.parse(obj[i].EffectiveDate), obj[i].Value]); - } - // Sort array by date ... - prefillArray.sort() - // ... and fill the chart. - chartSOFR.series[0].setData(prefillArray) -}); - - - -document.addEventListener('DOMContentLoaded', function() { - - chartSOFR = Highcharts.stockChart('container', { - chart: { - type: 'spline', - events: { - load: function(){ - requestData.call(this, InfoSOFR) - } - } - }, - credits: { - text: 'DIADATA', - href: 'https://diadata.org' - }, - title: { - text: InfoSOFR.name, - }, - xAxis: { - tickPixelInterval: 150, - maxZoom: 20 * 1000, - title: { - text: 'Time', - margin: 10, - } - }, - yAxis: { - minPadding: 0.2, - maxPadding: 0.2, - title: { - text: 'Index value', - margin: 80 - } - }, - series: [ - { - name: InfoSOFR.symbol, - data: [] - }, - ] - }); -}); - diff --git a/frontend/interestRates/createCharts/ownChart.html b/frontend/interestRates/createCharts/ownChart.html deleted file mode 100644 index 7fe0e536b..000000000 --- a/frontend/interestRates/createCharts/ownChart.html +++ /dev/null @@ -1,32 +0,0 @@ - - - - - - - - - - - - Create Your Own Index - - - - - - - - - - -
- -
- - -
- - - - diff --git a/frontend/interestRates/createCharts/ownChart.js b/frontend/interestRates/createCharts/ownChart.js deleted file mode 100644 index cc63cd645..000000000 --- a/frontend/interestRates/createCharts/ownChart.js +++ /dev/null @@ -1,166 +0,0 @@ -// ------------------------------------------------------------------------------------------------ -// General variables and functions -// ------------------------------------------------------------------------------------------------ - -// // Load meta information on rates -var dateUrl = 'https://api.diadata.org/v1/interestrates'; -$.holdReady(true); -var firstPublications = null; -$.getJSON(dateUrl, function(data) { - firstPublications = data; - $.holdReady(false); -}); - -let today = new Date().toISOString().slice(0, 10); -var yourOwnChart; -var firstPublications = {}; - -function addDays(date, days) { - var result = new Date(date).getTime(); - result += days * 864e5 - result = new Date(result); - return result.toISOString() -} - -// getHistoric fetches historic data from our API with address @url -function getData(url, callback) { - - // Instantiate request object - var request = new XMLHttpRequest() - request.open('GET', url, true) - - // Load data in GET request - request.onload = function() { - var data = JSON.parse(this.response) - if(this.status == 200) { - if (typeof callback === "function") { - callback(data) - } - } else if(this.status == 404) { - console.log('Not found error') - } - } - request.onerror = function() { - console.log('Request error.') - } - request.send() -} - -function makechart(rate) { - yourOwnChart = Highcharts.stockChart(rate.container, { - rangeSelector: { - buttonTheme: { - width: 20, - }, - inputBoxWidth: 75, - }, - chart: { - type: 'spline', - - }, - credits: { - text: 'DIADATA', - href: 'https://diadata.org' - }, - title: { - text: rate.name, - style: { - fontSize: '20px', - }, - }, - xAxis: { - tickPixelInterval: 150, - maxZoom: 20 * 1000, - title: { - margin: 10, - } - }, - yAxis: { - minPadding: 0.2, - maxPadding: 0.2, - title: { - text: 'Index value', - margin: 80 - } - }, - series: [ - { - name: rate.name, - data: [] - }, - // { - // name: 'ESTER', - // data: [[1, 1], [2,10], [3, 10], [4, 20], [5, 1]] - // } - ] - }); -} - -// ------------------------------------------------------------------------------------------------ -// First fill of chart when loading the page -// ------------------------------------------------------------------------------------------------ - -// Rate info for the first fill -var RateInfo = { - name: 'SOFR30', - container: 'yourOwnContainer', - firstPublication: "2018-04-03", - url: 'https://api.diadata.org/v1/compoundedAvg/SOFR/30/360?dateInit=2018-05-14&dateFinal=' + today, -}; - -// Initial fill -getData(RateInfo.url, function(obj) { - prefillArray = [] - for(i = 0; i < obj.length; i++) { - prefillArray.push([Date.parse(obj[i].EffectiveDate), obj[i].Value]); - } - prefillArray.sort() - yourOwnChart.series[0].setData(prefillArray) - // yourOwnChartSOFR.redraw(); -}); -makechart(RateInfo); - -// ------------------------------------------------------------------------------------------------ -// Update upon clicking button -// ------------------------------------------------------------------------------------------------ -function updateChart() { - - // Retrieve user data -------------------------------------------------------------------- - var lenPeriod = document.getElementById("lenPeriod").value; - var dpy = document.getElementById("dpy").value; - var symbol = document.getElementById("symbol").value; - - - // update rate information --------------------------------------------------------------- - RateInfo.name = symbol + lenPeriod; - // retrieve first publication date - const found = Object.values(firstPublications).find(element => element.Symbol == symbol); - RateInfo.firstPublication = found.FirstDate.slice(0,10); - // Increase initial date according to observation period - dateInit = addDays(RateInfo.firstPublication, lenPeriod).slice(0,10); - RateInfo.url = 'https://api.diadata.org/v1/compoundedAvg/' + symbol + '/' + lenPeriod + '/' + dpy + '?dateInit=' + dateInit + '&dateFinal=' + today; - - // Fill the chart with updated information - getData(RateInfo.url, function(obj) { - var prefillArray = []; - for(i = 0; i < obj.length; i++) { - prefillArray.push([Date.parse(obj[i].EffectiveDate), obj[i].Value]); - } - prefillArray.sort(function(a,b) { return a - b; }); - yourOwnChart.series[0].setData(prefillArray); - }); - - // // Fill the chart with reference rate - // getData('http://localhost:8081/v1/interestrate/' + symbol + '/' + '?dateInit=' + dateInit + '&dateFinal=' + today, function(obj) { - // var prefillArray = []; - // for(i = 0; i < obj.length; i++) { - // prefillArray.push([Date.parse(obj[i].EffectiveDate), obj[i].Value]); - // } - // prefillArray.sort(); - // yourOwnChart.series[1].setData(prefillArray); - - // }); - - makechart(RateInfo); - -}; diff --git a/frontend/interestRates/createCharts/ownChart_v2.html b/frontend/interestRates/createCharts/ownChart_v2.html deleted file mode 100644 index 7c7ac30c5..000000000 --- a/frontend/interestRates/createCharts/ownChart_v2.html +++ /dev/null @@ -1,100 +0,0 @@ - - - - - - - - - - - - - - - Create Your Own Index - - - - - -
- -
- - - - - - - - - - - - - - - - - - - - -
- Interest rate type: - - -
- Business Day Convention: - - -
- Interest period in days: -
- Rounding to decimals: -
-
-
- - - - - - - -
 
 
-
- -
- -
-
- -
- - - - - - diff --git a/frontend/interestRates/createCharts/ownChart_v2.js b/frontend/interestRates/createCharts/ownChart_v2.js deleted file mode 100644 index 2f3fdd31c..000000000 --- a/frontend/interestRates/createCharts/ownChart_v2.js +++ /dev/null @@ -1,166 +0,0 @@ -// ------------------------------------------------------------------------------------------------ -// General variables and functions -// ------------------------------------------------------------------------------------------------ - -// // Load meta information on rates before page loads -var dateUrl = 'https://api.diadata.org/v1/interestrates'; -$.holdReady(true); -var firstPublications = null; -$.getJSON(dateUrl, function(data) { - firstPublications = data; - $.holdReady(false); -}); - -let today = new Date().toISOString().slice(0, 10); -var yourOwnChart; -var firstPublications = {}; - -function addDays(date, days) { - var result = new Date(date).getTime(); - result += days * 864e5 - result = new Date(result); - return result.toISOString() -} - -// getHistoric fetches historic data from our API with address @url -function getData(url, callback) { - - // Instantiate request object - var request = new XMLHttpRequest() - request.open('GET', url, true) - - // Load data in GET request - request.onload = function() { - var data = JSON.parse(this.response) - if(this.status == 200) { - if (typeof callback === "function") { - callback(data) - } - } else if(this.status == 404) { - console.log('Not found error') - } - } - request.onerror = function() { - console.log('Request error.') - } - request.send() -} - -function makechart(rate, loading) { - - yourOwnChart = Highcharts.stockChart(rate.container, { - rangeSelector: { - buttonTheme: { - width: 20, - }, - inputBoxWidth: 75, - }, - chart: { - type: 'spline', - }, - credits: { - text: 'DIADATA', - href: 'https://diadata.org' - }, - title: { - text: rate.name, - style: { - fontSize: '20px', - }, - }, - xAxis: { - tickPixelInterval: 150, - maxZoom: 20 * 1000, - title: { - margin: 10, - } - }, - yAxis: { - minPadding: 0.2, - maxPadding: 0.2, - title: { - text: 'Index value', - margin: 80 - } - }, - series: [ - { - name: rate.name, - data: [] - }, - ] - }); - if(loading) { - yourOwnChart.showLoading(); - } -} - -// ------------------------------------------------------------------------------------------------ -// First fill of chart when loading the page -// ------------------------------------------------------------------------------------------------ - -// Rate info for the first fill -var RateInfo = { - name: 'SOFR30', - container: 'yourOwnContainer', - firstPublication: "2018-04-03", - url: 'https://api.diadata.org/v1/compoundedAvg/SOFR/30/360?dateInit=2018-05-14&dateFinal=' + today, -}; - -// Initial fill -getData(RateInfo.url, function(obj) { - prefillArray = [] - for(i = 0; i < obj.length; i++) { - var value = obj[i].Value; - // prefillArray.push([Date.parse(obj[i].EffectiveDate), +value.toFixed(document.getElementById('rounding').value)]); - prefillArray.push([Date.parse(obj[i].EffectiveDate), +value.toFixed(4)]); - } - prefillArray.sort() - yourOwnChart.series[0].setData(prefillArray) - // yourOwnChartSOFR.redraw(); -}); -makechart(RateInfo, false); - -// ------------------------------------------------------------------------------------------------ -// Update upon clicking button -// ------------------------------------------------------------------------------------------------ -function updateChart() { - - // Retrieve user data -------------------------------------------------------------------- - var lenPeriod = document.getElementById('lenPeriod').value; - var dpy = document.getElementById('dpy').value; - var symbol = document.getElementById('symbol').value; - var rounding = document.getElementById('rounding').value; - var dia = document.getElementById('DIA').checked; - - // update rate information --------------------------------------------------------------- - // retrieve first publication date - const found = Object.values(firstPublications).find(element => element.Symbol == symbol); - RateInfo.firstPublication = found.FirstDate.slice(0,10); - // Increase initial date according to observation period - dateInit = addDays(RateInfo.firstPublication, lenPeriod).slice(0,10); - // Check which Index should be displayed - if(dia) { - RateInfo.name = symbol + lenPeriod + '_by_DIA'; - RateInfo.url = 'http://localhost:8081/v1/compoundedAvgDIA/' + symbol + '/' + lenPeriod + '/' + dpy + '?dateInit=' + dateInit + '&dateFinal=' + today; - } else { - RateInfo.name = symbol + lenPeriod; - RateInfo.url = 'https://api.diadata.org/v1/compoundedAvg/' + symbol + '/' + lenPeriod + '/' + dpy + '?dateInit=' + dateInit + '&dateFinal=' + today; - } - - // update rate information --------------------------------------------------------------- - getData(RateInfo.url, function(obj) { - - var prefillArray = []; - for(i = 0; i < obj.length; i++) { - var value = obj[i].Value; - prefillArray.push([Date.parse(obj[i].EffectiveDate), +value.toFixed(rounding)]); - // prefillArray.push([Date.parse(obj[i].EffectiveDate), parseFloat(value.toFixed(rounding))]); - } - yourOwnChart.series[0].setData(prefillArray); - yourOwnChart.hideLoading(); - }); - - makechart(RateInfo, true); - -}; \ No newline at end of file diff --git a/frontend/interestRates/createCharts/ownChart_v3.html b/frontend/interestRates/createCharts/ownChart_v3.html deleted file mode 100644 index 8f6c59282..000000000 --- a/frontend/interestRates/createCharts/ownChart_v3.html +++ /dev/null @@ -1,100 +0,0 @@ - - - - - - - - - - - - - - - Create Your Own Index - - - - - -
- -
- - - - - - - - - - - - - - - - - - - - -
- Interest rate type: - - -
- Business Day Convention: - - -
- Interest period in days: -
- Rounding to decimals: -
-
-
- - - - - - - -
 
-
- -
- -
-
- -
- - - - - - diff --git a/frontend/interestRates/createCharts/ownChart_v3.js b/frontend/interestRates/createCharts/ownChart_v3.js deleted file mode 100644 index 90ef1e8df..000000000 --- a/frontend/interestRates/createCharts/ownChart_v3.js +++ /dev/null @@ -1,262 +0,0 @@ -// ------------------------------------------------------------------------------------------------ -// General variables and functions -// ------------------------------------------------------------------------------------------------ - -// // Load meta information on rates before page loads -var dateUrl = 'https://api.diadata.org/v1/interestrates'; -$.holdReady(true); -var firstPublications = null; -$.getJSON(dateUrl, function(data) { - firstPublications = data; - $.holdReady(false); -}); - -let today = new Date().toISOString().slice(0, 10); -var yourOwnChart; -var firstPublications = {}; - -function addDays(date, days) { - var result = new Date(date).getTime(); - result += days * 864e5 - result = new Date(result); - return result.toISOString() -} - -// getHistoric fetches historic data from our API with address @url -function getData(url, callback) { - - // Instantiate request object - var request = new XMLHttpRequest() - request.open('GET', url, true) - - // Load data in GET request - request.onload = function() { - var data = JSON.parse(this.response) - if(this.status == 200) { - if (typeof callback === "function") { - callback(data) - } - } else if(this.status == 404) { - console.log('Not found error') - } - } - request.onerror = function() { - console.log('Request error.') - } - request.send() -} - -function makechart(rate, loading) { - - yourOwnChart = Highcharts.stockChart(rate.container, { - rangeSelector: { - buttonTheme: { - width: 20, - }, - inputBoxWidth: 75, - }, - chart: { - type: 'spline', - }, - credits: { - text: 'DIADATA', - href: 'https://diadata.org' - }, - title: { - text: rate.name, - style: { - fontSize: '20px', - }, - }, - xAxis: { - tickPixelInterval: 150, - maxZoom: 20 * 1000, - title: { - margin: 10, - } - }, - yAxis: { - minPadding: 0.2, - maxPadding: 0.2, - title: { - text: 'Index value', - margin: 80 - } - }, - series: [ - { - name: rate.name, - data: [] - }, - ] - }); - if(loading) { - yourOwnChart.showLoading(); - } -} - -function makechart2(rate1, rate2, loading) { - - yourOwnChart = Highcharts.stockChart(rate1.container, { - rangeSelector: { - buttonTheme: { - width: 20, - }, - inputBoxWidth: 75, - }, - chart: { - type: 'spline', - }, - credits: { - text: 'DIADATA', - href: 'https://diadata.org' - }, - title: { - text: rate1.name, - style: { - fontSize: '20px', - }, - }, - xAxis: { - tickPixelInterval: 150, - maxZoom: 20 * 1000, - title: { - margin: 10, - } - }, - yAxis: { - minPadding: 0.2, - maxPadding: 0.2, - title: { - text: 'Index value', - margin: 80 - } - }, - legend: { - enabled: true - }, - series: [ - { - name: rate2.name, - data: [], - color: '#707070', - lineWidth: 1 - // dashStyle: 'DashDot', - }, - { - name: rate1.name, - data: [], - }, - ] - }); - if(loading) { - yourOwnChart.showLoading(); - } -} - -// ------------------------------------------------------------------------------------------------ -// First fill of chart when loading the page -// ------------------------------------------------------------------------------------------------ - -// Rate info for the first fill -var RateInfo = { - name: 'SOFR30', - container: 'yourOwnContainer', - firstPublication: "2018-04-03", - url: 'https://api.diadata.org/v1/compoundedAvg/SOFR/30/360?dateInit=2018-05-14&dateFinal=' + today, -}; - -// Initial fill -getData(RateInfo.url, function(obj) { - prefillArray = [] - for(i = 0; i < obj.length; i++) { - var value = obj[i].Value; - // prefillArray.push([Date.parse(obj[i].EffectiveDate), +value.toFixed(document.getElementById('rounding').value)]); - prefillArray.push([Date.parse(obj[i].EffectiveDate), +value.toFixed(4)]); - } - prefillArray.sort() - yourOwnChart.series[0].setData(prefillArray) - // yourOwnChartSOFR.redraw(); -}); -makechart(RateInfo, false); - -// ------------------------------------------------------------------------------------------------ -// Update upon clicking button -// ------------------------------------------------------------------------------------------------ -function updateChart() { - - // Retrieve user data -------------------------------------------------------------------- - var lenPeriod = document.getElementById('lenPeriod').value; - var dpy = document.getElementById('dpy').value; - var symbol = document.getElementById('symbol').value; - var rounding = document.getElementById('rounding').value; - var dia = document.getElementById('DIA').checked; - var compare = document.getElementById('compare').checked; - - // update rate information --------------------------------------------------------------- - // retrieve first publication date - const found = Object.values(firstPublications).find(element => element.Symbol == symbol); - RateInfo.firstPublication = found.FirstDate.slice(0,10); - // Increase initial date according to observation period - dateInit = addDays(RateInfo.firstPublication, lenPeriod).slice(0,10); - // Check which Index should be displayed - if(dia) { - RateInfo.name = symbol + lenPeriod + '_by_DIA'; - RateInfo.url = 'https://api.diadata.org/v1/compoundedAvgDIA/' + symbol + '/' + lenPeriod + '/' + dpy + '?dateInit=' + dateInit + '&dateFinal=' + today; - } else { - RateInfo.name = symbol + lenPeriod; - RateInfo.url = 'https://api.diadata.org/v1/compoundedAvg/' + symbol + '/' + lenPeriod + '/' + dpy + '?dateInit=' + dateInit + '&dateFinal=' + today; - } - - if(compare){ - // Plot the original rate along with the custom compounded rate - - var RateInfoOriginal = { - name: symbol, - container: 'yourOwnContainer', - firstPublication: RateInfo.firstPublication, - url: 'https://api.diadata.org/v1/compoundedAvg/' + symbol + '/1/' + dpy + '?dateInit=' + dateInit + '&dateFinal=' + today, - }; - - getData(RateInfoOriginal.url, function(obj) { - var prefillArray = []; - for(i = 0; i < obj.length; i++) { - var value = obj[i].Value; - prefillArray.push([Date.parse(obj[i].EffectiveDate), +value.toFixed(rounding)]); - // prefillArray.push([Date.parse(obj[i].EffectiveDate), parseFloat(value.toFixed(rounding))]); - } - yourOwnChart.series[0].setData(prefillArray); - yourOwnChart.hideLoading(); - }); - - getData(RateInfo.url, function(obj) { - var prefillArray = []; - for(i = 0; i < obj.length; i++) { - var value = obj[i].Value; - prefillArray.push([Date.parse(obj[i].EffectiveDate), +value.toFixed(rounding)]); - // prefillArray.push([Date.parse(obj[i].EffectiveDate), parseFloat(value.toFixed(rounding))]); - } - yourOwnChart.series[1].setData(prefillArray); - yourOwnChart.hideLoading(); - }); - - makechart2(RateInfo, RateInfoOriginal, true); - - - } else { - - getData(RateInfo.url, function(obj) { - - var prefillArray = []; - for(i = 0; i < obj.length; i++) { - var value = obj[i].Value; - prefillArray.push([Date.parse(obj[i].EffectiveDate), +value.toFixed(rounding)]); - // prefillArray.push([Date.parse(obj[i].EffectiveDate), parseFloat(value.toFixed(rounding))]); - } - yourOwnChart.series[0].setData(prefillArray); - yourOwnChart.hideLoading(); - }); - makechart(RateInfo, true); - } - -}; \ No newline at end of file diff --git a/frontend/interestRates/index.html b/frontend/interestRates/index.html deleted file mode 100644 index 85cfd90b8..000000000 --- a/frontend/interestRates/index.html +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - - - - - - - - - - - - Interest rates - - - -
-
- - - diff --git a/oracle/diaoracle.sol b/oracle/diaoracle.sol deleted file mode 100644 index 4551585c3..000000000 --- a/oracle/diaoracle.sol +++ /dev/null @@ -1,47 +0,0 @@ -pragma solidity ^0.4.21; -// compiles with v0.4.25 - -contract DiaOracle { - address owner; - - struct CoinInfo { - uint256 price; - uint256 supply; - uint256 lastUpdateTimestamp; - string symbol; - } - - mapping(string => CoinInfo) diaOracles; - - event newCoinInfo( - string name, - string symbol, - uint256 price, - uint256 supply, - uint256 lastUpdateTimestamp - ); - - constructor() public { - owner = msg.sender; - } - - function changeOwner(address newOwner) public { - require(msg.sender == owner); - owner = newOwner; - } - - function updateCoinInfo(string name, string symbol, uint256 newPrice, uint256 newSupply, uint256 newTimestamp) public { - require(msg.sender == owner); - diaOracles[name] = (CoinInfo(newPrice, newSupply, newTimestamp, symbol)); - emit newCoinInfo(name, symbol, newPrice, newSupply, newTimestamp); - } - - function getCoinInfo(string name) public view returns (uint256, uint256, uint256, string) { - return ( - diaOracles[name].price, - diaOracles[name].supply, - diaOracles[name].lastUpdateTimestamp, - diaOracles[name].symbol - ); - } -} From 55ea86b7c5f57c7edc8463a78d317ea882e63026 Mon Sep 17 00:00:00 2001 From: Samuel Brack Date: Wed, 28 Jan 2026 18:06:39 +0100 Subject: [PATCH 2/2] restore pg scripts --- deployments/config/pginit.sql | 425 ++++++++++++++++++ deployments/config/pgtriggers.sql | 43 ++ .../config/postgres-docker-entrypoint.sh | 356 +++++++++++++++ 3 files changed, 824 insertions(+) create mode 100644 deployments/config/pginit.sql create mode 100644 deployments/config/pgtriggers.sql create mode 100755 deployments/config/postgres-docker-entrypoint.sh diff --git a/deployments/config/pginit.sql b/deployments/config/pginit.sql new file mode 100644 index 000000000..677471a59 --- /dev/null +++ b/deployments/config/pginit.sql @@ -0,0 +1,425 @@ +CREATE EXTENSION "pgcrypto"; + + +-- Table asset is the single source of truth for all assets handled at DIA. +-- If a field is not case sensitive (such as address for Ethereum) it should +-- be all lowercase for consistency reasons. +-- Otherwise it must be as defined in the underlying contract. +CREATE TABLE asset ( + asset_id UUID DEFAULT gen_random_uuid(), + symbol text NOT NULL, + name text NOT NULL, + decimals text, + blockchain text, + address text NOT NULL, + UNIQUE (asset_id), + UNIQUE (address, blockchain) +); + +-- Table exchangepair holds all trading pairs for the pair scrapers. +-- The format has to be the same as emitted by the exchange's API in order +-- for the pair scrapers to be able to scrape trading data from the API. +CREATE TABLE exchangepair ( + exchangepair_id UUID DEFAULT gen_random_uuid(), + symbol text NOT NULL, + foreignname text NOT NULL, + exchange text NOT NULL, + UNIQUE (foreignname, exchange), + -- These fields reference asset table and should be verified by pairdiscoveryservice. + -- Only trades with verified pairs are processed further and thereby enter price calculation. + verified boolean default false, + id_quotetoken UUID REFERENCES asset(asset_id), + id_basetoken UUID REFERENCES asset(asset_id) +); + +CREATE TABLE exchangesymbol ( + exchangesymbol_id UUID DEFAULT gen_random_uuid(), + symbol text NOT NULL, + exchange text NOT NULL, + UNIQUE (symbol,exchange), + verified boolean default false, + asset_id UUID REFERENCES asset(asset_id) +); + +CREATE TABLE exchange ( + exchange_id UUID DEFAULT gen_random_uuid(), + name text NOT NULL, + centralized boolean default false, + bridge boolean default false, + contract text, + blockchain text, + rest_api text, + ws_api text, + pairs_api text, + watchdog_delay numeric NOT NULL, + scraper_active boolean, + UNIQUE(exchange_id), + UNIQUE (name) +); + +CREATE TABLE pool ( + pool_id UUID DEFAULT gen_random_uuid(), + exchange text NOT NULL, + blockchain text NOT NULL, + address text NOT NULL, + UNIQUE (pool_id), + UNIQUE (blockchain,address) +); + +CREATE TABLE poolasset ( + poolasset_id UUID DEFAULT gen_random_uuid(), + pool_id UUID REFERENCES pool(pool_id) NOT NULL, + asset_id UUID REFERENCES asset(asset_id) NOT NULL, + liquidity numeric, + liquidity_usd numeric, + time_stamp timestamp, + token_index integer, + UNIQUE (poolasset_id), + UNIQUE(pool_id,asset_id) +); + +CREATE TABLE scraper_cronjob_state ( + scraper_cronjob_state_id UUID DEFAULT gen_random_uuid(), + scraper text NOT NULL, + index_type text NOT NULL, + index_value numeric, + UNIQUE(scraper_cronjob_state_id), + UNIQUE(scraper,index_type) +); + +CREATE TABLE chainconfig ( + chain_config_id UUID DEFAULT gen_random_uuid(), + rpcurl text NOT NULL, + wsurl text NOT NULL, + chainID text NOT NULL, + UNIQUE (chainID) +); + +-- blockchain table stores all blockchains available in our databases +CREATE TABLE blockchain ( + blockchain_id UUID DEFAULT gen_random_uuid(), + name text NOT NULL, + genesisdate numeric, + nativetoken_id UUID REFERENCES asset(asset_id), + verificationmechanism text, + chain_id text, + UNIQUE(blockchain_id), + UNIQUE(name) +); + +CREATE TABLE assetvolume ( + asset_id UUID primary key, + volume decimal, + time_stamp timestamp +); + +-- polling table stores data - required for HTTP polling +CREATE TABLE polling ( + polling_id UUID DEFAULT gen_random_uuid(), + blockchain text, + contract_address text NOT NULL, + page numeric DEFAULT 1, + UNIQUE(blockchain, contract_address) +); + +-- historicalquotation collects USD quotes with lower frequency +-- for a selection of assets. +CREATE TABLE historicalquotation ( + historicalquotation_id UUID DEFAULT gen_random_uuid(), + asset_id UUID REFERENCES asset(asset_id) NOT NULL, + price numeric, + quote_time timestamp, + source text, + UNIQUE(asset_id,quote_time,source), + UNIQUE(historicalquotation_id) +); + +CREATE TABLE IF NOT EXISTS scrapers ( + name character varying(255) NOT NULL, + conf json, + state json, + CONSTRAINT pk_scrapers PRIMARY KEY(name) +); + +CREATE TABLE blockdata ( + blockdata_id UUID DEFAULT gen_random_uuid(), + blockchain text NOT NULL, + block_number numeric NOT NULL, + block_data jsonb, + UNIQUE(blockchain, block_number), + UNIQUE(blockdata_id) +); + +CREATE TABLE assetpriceident ( + priceident_id UUID DEFAULT gen_random_uuid(), + asset_id UUID REFERENCES asset(asset_id), + group_id numeric NOT NULL, + rank_in_group numeric NOT NULL, + UNIQUE(asset_id), + UNIQUE(group_id, rank_in_group) +); + + +CREATE TABLE oracleconfig ( + id uuid DEFAULT gen_random_uuid(), + address text NOT NULL, + feeder_id text NOT NULL, + owner text NOT NULL, + symbols text NOT NULL, + chainid text NOT NULL, + active boolean DEFAULT true, + frequency text, + sleepseconds text, + deviationpermille text, + blockchainnode text DEFAULT ''::text, + feeder_address text, + mandatory_frequency text, + deleted boolean DEFAULT false, + createddate timestamp without time zone DEFAULT now() NOT NULL, + lastupdate timestamp without time zone, + creation_block bigint, + creation_block_time timestamp without time zone DEFAULT '1970-01-01 00:00:00'::timestamp without time zone, + feedselection text, + expired boolean DEFAULT false, + expired_time timestamp without time zone DEFAULT '1970-01-01 00:00:00'::timestamp without time zone +); + +ALTER TABLE oracleconfig ADD COLUMN name VARCHAR(255); +ALTER TABLE oracleconfig ADD COLUMN draft boolean DEFAULT true; +ALTER TABLE oracleconfig ADD COLUMN customer_id int ; +ALTER TABLE oracleconfig ADD COLUMN billable boolean DEFAULT false ; + + + + +ALTER TABLE oracleconfig ADD COLUMN ecosystem boolean DEFAULT false; + + + +ALTER TABLE oracleconfig +ADD CONSTRAINT unique_customer_chainid_address +UNIQUE (customer_id, chainid, address); + + + +ALTER TABLE oracleconfig +ADD CONSTRAINT unique_feeder_id UNIQUE (feeder_id); + + +-- CREATE TABLE oracleconfig ( +-- id UUID DEFAULT gen_random_uuid(), +-- address text NOT NULL, +-- feeder_id text NOT NULL, +-- owner text NOT NULL, +-- symbols text NOT NULL, +-- feeder_address text NOT NULL, +-- chainID text NOT NULL, +-- active boolean default true, +-- deleted boolean default false, +-- frequency text , +-- sleepseconds text, +-- deviationpermille text, +-- blockchainnode text, +-- mandatory_frequency text, +-- createddate TIMESTAMP NOT NULL DEFAULT NOW(), +-- lastupdate TIMESTAMP NOT NULL, +-- UNIQUE (id), +-- UNIQUE (feeder_id) +-- ); + +-- ALTER TABLE oracleconfig ADD COLUMN creation_block_time TIMESTAMP DEFAULT 'epoch'::timestamp; +-- ALTER TABLE oracleconfig ADD COLUMN feedSelection TEXT ; +-- ALTER TABLE oracleconfig ADD COLUMN expired boolean default false ; +-- ALTER TABLE oracleconfig ADD COLUMN expired_time TIMESTAMP DEFAULT 'epoch'::timestamp; + + + + +CREATE TABLE feederresource ( + id SERIAL PRIMARY KEY, + owner text NOT NULL, + total numeric NOT NULL, + UNIQUE (id), + UNIQUE (owner) +); + +CREATE TABLE asset_list ( + id SERIAL PRIMARY KEY, + asset_name VARCHAR(255) NOT NULL, + custom_name VARCHAR(255), + symbol VARCHAR(50), + methodology TEXT, + list_name TEXT + +); + +CREATE TABLE exchange_list ( + id SERIAL PRIMARY KEY, + name VARCHAR(255) NOT NULL, + asset_id INT REFERENCES asset_list(id) ON DELETE CASCADE +); + +CREATE TABLE exchange_pairs ( + id SERIAL PRIMARY KEY, + exchange_id INT REFERENCES exchange_list(id) ON DELETE CASCADE, + pair VARCHAR(255) NOT NULL +); + + + + + + + + CREATE TABLE customers ( + customer_id SERIAL PRIMARY KEY, + email VARCHAR(255) NOT NULL, + account_creation_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + customer_plan INTEGER REFERENCES plans(plan_id) ON DELETE SET NULL, + deployed_oracles INTEGER DEFAULT 0, + payment_status VARCHAR(50), + last_payment TIMESTAMP, + payment_source VARCHAR(255), + number_of_data_feeds INTEGER DEFAULT 0, + active BOOLEAN DEFAULT TRUE +); + + + +ALTER TABLE customers ADD COLUMN name VARCHAR(255); +ALTER TABLE customers ADD COLUMN payer_address text; + + + +CREATE TABLE wallet_public_keys ( + key_id SERIAL PRIMARY KEY, + customer_id INTEGER REFERENCES customers(customer_id) ON DELETE CASCADE, + public_key TEXT NOT NULL, + access_level VARCHAR(50) NOT NULL DEFAULT 'read_write', + UNIQUE (public_key), + CONSTRAINT check_access_level CHECK (access_level IN ('read', 'read_write')) + +); + + + +ALTER TABLE wallet_public_keys ADD COLUMN username VARCHAR(255) UNIQUE; + +CREATE TABLE wallet_public_keys_temp ( + key_id SERIAL PRIMARY KEY, + customer_id INTEGER REFERENCES customers(customer_id) ON DELETE CASCADE, + public_key TEXT NOT NULL, + access_level VARCHAR(50) NOT NULL DEFAULT 'read_write', + username VARCHAR(255) + ); + + ALTER TABLE wallet_public_keys_temp add invitor TEXT; + + ALTER TABLE wallet_public_keys_temp +ADD CONSTRAINT unique_customer_public_key UNIQUE (customer_id, public_key); + + +CREATE TABLE transfer_created ( + event VARCHAR(50), + transaction VARCHAR(50), + network_id INT, + network_name VARCHAR(50), + contract_address VARCHAR(42), + email VARCHAR(255), + company TEXT, + parent VARCHAR(50), + id UUID, + invoice_id VARCHAR(50), + bill_date TIMESTAMP, + to_address VARCHAR(42), + from_address VARCHAR(42), + token_symbol VARCHAR(10), + token_address VARCHAR(42), + payment_type VARCHAR(50), + usd BOOLEAN, + amount NUMERIC(10, 2), + item VARCHAR(255), + item_id INT, + source VARCHAR(50), + batch_id UUID, + transfer_id UUID, + ref_id VARCHAR(255), + agreement_id UUID +); + +CREATE TABLE loop_payment_transfer_processed ( + event VARCHAR(255) NOT NULL, + transaction VARCHAR(255) NOT NULL, + network_id INTEGER NOT NULL, + network_name VARCHAR(255) NOT NULL, + contract_address VARCHAR(255) NOT NULL, + email VARCHAR(255), + company VARCHAR(255), + parent VARCHAR(255), + transfer_id VARCHAR(255) NOT NULL, + success BOOLEAN NOT NULL, + payment_token_address VARCHAR(255), + payment_token_symbol VARCHAR(255), + end_user VARCHAR(255), + reason VARCHAR(255), + invoice_id VARCHAR(255), + amount_paid DOUBLE PRECISION, + agreement_id VARCHAR(255), + ref_id VARCHAR(255), + batch_id VARCHAR(255), + usd_amount VARCHAR(255) +); + +CREATE TABLE loop_payment_responses ( + id SERIAL PRIMARY KEY, + event TEXT, + transaction TEXT, + network_id INT, + network_name TEXT, + contract_address TEXT, + email TEXT, + company TEXT, + parent TEXT, + subscriber TEXT, + item TEXT, + item_id TEXT, + agreement_id TEXT, + agreement_amount TEXT, + frequency_number INT, + frequency_unit TEXT, + add_on_agreements TEXT, + add_on_items TEXT, + add_on_item_ids TEXT, + add_on_total_amount TEXT, + payment_token_symbol TEXT, + payment_token_address TEXT, + event_date INT, + ref_id TEXT, + invoice_id TEXT, + metadata JSONB DEFAULT '{}'::jsonb +); + + +CREATE TABLE plans ( + plan_id SERIAL PRIMARY KEY, + plan_name VARCHAR(50) NOT NULL UNIQUE, + plan_description TEXT, + plan_price NUMERIC(10, 2) NOT NULL, + plan_features TEXT +); + +ALTER TABLE plans ADD COLUMN total_feeds integer default 3; +ALTER TABLE plans ADD COLUMN total_oracles integer default 3; + + + + + +INSERT INTO "plans"("plan_id","plan_name","plan_description","plan_price","plan_features","total_feeds") +VALUES +(1,E'Plan 2',E'default',0,E'desc',10); + +INSERT INTO "plans"("plan_id","plan_name","plan_description","plan_price","plan_features","total_feeds") +VALUES +(2,E'Plan 1',E'default',0,E'desc',3); + diff --git a/deployments/config/pgtriggers.sql b/deployments/config/pgtriggers.sql new file mode 100644 index 000000000..c478a3cc6 --- /dev/null +++ b/deployments/config/pgtriggers.sql @@ -0,0 +1,43 @@ +CREATE OR REPLACE FUNCTION ensure_mandatory_read_write() +RETURNS TRIGGER AS $$ +BEGIN + IF TG_OP = 'DELETE' AND OLD.access_level = 'read_write' THEN + IF NOT EXISTS ( + SELECT 1 + FROM wallet_public_keys + WHERE customer_id = OLD.customer_id + AND access_level = 'read_write' + AND key_id != OLD.key_id + ) THEN + RAISE EXCEPTION 'Customer % must have at least one read_write access.', OLD.customer_id; + END IF; + END IF; + + IF TG_OP = 'UPDATE' AND OLD.access_level = 'read_write' AND NEW.access_level != 'read_write' THEN + IF NOT EXISTS ( + SELECT 1 + FROM wallet_public_keys + WHERE customer_id = OLD.customer_id + AND access_level = 'read_write' + AND key_id != OLD.key_id + ) THEN + RAISE EXCEPTION 'Customer % must have at least one read_write access.', OLD.customer_id; + END IF; + END IF; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + + +CREATE TRIGGER check_delete_read_write +BEFORE DELETE ON wallet_public_keys +FOR EACH ROW +EXECUTE FUNCTION ensure_mandatory_read_write(); + + + +CREATE TRIGGER check_update_read_write +BEFORE UPDATE ON wallet_public_keys +FOR EACH ROW +EXECUTE FUNCTION ensure_mandatory_read_write(); \ No newline at end of file diff --git a/deployments/config/postgres-docker-entrypoint.sh b/deployments/config/postgres-docker-entrypoint.sh new file mode 100755 index 000000000..beaf7421d --- /dev/null +++ b/deployments/config/postgres-docker-entrypoint.sh @@ -0,0 +1,356 @@ +#!/usr/bin/env bash +echo "Starting entrypoint script (${BASH_SOURCE[0]}) ..." +echo "Current user: $(whoami)" +echo "Current working directory: $(pwd)" +echo; echo "Injected scripts /docker-entrypoint-initdb.d/:" +ls -la /docker-entrypoint-initdb.d/ +echo; echo "Injected volumes (/mnt/):" +ls -la /mnt/ +set -Eeo pipefail +# TODO swap to -Eeuo pipefail above (after handling all potentially-unset variables) + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + printf >&2 'error: both %s and %s are set (but are exclusive)\n' "$var" "$fileVar" + exit 1 + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +# check to see if this file is being run or sourced from another script +_is_sourced() { + # https://unix.stackexchange.com/a/215279 + [ "${#FUNCNAME[@]}" -ge 2 ] \ + && [ "${FUNCNAME[0]}" = '_is_sourced' ] \ + && [ "${FUNCNAME[1]}" = 'source' ] +} + +# used to create initial postgres directories and if run as root, ensure ownership to the "postgres" user +docker_create_db_directories() { + local user; user="$(id -u)" + + mkdir -p "$PGDATA" + # ignore failure since there are cases where we can't chmod (and PostgreSQL might fail later anyhow - it's picky about permissions of this directory) + chmod 00700 "$PGDATA" || : + + # ignore failure since it will be fine when using the image provided directory; see also https://github.com/docker-library/postgres/pull/289 + mkdir -p /var/run/postgresql || : + chmod 03775 /var/run/postgresql || : + + # Create the transaction log directory before initdb is run so the directory is owned by the correct user + if [ -n "${POSTGRES_INITDB_WALDIR:-}" ]; then + mkdir -p "$POSTGRES_INITDB_WALDIR" + if [ "$user" = '0' ]; then + find "$POSTGRES_INITDB_WALDIR" \! -user postgres -exec chown postgres '{}' + + fi + chmod 700 "$POSTGRES_INITDB_WALDIR" + fi + + # allow the container to be started with `--user` + if [ "$user" = '0' ]; then + find "$PGDATA" \! -user postgres -exec chown postgres '{}' + + find /var/run/postgresql \! -user postgres -exec chown postgres '{}' + + fi +} + +# initialize empty PGDATA directory with new database via 'initdb' +# arguments to `initdb` can be passed via POSTGRES_INITDB_ARGS or as arguments to this function +# `initdb` automatically creates the "postgres", "template0", and "template1" dbnames +# this is also where the database user is created, specified by `POSTGRES_USER` env +docker_init_database_dir() { + # "initdb" is particular about the current user existing in "/etc/passwd", so we use "nss_wrapper" to fake that if necessary + # see https://github.com/docker-library/postgres/pull/253, https://github.com/docker-library/postgres/issues/359, https://cwrap.org/nss_wrapper.html + local uid; uid="$(id -u)" + if ! getent passwd "$uid" &> /dev/null; then + # see if we can find a suitable "libnss_wrapper.so" (https://salsa.debian.org/sssd-team/nss-wrapper/-/commit/b9925a653a54e24d09d9b498a2d913729f7abb15) + local wrapper + for wrapper in {/usr,}/lib{/*,}/libnss_wrapper.so; do + if [ -s "$wrapper" ]; then + NSS_WRAPPER_PASSWD="$(mktemp)" + NSS_WRAPPER_GROUP="$(mktemp)" + export LD_PRELOAD="$wrapper" NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP + local gid; gid="$(id -g)" + printf 'postgres:x:%s:%s:PostgreSQL:%s:/bin/false\n' "$uid" "$gid" "$PGDATA" > "$NSS_WRAPPER_PASSWD" + printf 'postgres:x:%s:\n' "$gid" > "$NSS_WRAPPER_GROUP" + break + fi + done + fi + + if [ -n "${POSTGRES_INITDB_WALDIR:-}" ]; then + set -- --waldir "$POSTGRES_INITDB_WALDIR" "$@" + fi + + # --pwfile refuses to handle a properly-empty file (hence the "\n"): https://github.com/docker-library/postgres/issues/1025 + eval 'initdb --username="$POSTGRES_USER" --pwfile=<(printf "%s\n" "$POSTGRES_PASSWORD") '"$POSTGRES_INITDB_ARGS"' "$@"' + + # unset/cleanup "nss_wrapper" bits + if [[ "${LD_PRELOAD:-}" == */libnss_wrapper.so ]]; then + rm -f "$NSS_WRAPPER_PASSWD" "$NSS_WRAPPER_GROUP" + unset LD_PRELOAD NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP + fi +} + +# print large warning if POSTGRES_PASSWORD is long +# error if both POSTGRES_PASSWORD is empty and POSTGRES_HOST_AUTH_METHOD is not 'trust' +# print large warning if POSTGRES_HOST_AUTH_METHOD is set to 'trust' +# assumes database is not set up, ie: [ -z "$DATABASE_ALREADY_EXISTS" ] +docker_verify_minimum_env() { + # check password first so we can output the warning before postgres + # messes it up + if [ "${#POSTGRES_PASSWORD}" -ge 100 ]; then + cat >&2 <<-'EOWARN' + + WARNING: The supplied POSTGRES_PASSWORD is 100+ characters. + + This will not work if used via PGPASSWORD with "psql". + + https://www.postgresql.org/message-id/flat/E1Rqxp2-0004Qt-PL%40wrigleys.postgresql.org (BUG #6412) + https://github.com/docker-library/postgres/issues/507 + + EOWARN + fi + if [ -z "$POSTGRES_PASSWORD" ] && [ 'trust' != "$POSTGRES_HOST_AUTH_METHOD" ]; then + # The - option suppresses leading tabs but *not* spaces. :) + cat >&2 <<-'EOE' + Error: Database is uninitialized and superuser password is not specified. + You must specify POSTGRES_PASSWORD to a non-empty value for the + superuser. For example, "-e POSTGRES_PASSWORD=password" on "docker run". + + You may also use "POSTGRES_HOST_AUTH_METHOD=trust" to allow all + connections without a password. This is *not* recommended. + + See PostgreSQL documentation about "trust": + https://www.postgresql.org/docs/current/auth-trust.html + EOE + exit 1 + fi + if [ 'trust' = "$POSTGRES_HOST_AUTH_METHOD" ]; then + cat >&2 <<-'EOWARN' + ******************************************************************************** + WARNING: POSTGRES_HOST_AUTH_METHOD has been set to "trust". This will allow + anyone with access to the Postgres port to access your database without + a password, even if POSTGRES_PASSWORD is set. See PostgreSQL + documentation about "trust": + https://www.postgresql.org/docs/current/auth-trust.html + In Docker's default configuration, this is effectively any other + container on the same system. + + It is not recommended to use POSTGRES_HOST_AUTH_METHOD=trust. Replace + it with "-e POSTGRES_PASSWORD=password" instead to set a password in + "docker run". + ******************************************************************************** + EOWARN + fi +} + +# usage: docker_process_init_files [file [file [...]]] +# ie: docker_process_init_files /always-initdb.d/* +# process initializer files, based on file extensions and permissions +docker_process_init_files() { + # psql here for backwards compatibility "${psql[@]}" + psql=( docker_process_sql ) + + printf '\n' + local f + for f; do + case "$f" in + *.sh) + # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 + # https://github.com/docker-library/postgres/pull/452 + if [ -x "$f" ]; then + printf '%s: running %s\n' "$0" "$f" + "$f" + else + printf '%s: sourcing %s\n' "$0" "$f" + . "$f" + fi + ;; + *.sql) printf '%s: running %s\n' "$0" "$f"; docker_process_sql -f "$f"; printf '\n' ;; + *.sql.gz) printf '%s: running %s\n' "$0" "$f"; gunzip -c "$f" | docker_process_sql; printf '\n' ;; + *.sql.xz) printf '%s: running %s\n' "$0" "$f"; xzcat "$f" | docker_process_sql; printf '\n' ;; + *.sql.zst) printf '%s: running %s\n' "$0" "$f"; zstd -dc "$f" | docker_process_sql; printf '\n' ;; + *) printf '%s: ignoring %s\n' "$0" "$f" ;; + esac + printf '\n' + done +} + +# Execute sql script, passed via stdin (or -f flag of pqsl) +# usage: docker_process_sql [psql-cli-args] +# ie: docker_process_sql --dbname=mydb <<<'INSERT ...' +# ie: docker_process_sql -f my-file.sql +# ie: docker_process_sql > "$PGDATA/pg_hba.conf" +} + +# start socket-only postgresql server for setting up or running scripts +# all arguments will be passed along as arguments to `postgres` (via pg_ctl) +docker_temp_server_start() { + if [ "$1" = 'postgres' ]; then + shift + fi + + # internal start of server in order to allow setup using psql client + # does not listen on external TCP/IP and waits until start finishes + set -- "$@" -c listen_addresses='' -p "${PGPORT:-5432}" + + PGUSER="${PGUSER:-$POSTGRES_USER}" \ + pg_ctl -D "$PGDATA" \ + -o "$(printf '%q ' "$@")" \ + -w start +} + +# stop postgresql server after done setting up user and running scripts +docker_temp_server_stop() { + PGUSER="${PGUSER:-postgres}" \ + pg_ctl -D "$PGDATA" -m fast -w stop +} + +# check arguments for an option that would cause postgres to stop +# return true if there is one +_pg_want_help() { + local arg + for arg; do + case "$arg" in + # postgres --help | grep 'then exit' + # leaving out -C on purpose since it always fails and is unhelpful: + # postgres: could not access the server configuration file "/var/lib/postgresql/data/postgresql.conf": No such file or directory + -'?'|--help|--describe-config|-V|--version) + return 0 + ;; + esac + done + return 1 +} + +_main() { + # if first arg looks like a flag, assume we want to run postgres server + if [ "${1:0:1}" = '-' ]; then + set -- postgres "$@" + fi + + if [ "$1" = 'postgres' ] && ! _pg_want_help "$@"; then + docker_setup_env + # setup data directories and permissions (when run as root) + docker_create_db_directories + if [ "$(id -u)" = '0' ]; then + # then restart script as postgres user + exec su-exec postgres "$BASH_SOURCE" "$@" + fi + + # only run initialization on an empty data directory + if [ -z "$DATABASE_ALREADY_EXISTS" ]; then + docker_verify_minimum_env + + # check dir permissions to reduce likelihood of half-initialized database + ls /docker-entrypoint-initdb.d/ > /dev/null + + docker_init_database_dir + pg_setup_hba_conf "$@" + + # PGPASSWORD is required for psql when authentication is required for 'local' connections via pg_hba.conf and is otherwise harmless + # e.g. when '--auth=md5' or '--auth-local=md5' is used in POSTGRES_INITDB_ARGS + export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}" + docker_temp_server_start "$@" + + docker_setup_db + docker_process_init_files /docker-entrypoint-initdb.d/* + + docker_temp_server_stop + unset PGPASSWORD + + cat <<-'EOM' + + PostgreSQL init process complete; ready for start up. + + EOM + else + cat <<-'EOM' + + PostgreSQL Database directory appears to contain a database; Skipping initialization + + EOM + fi + fi +} + +if ! _is_sourced; then + _main "$@" +fi \ No newline at end of file