Dev feb

.version

@@ -0,0 +1 @@
+0.01

Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.10.2
+
+ENV PYTHONUNBUFFERED 1
+
+WORKDIR /adi
+
+ADD requirements.txt /adi/
+RUN pip install --upgrade pip
+RUN pip install -r requirements.txt
+
+# ① Install some dependencies
+RUN apt-get update \
+    && apt-get install -y libsasl2-dev python-dev libldap2-dev libssl-dev \
+    && rm -rf /var/lib/apt/lists/* \
+    && apt-get clean
+
+
+# Copy adi
+ADD /adi /adi
+
+
+VOLUME /usr/src

Jenkinsfile_Devstart

@@ -0,0 +1,36 @@
+pipeline {
+
+    environment {
+        DOCKERHUB_CREDENTIALS = credentials('avicoiot-dockerhub')
+        PRODUCT = 'ADI'
+        GIT_HOST = 'somewhere'
+        GIT_REPO = 'repo'
+    }
+    agent any
+
+    options {
+        buildDiscarder(logRotator(numToKeepStr: '3'))
+    }
+    stages {
+        stage('Build') {
+            steps {
+                sh 'docker build -t avicoiot/adi-alpine:latest .'
+                }
+            }
+        stage('Login') {
+            steps {
+                sh 'echo $DOCKERHUB_CREDENTIALS_PSW | docker login --username $DOCKERHUB_CREDENTIALS_USR --password-stdin'
+                }
+            }
+        stage('Push') {
+            steps {
+                sh 'docker push avicoiot/adi-alpine:latest'
+                }
+            }
+        }
+        post {
+            always {
+                  sh 'docker logout'
+            }
+        }
+}

Untitled Diagram.drawio

@@ -0,0 +1 @@
+<mxfile host="app.diagrams.net" modified="2023-01-24T15:43:02.102Z" agent="5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36" version="20.8.10" etag="sVzuCxN98oUfDyyQU1v1" type="github"><diagram name="Page-1" id="odTgSxn9Hmhe4syxjLUJ">UzV2zq1wL0osyPDNT0nNUTV2VTV2LsrPL4GwciucU3NyVI0MMlNUjV1UjYwMgFjVyA2HrCFY1qAgsSg1rwSLBiADYTaQg2Y1AA==</diagram></mxfile>

adi/app_config/config.yaml

@@ -0,0 +1,77 @@
+# backend/webserver/config/config.yml
+PROJECT_NAME: 'ADI'
+
+application_conig:
+  db_archive: "/db_archive"
+
+  rules:
+    folder: "/csv_files"
+    files: ['source1.csv' ]
+    customers_list: [1,2,3,4,5,6,7]
+
+
+
+databases:
+
+  mongo:
+    DB_TYPE: 'mongodb'
+    ENGINE: 'mongodb'
+    DRIVER: 'motor'
+    NAME: 'webserver'
+    USER: 'admin'
+    PASSWORD: 'admin'
+    HOST: 'mongo_db'
+    PORT: 27017
+    DROP_COLLECTION_ON_START: ["sdad"]
+    DB_PREPARATION:
+      security:
+        index:
+          username
+          email
+      customer:
+        index:
+          customer_no
+          email
+
+    WATCH: ["customer","test"]
+
+  internal:
+    DB_TYPE: 'postgres'
+    ENGINE: 'postgres'
+    NAME: 'internal'
+    USER: 'admin'
+    PASSWORD: 'admin'
+    HOST: '192.168.1.113'
+    PORT: 5432
+
+  postgres:
+    DB_TYPE: 'postgres'
+    ENGINE: 'postgres'
+    NAME: 'dvdrental'
+    USER: 'admin'
+    PASSWORD: 'admin'
+    HOST: '192.168.1.113'
+    PORT: 5432
+
+  target:
+    DB_TYPE: 'postgres'
+    ENGINE: 'postgres'
+    NAME: 'target'
+    USER: 'admin'
+    PASSWORD: 'admin'
+    HOST: '192.168.1.113'
+    PORT: 5432
+
+  redis:
+    host: redis_db
+    port: 6379
+    db: 0
+
+
+files:
+  default:
+    input_file_path: '/webserver/input/'
+    output_file_path: '/webserver/output/'
+
+security:
+   trace_request: 'Y'

app_config/db_config.py → adi/app_config/db_config.py

@@ -1,12 +1,14 @@
-from app_config.settings import SingletonMeta
+
 from typing import Dict
 from enum import Enum
+import sys
+from pathlib import Path
+sys.path.append(str(Path(__file__).parent.parent))
+
 
 from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy_utils import database_exists, create_database
-from sqlalchemy.sql import text
 
+from app_config.settings import SingletonMeta
 class DBType(str, Enum):
     POSTGRES = "postgres"
     SQLITE = "sqlite"
@@ -38,7 +40,7 @@ def __init__(self, *args, **kwargs):
 
       #  self.Session = sessionmaker(bind=self.engine)
     def get_engine(self):
-        return self.engine
+        return self.engine.connect()
 
 
     #
@@ -94,11 +96,19 @@ def get_db(config: Dict) -> "DbSettings":
         return factory(**config)
 
 
-# Test
-db_test = {'DB_TYPE': 'postgres', 'ENGINE': 'postgres', 'NAME': 'dvdrental', 'USER': 'admin', 'PASSWORD': 'admin', 'HOST': '192.168.1.113', 'PORT': 5432}
-test = DBContext.get_db(db_test)
-sss = test.get_engine()
-print(sss)
+# # # Test
+# db_test = {'DB_TYPE': 'postgres', 'ENGINE': 'postgres', 'NAME': 'dvdrental', 'USER': 'admin', 'PASSWORD': 'admin', 'HOST': '192.168.1.113', 'PORT': 5432}
+# test = DBContext.get_db(db_test)
+
+# sss = test.get_engine()
+# print(sss)
+# from sqlalchemy import text
+# sql = text('SELECT * from customer WHERE customer_id=1')
+# results = sss.execute(sql)
+# for e in results:
+#     print(e)
+
+
 # import pandas as pd
 # sql = '''
 #     SELECT * FROM actor;

app_config/db_config_with_ABC.py → adi/app_config/db_config_with_ABC.py

@@ -1,4 +1,3 @@
-from app_config.settings import SingletonMeta
 from abc import ABC, abstractmethod
 from typing import Dict
 from enum import Enum

adi/app_config/settings.py

@@ -0,0 +1,63 @@
+import yaml
+from functools import reduce
+import operator
+from pathlib import Path
+
+import sys
+from pathlib import Path
+sys.path.append(str(Path(__file__).parent.parent))
+
+
+
+config_file = Path('app_config', 'config.yaml')
+rules = 'application_conig.rules.'
+
+
+class SingletonMeta(type):
+
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs ):
+        if cls not in cls._instances:
+            instance = super().__call__(*args, **kwargs)
+            cls._instances[cls] = instance
+        return cls._instances[cls]
+
+
+class Settings(metaclass=SingletonMeta):
+
+    def __init__(self, *args, **kwargs):
+        self.config_file = kwargs['config_file']
+
+
+        with open(self.config_file, "r") as stream:
+            try:
+                self.settings = yaml.safe_load(stream)
+
+            except yaml.YAMLError as exc:
+                print(exc)
+
+    def get(self, element):
+        return reduce(operator.getitem, element.split('.'), self.settings)
+
+
+settings = Settings(config_file=config_file)
+
+
+
+## adding sys.path.append(str(Path(__file__).parent.parent)) - will include the parent dir so can work directly
+# or from main
+
+# s1 = Settings(config_file='config.yaml')
+# print(s1.get('databases.mongo.ENGINE'))
+
+# if __name__ == "__main__":
+#     # The client code.
+#     config_file = Path('.', 'config.yaml')
+#     s1 = Settings(config_file=config_file)
+
+    # print(s1.get('databases.mongo.ENGINE'))
+#     if id(s1) == id(s2):
+#         print("Singleton works, both variables contain the same instance.")
+#     else:
+#         print("Singleton failed, variables contain different instances.")