Merge pull request #2 from nawinto99/dev

Dev

Author: n4en

src/config.cfg

@@ -1,3 +1,4 @@
 INGESTION="airbyte"
 WORKFLOW="airflow"
-TRANSFORMATION="dbt"
+TRANSFORM="dbt"
+DBT_PROJECT_NAME="proj_dbt"

src/ingestion/airbyte/setup.sh

@@ -1,20 +1,21 @@
 #!/usr/bin/env bash
 
+docker_compose_file="./ingestion/airbyte/docker-compose.yaml"
 
 up() {
     echo "Start: Airbyte"
-    docker-compose -f ./ingestion/airbyte/docker-compose.yaml up --detach --no-recreate
+    docker-compose -f $docker_compose_file up --detach --no-recreate
     echo "End: Airbyte"
 }
 
 down() {
     echo "Start: Airbyte"
-    docker-compose -f ./ingestion/airbyte/docker-compose.yaml down --volumes --rmi all
+    docker-compose -f $docker_compose_file down --volumes --rmi all
     echo "End: Airbyte"
 }
 
 case $1 in up)
-    up $0
+    up
     ;;
 down)
     down

src/ingestion/setup.sh

@@ -1,20 +1,24 @@
 #!/usr/bin/env bash
 
+#Setup files
+ingestion_setup_tool="./ingestion/$INGESTION/setup.sh"
+
+chmod +x $ingestion_setup_tool
 
 up() {
     echo "Setup the tool $INGESTION"
-    . ./ingestion/$INGESTION/setup.sh up
+    . $ingestion_setup_tool up
     echo "Setup the tool $INGESTION completed"
 }
 
 down() {
     echo "Down the tool $INGESTION"
-    . ./ingestion/$INGESTION/setup.sh down
+    . $ingestion_setup_tool down
     echo "Down the tool $INGESTION completed"
 }
 
 case $1 in up)
-    up $0
+    up
     ;;
 down)
     down

src/install_docker.sh

@@ -0,0 +1,24 @@
+sudo apt-get update
+
+sudo apt-get install \
+    ca-certificates \
+    curl \
+    gnupg \
+    lsb-release
+
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+
+echo \
+  "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
+  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+sudo apt-get update
+
+sudo apt-get install docker-ce docker-ce-cli containerd.io
+
+
+sudo curl -L "https://github.com/docker/compose/releases/download/v1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+
+sudo chmod +x /usr/local/bin/docker-compose
+
+sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose

src/setup.sh

@@ -2,25 +2,44 @@
 
 . ./config.cfg
 
+#Setup files
+ingestion_setup="./ingestion/setup.sh"
+workflow_setup="./workflow/setup.sh"
+transform_setup="./transform/setup.sh"
+
+
+chmod +x $ingestion_setup
+chmod +x $workflow_setup
+chmod +x $transform_setup
+
+
 up() {
     echo "Setup data ingestion tool"
-    . ./ingestion/setup.sh up
+    . $ingestion_setup up
     echo "Setup data ingestion tool completed"
 
     echo "Setup workflow tool"
-    . ./workflow/setup.sh up
+    . $workflow_setup up
     echo "Setup data workflow tool completed"
+
+    echo "Setup transform tool"
+    . $transform_setup up
+    echo "Setup data transform tool completed"
 }
 
 down() {
     echo "Down the data ingestion tool"
-    . ./ingestion/setup.sh down
+    . $ingestion_setup down
     echo "Down the data ingestion tool completed"
 
     echo "Down the workflow tool"
-    . ./workflow/setup.sh down
+    . $workflow_setup down
     echo "Down the workflow tool completed"
 
+    echo "Down the transform tool"
+    . $transform_setup down
+    echo "Down the transform tool completed"
+
 }
 
 case $1 in up)

src/transform/dbt/.env

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+up() {
+    echo "Start: dbt"
+    pip3 install dbt-postgres
+    mkdir ~/modelt_dbt
+    cd ~/modelt_dbt
+    mkdir profile
+    dbt init $DBT_PROJECT_NAME --profiles-dir ./profile
+    echo "dbt installed"
+    echo "$PWD"
+    cd $PWD
+    echo "End: dbt"
+}
+
+down() {
+    echo "Start: dbt"
+    pip3 uninstall dbt-postgres
+    pip3 uninstall dbt-core
+    rm -r ~/modelt_dbt
+    echo "End: dbt"
+}
+
+case $1 in up)
+    up
+    ;;
+down)
+    down
+    ;;
+*)
+    echo "Usage: $0 param"
+    echo "------ param -------"
+    echo "up - Builds, (re)creates, starts, and attaches to containers for a service."
+    echo "down - Stops containers and removes containers, networks, volumes, and images created by up"
+    ;;
+esac

src/transform/dbt/docker-compose.yaml

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+#Setup files
+transform_setup_tool="./transform/$TRANSFORM/setup.sh"
+
+chmod +x $transform_setup_tool
+
+up() {
+    echo "Setup the tool $TRANSFORM"
+    . $transform_setup_tool up
+    echo "Setup the tool $TRANSFORM completed"
+}
+
+down() {
+    echo "Down the tool $TRANSFORM"
+    . $transform_setup_tool down
+    echo "Down the tool $TRANSFORM completed"
+}
+
+case $1 in up)
+    up
+    ;;
+down)
+    down
+    ;;
+*)
+    echo "Usage: $0 param"
+    echo "------ param -------"
+    echo "up - Builds, (re)creates, starts, and attaches to containers for a service."
+    echo "down - Stops containers and removes containers, networks, volumes, and images created by up"
+    ;;
+esac

src/transform/dbt/setup.sh

@@ -1,17 +1,18 @@
 #!/usr/bin/env bash
 
+docker_compose_file="./workflow/airflow/docker-compose.yaml"
 
 up() {
     echo "Start: Airflow"
     mkdir -p ./workflow/airflow/dags ./workflow/airflow/logs ./workflow/airflow/plugins
-    docker-compose -f ./workflow/airflow/docker-compose.yaml up airflow-init
-    docker-compose -f ./workflow/airflow/docker-compose.yaml up --detach --no-recreate
+    docker-compose -f $docker_compose_file up airflow-init
+    docker-compose -f $docker_compose_file up --detach --no-recreate
     echo "End: Airflow"
 }
 
 down() {
     echo "Start: Airflow"
-    docker-compose -f ./workflow/airflow/docker-compose.yaml down --volumes --rmi all
+    docker-compose -f $docker_compose_file down --volumes --rmi all
     echo "End: Airflow"
 }