feat: add exclusion for all the gitsync containers if persistence is enabled

karakanb · karakanb · commit df8c4632ba37 · 2022-05-13T00:25:45.000+01:00
Signed-off-by: Burak Karakan &lt;burak.karakan@gmail.com&gt;
diff --git a/charts/airflow/files/pod_template.kubernetes-helm-yaml b/charts/airflow/files/pod_template.kubernetes-helm-yaml
@@ -48,7 +48,7 @@ spec:
     {{- if $extraPipPackages }}
     {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 4 }}
     {{- end }}
-    {{- if .Values.dags.gitSync.enabled }}
+    {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
     {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 4 }}
     {{- end }}
     {{- if .Values.airflow.kubernetesPodTemplate.extraInitContainers }}
diff --git a/charts/airflow/templates/_helpers/pods.tpl b/charts/airflow/templates/_helpers/pods.tpl
@@ -442,7 +442,7 @@ EXAMPLE USAGE: {{ include "airflow.volumes" (dict "Release" .Release "Values" .V
 {{- end }}
 
 {{- /* git-sync */ -}}
-{{- if .Values.dags.gitSync.enabled }}
+{{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
 {{- if .Values.dags.gitSync.sshSecret }}
 - name: git-secret
   secret:
diff --git a/charts/airflow/templates/_helpers/validate-values.tpl b/charts/airflow/templates/_helpers/validate-values.tpl
@@ -84,7 +84,7 @@
 {{- end }}
 
 {{/* Checks for `dags.gitSync` */}}
-{{- if .Values.dags.gitSync.enabled }}
+{{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
   {{- if not .Values.dags.gitSync.repo }}
   {{ required "If `dags.gitSync.enabled=true`, then `dags.gitSync.repo` must be non-empty!" nil }}
   {{- end }}
diff --git a/charts/airflow/templates/db-migrations/db-migrations-deployment.yaml b/charts/airflow/templates/db-migrations/db-migrations-deployment.yaml
@@ -82,7 +82,7 @@ spec:
         {{- if $extraPipPackages }}
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
@@ -107,7 +107,7 @@ spec:
             - name: scripts
               mountPath: /mnt/scripts
               readOnly: true
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" . | indent 8 }}
         {{- end }}
diff --git a/charts/airflow/templates/db-migrations/db-migrations-job.yaml b/charts/airflow/templates/db-migrations/db-migrations-job.yaml
@@ -76,7 +76,7 @@ spec:
         {{- if $extraPipPackages }}
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
diff --git a/charts/airflow/templates/flower/flower-deployment.yaml b/charts/airflow/templates/flower/flower-deployment.yaml
@@ -86,7 +86,7 @@ spec:
         {{- if $extraPipPackages }}
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
@@ -151,7 +151,7 @@ spec:
           volumeMounts:
             {{- $volumeMounts | indent 12 }}
           {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" . | indent 8 }}
         {{- end }}
diff --git a/charts/airflow/templates/scheduler/scheduler-deployment.yaml b/charts/airflow/templates/scheduler/scheduler-deployment.yaml
@@ -94,7 +94,7 @@ spec:
         {{- if $extraPipPackages }}
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
         {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
@@ -196,7 +196,7 @@ spec:
               readOnly: true
             {{- end }}
           {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         {{- include "airflow.container.git_sync" . | indent 8 }}
         {{- end }}
         {{- if .Values.scheduler.logCleanup.enabled }}
diff --git a/charts/airflow/templates/sync/sync-connections-deployment.yaml b/charts/airflow/templates/sync/sync-connections-deployment.yaml
@@ -82,7 +82,7 @@ spec:
         {{- if $extraPipPackages }}
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
@@ -113,7 +113,7 @@ spec:
               mountPath: "/mnt/templates"
               readOnly: true
             {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" . | indent 8 }}
         {{- end }}
diff --git a/charts/airflow/templates/sync/sync-connections-job.yaml b/charts/airflow/templates/sync/sync-connections-job.yaml
@@ -76,7 +76,7 @@ spec:
         {{- if $extraPipPackages }}
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
diff --git a/charts/airflow/templates/sync/sync-pools-deployment.yaml b/charts/airflow/templates/sync/sync-pools-deployment.yaml
@@ -82,7 +82,7 @@ spec:
         {{- if $extraPipPackages }}
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
@@ -108,7 +108,7 @@ spec:
             - name: scripts
               mountPath: /mnt/scripts
               readOnly: true
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" . | indent 8 }}
         {{- end }}
diff --git a/charts/airflow/templates/sync/sync-pools-job.yaml b/charts/airflow/templates/sync/sync-pools-job.yaml
@@ -76,7 +76,7 @@ spec:
         {{- if $extraPipPackages }}
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
diff --git a/charts/airflow/templates/sync/sync-users-deployment.yaml b/charts/airflow/templates/sync/sync-users-deployment.yaml
@@ -82,7 +82,7 @@ spec:
         {{- if $extraPipPackages }}
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
@@ -113,7 +113,7 @@ spec:
               mountPath: "/mnt/templates"
               readOnly: true
             {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" . | indent 8 }}
         {{- end }}
diff --git a/charts/airflow/templates/sync/sync-users-job.yaml b/charts/airflow/templates/sync/sync-users-job.yaml
@@ -76,7 +76,7 @@ spec:
         {{- if $extraPipPackages }}
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
diff --git a/charts/airflow/templates/sync/sync-variables-deployment.yaml b/charts/airflow/templates/sync/sync-variables-deployment.yaml
@@ -82,7 +82,7 @@ spec:
         {{- if $extraPipPackages }}
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
@@ -113,7 +113,7 @@ spec:
               mountPath: "/mnt/templates"
               readOnly: true
             {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" . | indent 8 }}
         {{- end }}
diff --git a/charts/airflow/templates/sync/sync-variables-job.yaml b/charts/airflow/templates/sync/sync-variables-job.yaml
@@ -76,7 +76,7 @@ spec:
         {{- if $extraPipPackages }}
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         ## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
         {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
diff --git a/charts/airflow/templates/triggerer/triggerer-deployment.yaml b/charts/airflow/templates/triggerer/triggerer-deployment.yaml
@@ -86,7 +86,7 @@ spec:
         {{- if $extraPipPackages }}
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
         {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
@@ -147,7 +147,7 @@ spec:
           volumeMounts:
             {{- $volumeMounts | indent 12 }}
           {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         {{- include "airflow.container.git_sync" . | indent 8 }}
         {{- end }}
         {{- if .Values.airflow.extraContainers }}
diff --git a/charts/airflow/templates/webserver/webserver-deployment.yaml b/charts/airflow/templates/webserver/webserver-deployment.yaml
@@ -86,7 +86,7 @@ spec:
         {{- if $extraPipPackages }}
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
         {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
@@ -140,7 +140,7 @@ spec:
               mountPath: /opt/airflow/webserver_config.py
               subPath: webserver_config.py
               readOnly: true
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         {{- include "airflow.container.git_sync" . | indent 8 }}
         {{- end }}
         {{- if .Values.airflow.extraContainers }}
diff --git a/charts/airflow/templates/worker/worker-statefulset.yaml b/charts/airflow/templates/worker/worker-statefulset.yaml
@@ -91,7 +91,7 @@ spec:
         {{- if $extraPipPackages }}
         {{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
         {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         {{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
         {{- end }}
         {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
@@ -168,7 +168,7 @@ spec:
           volumeMounts:
             {{- $volumeMounts | indent 12 }}
           {{- end }}
-        {{- if .Values.dags.gitSync.enabled }}
+        {{- if and (.Values.dags.gitSync.enabled)  (not .Values.dags.persistence.enabled)  }}
         {{- include "airflow.container.git_sync" . | indent 8 }}
         {{- end }}
         {{- if .Values.workers.logCleanup.enabled }}
