Skip to content

Commit 3512922

Browse files
karakanbkarakanb
committed
feat: copy dags from the mounted pvc to the local dag path
Signed-off-by: Burak Karakan <[email protected]>
1 parent 828e3e8 commit 3512922

17 files changed

+104
-18
lines changed

‎charts/airflow/files/pod_template.kubernetes-helm-yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,13 @@ spec:
4848
{{- if $extraPipPackages }}
4949
{{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 4 }}
5050
{{- end }}
51-
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
51+
{{- if .Values.dags.gitSync.enabled }}
52+
{{- if .Values.dags.persistence.enabled }}
53+
{{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 4 }}
54+
{{- else }}
5255
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 4 }}
5356
{{- end }}
57+
{{- end }}
5458
{{- if .Values.airflow.kubernetesPodTemplate.extraInitContainers }}
5559
{{- toYaml .Values.airflow.kubernetesPodTemplate.extraInitContainers | nindent 4 }}
5660
{{- end }}

‎charts/airflow/templates/_helpers/pods.tpl‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -254,8 +254,6 @@ EXAMPLE USAGE: {{ include "airflow.container.git_sync" (dict "Release" .Release
254254
name: {{ .Values.dags.gitSync.httpSecret }}
255255
key: {{ .Values.dags.gitSync.httpSecretPasswordKey }}
256256
{{- end }}
257-
{{- /* this has user-defined variables, so must be included BELOW (so the ABOVE `env` take precedence) */ -}}
258-
{{- include "airflow.env" . | indent 4 }}
259257
volumeMounts:
260258
- name: dags-data
261259
mountPath: /dags
@@ -273,6 +271,27 @@ EXAMPLE USAGE: {{ include "airflow.container.git_sync" (dict "Release" .Release
273271
{{- end }}
274272
{{- end }}
275273

274+
{{/*
275+
Define a container which copies the contents of the DAG PVC to the DAG directory in the pod
276+
EXAMPLE USAGE: {{ include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) }}
277+
*/}}
278+
{{- define "airflow.container.dag_pvc_copy" }}
279+
- name: dags-copy-from-pvc
280+
image: busybox:1.35
281+
imagePullPolicy: Never
282+
securityContext:
283+
runAsUser: {{ .Values.dags.gitSync.image.uid }}
284+
runAsGroup: {{ .Values.dags.gitSync.image.gid }}
285+
volumeMounts:
286+
- name: dags-data
287+
mountPath: /dags
288+
command:
289+
- "cp"
290+
- "-r"
291+
- "/dags"
292+
- {{ .Values.dags.path }}
293+
{{- end }}
294+
276295
{{/*
277296
Define a container which regularly deletes airflow logs older than a retention period.
278297
EXAMPLE USAGE: {{ include "airflow.container.log_cleanup" (dict "Release" .Release "Values" .Values "resources" $lc_resources "retention_min" $lc_retention_min "interval_sec" $lc_interval_sec) }}

‎charts/airflow/templates/db-migrations/db-migrations-deployment.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,10 +82,14 @@ spec:
8282
{{- if $extraPipPackages }}
8383
{{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
8484
{{- end }}
85-
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
85+
{{- if .Values.dags.gitSync.enabled }}
86+
{{- if .Values.dags.persistence.enabled }}
87+
{{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }}
88+
{{- else }}
8689
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
8790
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
8891
{{- end }}
92+
{{- end }}
8993
{{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
9094
containers:
9195
- name: db-migrations

‎charts/airflow/templates/db-migrations/db-migrations-job.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,10 +76,14 @@ spec:
7676
{{- if $extraPipPackages }}
7777
{{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
7878
{{- end }}
79-
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
79+
{{- if .Values.dags.gitSync.enabled }}
80+
{{- if .Values.dags.persistence.enabled }}
81+
{{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }}
82+
{{- else }}
8083
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
8184
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
8285
{{- end }}
86+
{{- end }}
8387
{{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
8488
containers:
8589
- name: db-migrations

‎charts/airflow/templates/flower/flower-deployment.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,10 +86,14 @@ spec:
8686
{{- if $extraPipPackages }}
8787
{{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
8888
{{- end }}
89-
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
89+
{{- if .Values.dags.gitSync.enabled }}
90+
{{- if .Values.dags.persistence.enabled }}
91+
{{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }}
92+
{{- else }}
9093
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
9194
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
9295
{{- end }}
96+
{{- end }}
9397
{{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
9498
{{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
9599
containers:

‎charts/airflow/templates/scheduler/scheduler-deployment.yaml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,9 +94,14 @@ spec:
9494
{{- if $extraPipPackages }}
9595
{{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
9696
{{- end }}
97-
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
97+
{{- if .Values.dags.gitSync.enabled }}
98+
{{- if .Values.dags.persistence.enabled }}
99+
{{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }}
100+
{{- else }}
101+
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
98102
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
99103
{{- end }}
104+
{{- end }}
100105
{{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
101106
{{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
102107
{{- if .Values.scheduler.extraInitContainers }}

‎charts/airflow/templates/sync/sync-connections-deployment.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,10 +82,14 @@ spec:
8282
{{- if $extraPipPackages }}
8383
{{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
8484
{{- end }}
85-
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
85+
{{- if .Values.dags.gitSync.enabled }}
86+
{{- if .Values.dags.persistence.enabled }}
87+
{{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }}
88+
{{- else }}
8689
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
8790
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
8891
{{- end }}
92+
{{- end }}
8993
{{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
9094
{{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
9195
containers:

‎charts/airflow/templates/sync/sync-connections-job.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,10 +76,14 @@ spec:
7676
{{- if $extraPipPackages }}
7777
{{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
7878
{{- end }}
79-
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
79+
{{- if .Values.dags.gitSync.enabled }}
80+
{{- if .Values.dags.persistence.enabled }}
81+
{{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }}
82+
{{- else }}
8083
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
8184
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
8285
{{- end }}
86+
{{- end }}
8387
{{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
8488
{{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
8589
containers:

‎charts/airflow/templates/sync/sync-pools-deployment.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,10 +82,14 @@ spec:
8282
{{- if $extraPipPackages }}
8383
{{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
8484
{{- end }}
85-
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
85+
{{- if .Values.dags.gitSync.enabled }}
86+
{{- if .Values.dags.persistence.enabled }}
87+
{{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }}
88+
{{- else }}
8689
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
8790
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
8891
{{- end }}
92+
{{- end }}
8993
{{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
9094
{{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
9195
containers:

‎charts/airflow/templates/sync/sync-pools-job.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,10 +76,14 @@ spec:
7676
{{- if $extraPipPackages }}
7777
{{- include "airflow.init_container.install_pip_packages" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages) | indent 8 }}
7878
{{- end }}
79-
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
79+
{{- if .Values.dags.gitSync.enabled }}
80+
{{- if .Values.dags.persistence.enabled }}
81+
{{- include "airflow.container.dag_pvc_copy" (dict "Release" .Release "Values" .Values) | indent 8 }}
82+
{{- else }}
8083
## git-sync is included so "airflow plugins" & "python packages" can be stored in the dags repo
8184
{{- include "airflow.container.git_sync" (dict "Release" .Release "Values" .Values "sync_one_time" "true") | indent 8 }}
8285
{{- end }}
86+
{{- end }}
8387
{{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
8488
{{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
8589
containers:

0 commit comments

Comments
 (0)