Env per pod used to exist but has been removed - why? #629
Description
Checks
- I have checked for existing issues.
- This report is about the
User-Community Airflow Helm Chart.
Motivation
I'm trying to (finally) move from a patched older version of the community chart to the latest version (and working on other patches - issue and PR to hopefully follow), but I noticed that there's no longer a way to specify an environment variable for a single pod. In my case, I set the AIRFLOW__CORE__LOGGING (I'm still running 1.10.14) to INFO everywhere except the workers where I set it to DEBUG.
I also have my entity syncer (which has a little bit extra from the ones now in the 8.x.x chart), which has a couple of extra useful environment variables that I don't want to specify everywhere (obviously copying in the custom template is easy enough for the latter, but annoying for the workers). There are overrides for many of the other things, and there's presumably a good reason (that I'm missing?) why these used to exist in the 7.x.x versions of the chart and were removed. But is there a better way to solve my logging problem above, for example that I've missed?
Implementation
diff --git a/charts/airflow/templates/_helpers/pods.tpl b/charts/airflow/templates/_helpers/pods.tpl
index 0e5bdd8..47a1bae 100644
--- a/charts/airflow/templates/_helpers/pods.tpl
+++ b/charts/airflow/templates/_helpers/pods.tpl
@@ -583,4 +583,7 @@ EXAMPLE USAGE: {{ include "airflow.env" (dict "Release" .Release "Values" .Value
{{- if .Values.airflow.extraEnv }}
{{ toYaml .Values.airflow.extraEnv }}
{{- end }}
+{{- if .extraEnv }}
+{{ toYaml .extraEnv }}
+{{- end }}
{{- end }}
diff --git a/charts/airflow/templates/flower/flower-deployment.yaml b/charts/airflow/templates/flower/flower-deployment.yaml
index 436f7de..4a88dcb 100644
--- a/charts/airflow/templates/flower/flower-deployment.yaml
+++ b/charts/airflow/templates/flower/flower-deployment.yaml
@@ -3,6 +3,7 @@
{{- $podAffinity := include "airflow.podAffinity" (dict "Release" .Release "Values" .Values "affinity" .Values.flower.affinity) }}
{{- $podTolerations := include "airflow.podTolerations" (dict "Release" .Release "Values" .Values "tolerations" .Values.flower.tolerations) }}
{{- $podSecurityContext := include "airflow.podSecurityContext" (dict "Release" .Release "Values" .Values "securityContext" .Values.flower.securityContext) }}
+{{- $env := include "airflow.env" (dict "Release" .Release "Values" .Values "extraEnv" .Values.flower.extraEnv) }}
{{- $extraPipPackages := concat .Values.airflow.extraPipPackages .Values.flower.extraPipPackages }}
{{- $extraVolumeMounts := .Values.flower.extraVolumeMounts }}
{{- $volumeMounts := include "airflow.volumeMounts" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages "extraVolumeMounts" $extraVolumeMounts) }}
@@ -100,7 +101,7 @@ spec:
envFrom:
{{- include "airflow.envFrom" . | indent 12 }}
env:
- {{- include "airflow.env" . | indent 12 }}
+ {{- $env | indent 12 }}
ports:
- name: flower
containerPort: 5555
diff --git a/charts/airflow/templates/scheduler/scheduler-deployment.yaml b/charts/airflow/templates/scheduler/scheduler-deployment.yaml
index 529aa2c..7b5b85f 100644
--- a/charts/airflow/templates/scheduler/scheduler-deployment.yaml
+++ b/charts/airflow/templates/scheduler/scheduler-deployment.yaml
@@ -2,6 +2,7 @@
{{- $podAffinity := include "airflow.podAffinity" (dict "Release" .Release "Values" .Values "affinity" .Values.scheduler.affinity) }}
{{- $podTolerations := include "airflow.podTolerations" (dict "Release" .Release "Values" .Values "tolerations" .Values.scheduler.tolerations) }}
{{- $podSecurityContext := include "airflow.podSecurityContext" (dict "Release" .Release "Values" .Values "securityContext" .Values.scheduler.securityContext) }}
+{{- $env := include "airflow.env" (dict "Release" .Release "Values" .Values "extraEnv" .Values.scheduler.extraEnv) }}
{{- $extraPipPackages := concat .Values.airflow.extraPipPackages .Values.scheduler.extraPipPackages }}
{{- $extraVolumeMounts := .Values.scheduler.extraVolumeMounts }}
{{- $volumeMounts := include "airflow.volumeMounts" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages "extraVolumeMounts" $extraVolumeMounts) }}
@@ -110,7 +111,7 @@ spec:
envFrom:
{{- include "airflow.envFrom" . | indent 12 }}
env:
- {{- include "airflow.env" . | indent 12 }}
+ {{- $env | indent 12 }}
command:
{{- include "airflow.command" . | indent 12 }}
args:
@@ -227,4 +228,4 @@ spec:
configMap:
name: {{ include "airflow.fullname" . }}-pod-template
{{- end }}
- {{- end }}
\ No newline at end of file
+ {{- end }}
diff --git a/charts/airflow/templates/triggerer/triggerer-deployment.yaml b/charts/airflow/templates/triggerer/triggerer-deployment.yaml
index 2aea21d..739580b 100644
--- a/charts/airflow/templates/triggerer/triggerer-deployment.yaml
+++ b/charts/airflow/templates/triggerer/triggerer-deployment.yaml
@@ -3,6 +3,7 @@
{{- $podAffinity := include "airflow.podAffinity" (dict "Release" .Release "Values" .Values "affinity" .Values.triggerer.affinity) }}
{{- $podTolerations := include "airflow.podTolerations" (dict "Release" .Release "Values" .Values "tolerations" .Values.triggerer.tolerations) }}
{{- $podSecurityContext := include "airflow.podSecurityContext" (dict "Release" .Release "Values" .Values "securityContext" .Values.triggerer.securityContext) }}
+{{- $env := include "airflow.env" (dict "Release" .Release "Values" .Values "extraEnv" .Values.triggerer.extraEnv) }}
{{- $extraPipPackages := concat .Values.airflow.extraPipPackages .Values.triggerer.extraPipPackages }}
{{- $extraVolumeMounts := .Values.triggerer.extraVolumeMounts }}
{{- $volumeMounts := include "airflow.volumeMounts" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages "extraVolumeMounts" $extraVolumeMounts) }}
@@ -99,7 +100,7 @@ spec:
envFrom:
{{- include "airflow.envFrom" . | indent 12 }}
env:
- {{- include "airflow.env" . | indent 12 }}
+ {{- $env | indent 12 }}
command:
{{- include "airflow.command" . | indent 12 }}
args:
diff --git a/charts/airflow/templates/webserver/webserver-deployment.yaml b/charts/airflow/templates/webserver/webserver-deployment.yaml
index 4e536da..92139de 100644
--- a/charts/airflow/templates/webserver/webserver-deployment.yaml
+++ b/charts/airflow/templates/webserver/webserver-deployment.yaml
@@ -2,6 +2,7 @@
{{- $podAffinity := include "airflow.podAffinity" (dict "Release" .Release "Values" .Values "affinity" .Values.web.affinity) }}
{{- $podTolerations := include "airflow.podTolerations" (dict "Release" .Release "Values" .Values "tolerations" .Values.web.tolerations) }}
{{- $podSecurityContext := include "airflow.podSecurityContext" (dict "Release" .Release "Values" .Values "securityContext" .Values.web.securityContext) }}
+{{- $env := include "airflow.env" (dict "Release" .Release "Values" .Values "extraEnv" .Values.web.extraEnv) }}
{{- $extraPipPackages := concat .Values.airflow.extraPipPackages .Values.web.extraPipPackages }}
{{- $extraVolumeMounts := .Values.web.extraVolumeMounts }}
{{- $volumeMounts := include "airflow.volumeMounts" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages "extraVolumeMounts" $extraVolumeMounts) }}
@@ -103,7 +104,7 @@ spec:
envFrom:
{{- include "airflow.envFrom" . | indent 12 }}
env:
- {{- include "airflow.env" . | indent 12 }}
+ {{- $env | indent 12 }}
command:
{{- include "airflow.command" . | indent 12 }}
args:
@@ -155,4 +156,4 @@ spec:
{{- else }}
secretName: {{ include "airflow.fullname" . }}-webserver-config
{{- end }}
- defaultMode: 0644
\ No newline at end of file
+ defaultMode: 0644
diff --git a/charts/airflow/templates/worker/worker-statefulset.yaml b/charts/airflow/templates/worker/worker-statefulset.yaml
index d2e50bc..2266314 100644
--- a/charts/airflow/templates/worker/worker-statefulset.yaml
+++ b/charts/airflow/templates/worker/worker-statefulset.yaml
@@ -3,6 +3,7 @@
{{- $podAffinity := include "airflow.podAffinity" (dict "Release" .Release "Values" .Values "affinity" .Values.workers.affinity) }}
{{- $podTolerations := include "airflow.podTolerations" (dict "Release" .Release "Values" .Values "tolerations" .Values.workers.tolerations) }}
{{- $podSecurityContext := include "airflow.podSecurityContext" (dict "Release" .Release "Values" .Values "securityContext" .Values.workers.securityContext) }}
+{{- $env := include "airflow.env" (dict "Release" .Release "Values" .Values "extraEnv" .Values.workers.extraEnv) }}
{{- $extraPipPackages := concat .Values.airflow.extraPipPackages .Values.workers.extraPipPackages }}
{{- $extraVolumeMounts := .Values.workers.extraVolumeMounts }}
{{- $volumeMounts := include "airflow.volumeMounts" (dict "Release" .Release "Values" .Values "extraPipPackages" $extraPipPackages "extraVolumeMounts" $extraVolumeMounts) }}
@@ -104,7 +105,7 @@ spec:
envFrom:
{{- include "airflow.envFrom" . | indent 12 }}
env:
- {{- include "airflow.env" . | indent 12 }}
+ {{- $env | indent 12 }}
# have dumb-init only send signals to direct child process (needed for celery workers to warm shutdown)
- name: DUMB_INIT_SETSID
value: "0"
diff --git a/charts/airflow/values.yaml b/charts/airflow/values.yaml
index 9b7b987..96d60ab 100644
--- a/charts/airflow/values.yaml
+++ b/charts/airflow/values.yaml
@@ -629,6 +629,9 @@ scheduler:
##
schedulerAgeBeforeCheck: 180
+ ## extra environment for just the scheduler Pods
+ extraEnv: {}
+
## extra pip packages to install in the scheduler Pods
##
## ____ EXAMPLE _______________
@@ -786,6 +789,9 @@ web:
timeoutSeconds: 5
failureThreshold: 6
+ ## extra environment for just the web Pods
+ extraEnv: {}
+
## extra pip packages to install in the web Pods
##
## ____ EXAMPLE _______________
@@ -955,6 +961,9 @@ workers:
##
intervalSeconds: 900
+ ## extra environment for just the worker Pods
+ extraEnv: {}
+
## extra pip packages to install in the worker Pod
##
## ____ EXAMPLE _______________
@@ -1068,6 +1077,9 @@ triggerer:
timeoutSeconds: 60
failureThreshold: 5
+ ## extra environment for just the triggerer Pods
+ extraEnv: {}
+
## extra pip packages to install in the triggerer Pod
##
## ____ EXAMPLE _______________
@@ -1204,6 +1216,9 @@ flower:
timeoutSeconds: 5
failureThreshold: 6
+ ## extra environment for just the flower Pods
+ extraEnv: {}
+
## extra pip packages to install in the flower Pod
##
## ____ EXAMPLE _______________Are you willing & able to help?
- I am able to submit a PR!
- I can help test the feature!