Skip to content

Commit ea5826a

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #2376 from mrueg/cronjob-add-timezone
feat: Add timezone to kube_cronjob_info / Make kube_cronjob_next_schedule_time timezone-aware
2 parents 17151ac + 569e820 commit ea5826a

File tree

3 files changed

+187
-49
lines changed

3 files changed

+187
-49
lines changed

‎docs/metrics/workload/cronjob-metrics.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
| Metric name | Metric type | Description | Labels/tags | Status |
44
| ---------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
55
| kube_cronjob_annotations | Gauge | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; <br> `annotation_CRONJOB_ANNOTATION`=&lt;CRONJOB_ANNOTATION&gt; | EXPERIMENTAL |
6-
| kube_cronjob_info | Gauge | | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; <br> `schedule`=&lt;schedule&gt; <br> `concurrency_policy`=&lt;concurrency-policy&gt; | STABLE |
6+
| kube_cronjob_info | Gauge | | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; <br> `schedule`=&lt;schedule&gt; <br> `concurrency_policy`=&lt;concurrency-policy&gt; <br> `timezone`=&lt;timezone&gt; | STABLE |
77
| kube_cronjob_labels | Gauge | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md) | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; <br> `label_CRONJOB_LABEL`=&lt;CRONJOB_LABEL&gt; | STABLE |
88
| kube_cronjob_created | Gauge | | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; | STABLE |
99
| kube_cronjob_next_schedule_time | Gauge | | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; | STABLE |

‎internal/store/cronjob.go‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -96,11 +96,15 @@ func cronJobMetricFamilies(allowAnnotationsList, allowLabelsList []string) []gen
9696
basemetrics.STABLE,
9797
"",
9898
wrapCronJobFunc(func(j *batchv1.CronJob) *metric.Family {
99+
timeZone := "local"
100+
if j.Spec.TimeZone != nil {
101+
timeZone = *j.Spec.TimeZone
102+
}
99103
return &metric.Family{
100104
Metrics: []*metric.Metric{
101105
{
102-
LabelKeys: []string{"schedule", "concurrency_policy"},
103-
LabelValues: []string{j.Spec.Schedule, string(j.Spec.ConcurrencyPolicy)},
106+
LabelKeys: []string{"schedule", "concurrency_policy", "timezone"},
107+
LabelValues: []string{j.Spec.Schedule, string(j.Spec.ConcurrencyPolicy), timeZone},
104108
Value: 1,
105109
},
106110
},
@@ -245,7 +249,7 @@ func cronJobMetricFamilies(allowAnnotationsList, allowLabelsList []string) []gen
245249
ms := []*metric.Metric{}
246250

247251
// If the cron job is suspended, don't track the next scheduled time
248-
nextScheduledTime, err := getNextScheduledTime(j.Spec.Schedule, j.Status.LastScheduleTime, j.CreationTimestamp)
252+
nextScheduledTime, err := getNextScheduledTime(j.Spec.Schedule, j.Status.LastScheduleTime, j.CreationTimestamp, j.Spec.TimeZone)
249253
if err != nil {
250254
panic(err)
251255
} else if !*j.Spec.Suspend {
@@ -347,7 +351,11 @@ func createCronJobListWatch(kubeClient clientset.Interface, ns string, fieldSele
347351
}
348352
}
349353

350-
func getNextScheduledTime(schedule string, lastScheduleTime *metav1.Time, createdTime metav1.Time) (time.Time, error) {
354+
func getNextScheduledTime(schedule string, lastScheduleTime *metav1.Time, createdTime metav1.Time, timeZone *string) (time.Time, error) {
355+
if timeZone != nil {
356+
schedule = fmt.Sprintf("CRON_TZ=%s %s", *timeZone, schedule)
357+
}
358+
351359
sched, err := cron.ParseStandard(schedule)
352360
if err != nil {
353361
return time.Time{}, fmt.Errorf("Failed to parse cron job schedule '%s': %w", schedule, err)

‎internal/store/cronjob_test.go‎

Lines changed: 174 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -40,68 +40,163 @@ var (
4040
ActiveRunningCronJob1LastScheduleTime = time.Unix(1520742896, 0)
4141
SuspendedCronJob1LastScheduleTime = time.Unix(1520742896+5.5*3600, 0) // 5.5 hours later
4242
ActiveCronJob1NoLastScheduledCreationTimestamp = time.Unix(1520742896+6.5*3600, 0)
43+
TimeZone = "Asia/Shanghai"
4344
)
4445

45-
func TestCronJobStore(t *testing.T) {
46-
hour := ActiveRunningCronJob1LastScheduleTime.Hour()
47-
ActiveRunningCronJob1NextScheduleTime := time.Time{}
46+
func calculateNextSchedule6h(timestamp time.Time, timezone string) time.Time {
47+
loc, _ := time.LoadLocation(timezone)
48+
hour := timestamp.In(loc).Hour()
4849
switch {
4950
case hour < 6:
50-
ActiveRunningCronJob1NextScheduleTime = time.Date(
51-
ActiveRunningCronJob1LastScheduleTime.Year(),
52-
ActiveRunningCronJob1LastScheduleTime.Month(),
53-
ActiveRunningCronJob1LastScheduleTime.Day(),
51+
return time.Date(
52+
timestamp.Year(),
53+
timestamp.Month(),
54+
timestamp.Day(),
5455
6,
5556
0,
56-
0, 0, time.Local)
57+
0, 0, loc)
5758
case hour < 12:
58-
ActiveRunningCronJob1NextScheduleTime = time.Date(
59-
ActiveRunningCronJob1LastScheduleTime.Year(),
60-
ActiveRunningCronJob1LastScheduleTime.Month(),
61-
ActiveRunningCronJob1LastScheduleTime.Day(),
59+
return time.Date(
60+
timestamp.Year(),
61+
timestamp.Month(),
62+
timestamp.Day(),
6263
12,
6364
0,
64-
0, 0, time.Local)
65+
0, 0, loc)
6566
case hour < 18:
66-
ActiveRunningCronJob1NextScheduleTime = time.Date(
67-
ActiveRunningCronJob1LastScheduleTime.Year(),
68-
ActiveRunningCronJob1LastScheduleTime.Month(),
69-
ActiveRunningCronJob1LastScheduleTime.Day(),
67+
return time.Date(
68+
timestamp.Year(),
69+
timestamp.Month(),
70+
timestamp.Day(),
7071
18,
7172
0,
72-
0, 0, time.Local)
73-
case hour < 24:
74-
ActiveRunningCronJob1NextScheduleTime = time.Date(
75-
ActiveRunningCronJob1LastScheduleTime.Year(),
76-
ActiveRunningCronJob1LastScheduleTime.Month(),
77-
ActiveRunningCronJob1LastScheduleTime.Day(),
78-
24,
73+
0, 0, loc)
74+
default:
75+
return time.Date(
76+
timestamp.Year(),
77+
timestamp.Month(),
78+
timestamp.Day()+1,
79+
0,
7980
0,
80-
0, 0, time.Local)
81+
0, 0, loc)
8182
}
83+
}
8284

83-
minute := ActiveCronJob1NoLastScheduledCreationTimestamp.Minute()
84-
ActiveCronJob1NoLastScheduledNextScheduleTime := time.Time{}
85+
func calculateNextSchedule25m(timestamp time.Time, timezone string) time.Time {
86+
loc, _ := time.LoadLocation(timezone)
87+
minute := timestamp.In(loc).Minute()
8588
switch {
8689
case minute < 25:
87-
ActiveCronJob1NoLastScheduledNextScheduleTime = time.Date(
88-
ActiveCronJob1NoLastScheduledCreationTimestamp.Year(),
89-
ActiveCronJob1NoLastScheduledCreationTimestamp.Month(),
90-
ActiveCronJob1NoLastScheduledCreationTimestamp.Day(),
91-
ActiveCronJob1NoLastScheduledCreationTimestamp.Hour(),
90+
return time.Date(
91+
timestamp.Year(),
92+
timestamp.Month(),
93+
timestamp.Day(),
94+
timestamp.Hour(),
9295
25,
93-
0, 0, time.Local)
96+
0, 0, loc)
9497
default:
95-
ActiveCronJob1NoLastScheduledNextScheduleTime = time.Date(
96-
ActiveCronJob1NoLastScheduledNextScheduleTime.Year(),
97-
ActiveCronJob1NoLastScheduledNextScheduleTime.Month(),
98-
ActiveCronJob1NoLastScheduledNextScheduleTime.Day(),
99-
ActiveCronJob1NoLastScheduledNextScheduleTime.Hour()+1,
98+
return time.Date(
99+
timestamp.Year(),
100+
timestamp.Month(),
101+
timestamp.Day(),
102+
timestamp.Hour()+1,
100103
25,
101-
0, 0, time.Local)
104+
0, 0, loc)
102105
}
103106

107+
}
108+
func TestCronJobStore(t *testing.T) {
109+
110+
ActiveRunningCronJob1NextScheduleTime := calculateNextSchedule6h(ActiveRunningCronJob1LastScheduleTime, "Local")
111+
ActiveRunningCronJobWithTZ1NextScheduleTime := calculateNextSchedule6h(ActiveRunningCronJob1LastScheduleTime, TimeZone)
112+
113+
ActiveCronJob1NoLastScheduledNextScheduleTime := calculateNextSchedule25m(ActiveCronJob1NoLastScheduledCreationTimestamp, "Local")
114+
104115
cases := []generateMetricsTestCase{
116+
{
117+
AllowAnnotationsList: []string{
118+
"app.k8s.io/owner",
119+
},
120+
Obj: &batchv1.CronJob{
121+
ObjectMeta: metav1.ObjectMeta{
122+
Name: "ActiveRunningCronJobWithTZ1",
123+
Namespace: "ns1",
124+
Generation: 1,
125+
ResourceVersion: "11111",
126+
Labels: map[string]string{
127+
"app": "example-active-running-with-tz-1",
128+
},
129+
Annotations: map[string]string{
130+
"app": "mysql-server",
131+
"app.k8s.io/owner": "@foo",
132+
},
133+
},
134+
Status: batchv1.CronJobStatus{
135+
Active: []v1.ObjectReference{{Name: "FakeJob1"}, {Name: "FakeJob2"}},
136+
LastScheduleTime: &metav1.Time{Time: ActiveRunningCronJob1LastScheduleTime},
137+
LastSuccessfulTime: nil,
138+
},
139+
Spec: batchv1.CronJobSpec{
140+
StartingDeadlineSeconds: &StartingDeadlineSeconds300,
141+
ConcurrencyPolicy: "Forbid",
142+
Suspend: &SuspendFalse,
143+
Schedule: "0 */6 * * *",
144+
SuccessfulJobsHistoryLimit: &SuccessfulJobHistoryLimit3,
145+
FailedJobsHistoryLimit: &FailedJobHistoryLimit1,
146+
TimeZone: &TimeZone,
147+
},
148+
},
149+
Want: `
150+
# HELP kube_cronjob_created [STABLE] Unix creation timestamp
151+
# HELP kube_cronjob_info [STABLE] Info about cronjob.
152+
# HELP kube_cronjob_annotations Kubernetes annotations converted to Prometheus labels.
153+
# HELP kube_cronjob_labels [STABLE] Kubernetes labels converted to Prometheus labels.
154+
# HELP kube_cronjob_next_schedule_time [STABLE] Next time the cronjob should be scheduled. The time after lastScheduleTime, or after the cron job's creation time if it's never been scheduled. Use this to determine if the job is delayed.
155+
# HELP kube_cronjob_spec_failed_job_history_limit Failed job history limit tells the controller how many failed jobs should be preserved.
156+
# HELP kube_cronjob_spec_starting_deadline_seconds [STABLE] Deadline in seconds for starting the job if it misses scheduled time for any reason.
157+
# HELP kube_cronjob_spec_successful_job_history_limit Successful job history limit tells the controller how many completed jobs should be preserved.
158+
# HELP kube_cronjob_spec_suspend [STABLE] Suspend flag tells the controller to suspend subsequent executions.
159+
# HELP kube_cronjob_status_active [STABLE] Active holds pointers to currently running jobs.
160+
# HELP kube_cronjob_metadata_resource_version [STABLE] Resource version representing a specific version of the cronjob.
161+
# HELP kube_cronjob_status_last_schedule_time [STABLE] LastScheduleTime keeps information of when was the last time the job was successfully scheduled.
162+
# TYPE kube_cronjob_created gauge
163+
# TYPE kube_cronjob_info gauge
164+
# TYPE kube_cronjob_annotations gauge
165+
# TYPE kube_cronjob_labels gauge
166+
# TYPE kube_cronjob_next_schedule_time gauge
167+
# TYPE kube_cronjob_spec_failed_job_history_limit gauge
168+
# TYPE kube_cronjob_spec_starting_deadline_seconds gauge
169+
# TYPE kube_cronjob_spec_successful_job_history_limit gauge
170+
# TYPE kube_cronjob_spec_suspend gauge
171+
# TYPE kube_cronjob_status_active gauge
172+
# TYPE kube_cronjob_metadata_resource_version gauge
173+
# TYPE kube_cronjob_status_last_schedule_time gauge
174+
kube_cronjob_info{concurrency_policy="Forbid",cronjob="ActiveRunningCronJobWithTZ1",namespace="ns1",schedule="0 */6 * * *",timezone="Asia/Shanghai"} 1
175+
kube_cronjob_annotations{annotation_app_k8s_io_owner="@foo",cronjob="ActiveRunningCronJobWithTZ1",namespace="ns1"} 1
176+
kube_cronjob_spec_failed_job_history_limit{cronjob="ActiveRunningCronJobWithTZ1",namespace="ns1"} 1
177+
kube_cronjob_spec_starting_deadline_seconds{cronjob="ActiveRunningCronJobWithTZ1",namespace="ns1"} 300
178+
kube_cronjob_spec_successful_job_history_limit{cronjob="ActiveRunningCronJobWithTZ1",namespace="ns1"} 3
179+
kube_cronjob_spec_suspend{cronjob="ActiveRunningCronJobWithTZ1",namespace="ns1"} 0
180+
kube_cronjob_status_active{cronjob="ActiveRunningCronJobWithTZ1",namespace="ns1"} 2
181+
kube_cronjob_metadata_resource_version{cronjob="ActiveRunningCronJobWithTZ1",namespace="ns1"} 11111
182+
kube_cronjob_status_last_schedule_time{cronjob="ActiveRunningCronJobWithTZ1",namespace="ns1"} 1.520742896e+09
183+
` + fmt.Sprintf("kube_cronjob_next_schedule_time{cronjob=\"ActiveRunningCronJobWithTZ1\",namespace=\"ns1\"} %ve+09\n",
184+
float64(ActiveRunningCronJobWithTZ1NextScheduleTime.Unix())/math.Pow10(9)),
185+
MetricNames: []string{
186+
"kube_cronjob_next_schedule_time",
187+
"kube_cronjob_spec_starting_deadline_seconds",
188+
"kube_cronjob_status_active",
189+
"kube_cronjob_metadata_resource_version",
190+
"kube_cronjob_spec_suspend",
191+
"kube_cronjob_info",
192+
"kube_cronjob_created",
193+
"kube_cronjob_annotations",
194+
"kube_cronjob_labels",
195+
"kube_cronjob_status_last_schedule_time",
196+
"kube_cronjob_spec_successful_job_history_limit",
197+
"kube_cronjob_spec_failed_job_history_limit",
198+
},
199+
},
105200
{
106201
AllowAnnotationsList: []string{
107202
"app.k8s.io/owner",
@@ -159,7 +254,7 @@ func TestCronJobStore(t *testing.T) {
159254
# TYPE kube_cronjob_status_active gauge
160255
# TYPE kube_cronjob_metadata_resource_version gauge
161256
# TYPE kube_cronjob_status_last_schedule_time gauge
162-
kube_cronjob_info{concurrency_policy="Forbid",cronjob="ActiveRunningCronJob1",namespace="ns1",schedule="0 */6 * * *"} 1
257+
kube_cronjob_info{concurrency_policy="Forbid",cronjob="ActiveRunningCronJob1",namespace="ns1",schedule="0 */6 * * *",timezone="local"} 1
163258
kube_cronjob_annotations{annotation_app_k8s_io_owner="@foo",cronjob="ActiveRunningCronJob1",namespace="ns1"} 1
164259
kube_cronjob_spec_failed_job_history_limit{cronjob="ActiveRunningCronJob1",namespace="ns1"} 1
165260
kube_cronjob_spec_starting_deadline_seconds{cronjob="ActiveRunningCronJob1",namespace="ns1"} 300
@@ -206,6 +301,7 @@ func TestCronJobStore(t *testing.T) {
206301
ConcurrencyPolicy: "Forbid",
207302
Suspend: &SuspendTrue,
208303
Schedule: "0 */3 * * *",
304+
TimeZone: &TimeZone,
209305
SuccessfulJobsHistoryLimit: &SuccessfulJobHistoryLimit3,
210306
FailedJobsHistoryLimit: &FailedJobHistoryLimit1,
211307
},
@@ -233,7 +329,7 @@ func TestCronJobStore(t *testing.T) {
233329
# TYPE kube_cronjob_metadata_resource_version gauge
234330
# TYPE kube_cronjob_status_last_schedule_time gauge
235331
# TYPE kube_cronjob_status_last_successful_time gauge
236-
kube_cronjob_info{concurrency_policy="Forbid",cronjob="SuspendedCronJob1",namespace="ns1",schedule="0 */3 * * *"} 1
332+
kube_cronjob_info{concurrency_policy="Forbid",cronjob="SuspendedCronJob1",namespace="ns1",schedule="0 */3 * * *",timezone="Asia/Shanghai"} 1
237333
kube_cronjob_spec_failed_job_history_limit{cronjob="SuspendedCronJob1",namespace="ns1"} 1
238334
kube_cronjob_spec_starting_deadline_seconds{cronjob="SuspendedCronJob1",namespace="ns1"} 300
239335
kube_cronjob_spec_successful_job_history_limit{cronjob="SuspendedCronJob1",namespace="ns1"} 3
@@ -292,7 +388,7 @@ func TestCronJobStore(t *testing.T) {
292388
# TYPE kube_cronjob_metadata_resource_version gauge
293389
# TYPE kube_cronjob_status_last_schedule_time gauge
294390
# TYPE kube_cronjob_status_last_successful_time gauge
295-
kube_cronjob_info{concurrency_policy="Forbid",cronjob="SuspendedCronJob1",namespace="ns1",schedule="0 */3 * * *"} 1
391+
kube_cronjob_info{concurrency_policy="Forbid",cronjob="SuspendedCronJob1",namespace="ns1",schedule="0 */3 * * *",timezone="local"} 1
296392
kube_cronjob_spec_failed_job_history_limit{cronjob="SuspendedCronJob1",namespace="ns1"} 1
297393
kube_cronjob_spec_starting_deadline_seconds{cronjob="SuspendedCronJob1",namespace="ns1"} 300
298394
kube_cronjob_spec_successful_job_history_limit{cronjob="SuspendedCronJob1",namespace="ns1"} 3
@@ -351,15 +447,15 @@ func TestCronJobStore(t *testing.T) {
351447
# TYPE kube_cronjob_spec_successful_job_history_limit gauge
352448
# TYPE kube_cronjob_spec_suspend gauge
353449
# TYPE kube_cronjob_status_active gauge
354-
# TYPE kube_cronjob_metadata_resource_version gauge
450+
# TYPE kube_cronjob_metadata_resource_version gauge
355451
# TYPE kube_cronjob_status_last_successful_time gauge
356452
kube_cronjob_spec_starting_deadline_seconds{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 300
357453
kube_cronjob_status_active{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 0
358454
kube_cronjob_metadata_resource_version{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 33333
359455
kube_cronjob_spec_failed_job_history_limit{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 1
360456
kube_cronjob_spec_successful_job_history_limit{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 3
361457
kube_cronjob_spec_suspend{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 0
362-
kube_cronjob_info{concurrency_policy="Forbid",cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1",schedule="25 * * * *"} 1
458+
kube_cronjob_info{concurrency_policy="Forbid",cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1",schedule="25 * * * *",timezone="local"} 1
363459
kube_cronjob_created{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 1.520766296e+09
364460
` +
365461
fmt.Sprintf("kube_cronjob_next_schedule_time{cronjob=\"ActiveCronJob1NoLastScheduled\",namespace=\"ns1\"} %ve+09\n",
@@ -375,3 +471,37 @@ func TestCronJobStore(t *testing.T) {
375471
}
376472
}
377473
}
474+
475+
func TestGetNextScheduledTime(t *testing.T) {
476+
477+
testCases := []struct {
478+
schedule string
479+
lastScheduleTime metav1.Time
480+
createdTime metav1.Time
481+
timeZone string
482+
expected time.Time
483+
}{
484+
{
485+
schedule: "0 */6 * * *",
486+
lastScheduleTime: metav1.Time{Time: ActiveRunningCronJob1LastScheduleTime},
487+
createdTime: metav1.Time{Time: ActiveRunningCronJob1LastScheduleTime},
488+
timeZone: "UTC",
489+
expected: ActiveRunningCronJob1LastScheduleTime.Add(time.Second*4 + time.Minute*25 + time.Hour),
490+
},
491+
{
492+
schedule: "0 */6 * * *",
493+
lastScheduleTime: metav1.Time{Time: ActiveRunningCronJob1LastScheduleTime},
494+
createdTime: metav1.Time{Time: ActiveRunningCronJob1LastScheduleTime},
495+
timeZone: TimeZone,
496+
expected: ActiveRunningCronJob1LastScheduleTime.Add(time.Second*4 + time.Minute*25 + time.Hour*5),
497+
},
498+
}
499+
500+
for _, test := range testCases {
501+
actual, _ := getNextScheduledTime(test.schedule, &test.lastScheduleTime, test.createdTime, &test.timeZone) // #nosec G601
502+
if !actual.Equal(test.expected) {
503+
t.Fatalf("%v: expected %v, actual %v", test.schedule, test.expected, actual)
504+
}
505+
}
506+
507+
}

0 commit comments

Comments
 (0)