diff --git a/api/api.go b/api/api.go index 13695a158a..0ab1b3ec96 100644 --- a/api/api.go +++ b/api/api.go @@ -83,7 +83,7 @@ type Options struct { // GroupFunc returns a list of alert groups. The alerts are grouped // according to the current active configuration. Alerts returned are // filtered by the arguments provided to the function. - GroupFunc func(func(*dispatch.Route) bool, func(*types.Alert, time.Time) bool) (dispatch.AlertGroups, map[model.Fingerprint][]string) + GroupFunc func(func(dispatch.Route) bool, func(*types.Alert, time.Time) bool) map[dispatch.Route]map[model.Fingerprint]dispatch.AggregationGroup } func (o Options) validate() error { diff --git a/api/v2/api.go b/api/v2/api.go index dd0ffd6f60..ea92cd1a6c 100644 --- a/api/v2/api.go +++ b/api/v2/api.go @@ -1,4 +1,4 @@ -// Copyright 2018 Prometheus Team +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -57,7 +57,7 @@ type API struct { peer cluster.ClusterPeer silences *silence.Silences alerts provider.Alerts - alertGroups groupsFn + dispatchGroups dispatchGroupsFn getAlertStatus getAlertStatusFn groupMutedFunc groupMutedFunc uptime time.Time @@ -67,7 +67,7 @@ type API struct { // resolveTimeout represents the default resolve timeout that an alert is // assigned if no end time is specified. alertmanagerConfig *config.Config - route *dispatch.Route + route dispatch.Route setAlertStatus setAlertStatusFn logger *slog.Logger @@ -77,7 +77,7 @@ type API struct { } type ( - groupsFn func(func(*dispatch.Route) bool, func(*types.Alert, time.Time) bool) (dispatch.AlertGroups, map[prometheus_model.Fingerprint][]string) + dispatchGroupsFn func(func(dispatch.Route) bool, func(*types.Alert, time.Time) bool) map[dispatch.Route]map[prometheus_model.Fingerprint]dispatch.AggregationGroup groupMutedFunc func(routeID, groupKey string) ([]string, bool) getAlertStatusFn func(prometheus_model.Fingerprint) types.AlertStatus setAlertStatusFn func(prometheus_model.LabelSet) @@ -86,7 +86,7 @@ type ( // NewAPI returns a new Alertmanager API v2. func NewAPI( alerts provider.Alerts, - gf groupsFn, + dgf dispatchGroupsFn, asf getAlertStatusFn, gmf groupMutedFunc, silences *silence.Silences, @@ -97,7 +97,7 @@ func NewAPI( api := API{ alerts: alerts, getAlertStatus: asf, - alertGroups: gf, + dispatchGroups: dgf, groupMutedFunc: gmf, peer: peer, silences: silences, @@ -282,7 +282,7 @@ func (api *API) getAlertsHandler(params alert_ops.GetAlertsParams) middleware.Re routes := api.route.Match(a.Labels) receivers := make([]string, 0, len(routes)) for _, r := range routes { - receivers = append(receivers, r.RouteOpts.Receiver) + receivers = append(receivers, r.Options().Receiver) } if receiverFilter != nil && !receiversMatchFilter(receivers, receiverFilter) { @@ -394,9 +394,9 @@ func (api *API) getAlertGroupsHandler(params alertgroup_ops.GetAlertGroupsParams } } - rf := func(receiverFilter *regexp.Regexp) func(r *dispatch.Route) bool { - return func(r *dispatch.Route) bool { - receiver := r.RouteOpts.Receiver + rf := func(receiverFilter *regexp.Regexp) func(r dispatch.Route) bool { + return func(r dispatch.Route) bool { + receiver := r.Options().Receiver if receiverFilter != nil && !receiverFilter.MatchString(receiver) { return false } @@ -405,11 +405,68 @@ func (api *API) getAlertGroupsHandler(params alertgroup_ops.GetAlertGroupsParams }(receiverFilter) af := api.alertFilter(matchers, *params.Silenced, *params.Inhibited, *params.Active) - alertGroups, allReceivers := api.alertGroups(rf, af) + dispatchGroups := api.dispatchGroups(rf, af) - res := make(open_api_models.AlertGroups, 0, len(alertGroups)) + groups := AlertGroups{} + // Keep a list of receivers for an alert to prevent checking each alert + // again against all routes. The alert has already matched against this + // route on ingestion. + receivers := map[prometheus_model.Fingerprint][]string{} - for _, alertGroup := range alertGroups { + now := time.Now() + for route, ags := range dispatchGroups { + if !rf(route) { + continue + } + + for _, ag := range ags { + receiver := route.Options().Receiver + alertGroup := &AlertGroup{ + Labels: ag.Labels(), + Receiver: receiver, + GroupKey: ag.GroupKey(), + RouteID: ag.RouteID(), + } + + alerts := ag.Alerts() + filteredAlerts := make([]*types.Alert, 0, len(alerts)) + for _, a := range alerts { + if !af(a, now) { + continue + } + + fp := a.Fingerprint() + if r, ok := receivers[fp]; ok { + // Receivers slice already exists. Add + // the current receiver to the slice. + receivers[fp] = append(r, receiver) + } else { + // First time we've seen this alert fingerprint. + // Initialize a new receivers slice. + receivers[fp] = []string{receiver} + } + + filteredAlerts = append(filteredAlerts, a) + } + if len(filteredAlerts) == 0 { + continue + } + alertGroup.Alerts = filteredAlerts + + groups = append(groups, alertGroup) + } + } + sort.Sort(groups) + for i := range groups { + sort.Sort(groups[i].Alerts) + } + for i := range receivers { + sort.Strings(receivers[i]) + } + + res := make(open_api_models.AlertGroups, 0, len(groups)) + + for _, alertGroup := range groups { mutedBy, isMuted := api.groupMutedFunc(alertGroup.RouteID, alertGroup.GroupKey) if !*params.Muted && isMuted { continue @@ -423,7 +480,7 @@ func (api *API) getAlertGroupsHandler(params alertgroup_ops.GetAlertGroupsParams for _, alert := range alertGroup.Alerts { fp := alert.Fingerprint() - receivers := allReceivers[fp] + receivers := receivers[fp] status := api.getAlertStatus(fp) apiAlert := AlertToOpenAPIAlert(alert, status, receivers, mutedBy) ag.Alerts = append(ag.Alerts, apiAlert) diff --git a/api/v2/api_test.go b/api/v2/api_test.go index ea5f9ae99b..7ae655a4f9 100644 --- a/api/v2/api_test.go +++ b/api/v2/api_test.go @@ -31,10 +31,12 @@ import ( "github.com/stretchr/testify/require" open_api_models "github.com/prometheus/alertmanager/api/v2/models" + alertgroup_ops "github.com/prometheus/alertmanager/api/v2/restapi/operations/alertgroup" general_ops "github.com/prometheus/alertmanager/api/v2/restapi/operations/general" receiver_ops "github.com/prometheus/alertmanager/api/v2/restapi/operations/receiver" silence_ops "github.com/prometheus/alertmanager/api/v2/restapi/operations/silence" "github.com/prometheus/alertmanager/config" + "github.com/prometheus/alertmanager/dispatch" "github.com/prometheus/alertmanager/pkg/labels" "github.com/prometheus/alertmanager/silence" "github.com/prometheus/alertmanager/silence/silencepb" @@ -583,3 +585,326 @@ receivers: require.Equal(t, tc.body, string(body)) } } + +func TestGetAlertGroupsHandler(t *testing.T) { + now := time.Now() + + // Create test alerts + alert1 := &types.Alert{ + Alert: model.Alert{ + Labels: model.LabelSet{ + "alertname": "TestAlert1", + "severity": "critical", + "env": "prod", + }, + Annotations: model.LabelSet{}, + StartsAt: now.Add(-10 * time.Minute), + }, + UpdatedAt: now, + } + + alert2 := &types.Alert{ + Alert: model.Alert{ + Labels: model.LabelSet{ + "alertname": "TestAlert2", + "severity": "warning", + "env": "staging", + }, + Annotations: model.LabelSet{}, + StartsAt: now.Add(-5 * time.Minute), + }, + UpdatedAt: now, + } + + alert3 := &types.Alert{ + Alert: model.Alert{ + Labels: model.LabelSet{ + "alertname": "TestAlert3", + "severity": "info", + "env": "prod", + }, + Annotations: model.LabelSet{}, + StartsAt: now.Add(-2 * time.Minute), + EndsAt: now.Add(-1 * time.Minute), // Resolved + }, + UpdatedAt: now, + } + + // Mock route interface + mockRoute := &mockRouteImpl{ + receiver: "team-X", + groupKey: "{}:{alertname=\"TestAlert1\"}", + routeID: "{}", + } + + mockRoute2 := &mockRouteImpl{ + receiver: "team-Y", + groupKey: "{}:{alertname=\"TestAlert2\"}", + routeID: "{}1", + } + + // Mock aggregation group + mockAggGroup1 := &mockAggGroup{ + alerts: []*types.Alert{alert1}, + labels: model.LabelSet{"alertname": "TestAlert1"}, + groupKey: "{}:{alertname=\"TestAlert1\"}", + routeID: "{}", + } + + mockAggGroup2 := &mockAggGroup{ + alerts: []*types.Alert{alert2}, + labels: model.LabelSet{"alertname": "TestAlert2"}, + groupKey: "{}:{alertname=\"TestAlert2\"}", + routeID: "{}1", + } + + mockAggGroup3 := &mockAggGroup{ + alerts: []*types.Alert{alert3}, + labels: model.LabelSet{"alertname": "TestAlert3"}, + groupKey: "{}:{alertname=\"TestAlert3\"}", + routeID: "{}", + } + + // Create test cases + tests := []struct { + name string + filter []string + receiver *string + active *bool + silenced *bool + inhibited *bool + muted *bool + dispatchGroups map[dispatch.Route]map[model.Fingerprint]dispatch.AggregationGroup + expectedCount int + expectedCode int + }{ + { + name: "get all alert groups", + filter: nil, + receiver: nil, + active: ptrBool(true), + silenced: ptrBool(true), + inhibited: ptrBool(true), + muted: ptrBool(true), + dispatchGroups: map[dispatch.Route]map[model.Fingerprint]dispatch.AggregationGroup{ + mockRoute: { + alert1.Fingerprint(): mockAggGroup1, + }, + mockRoute2: { + alert2.Fingerprint(): mockAggGroup2, + }, + }, + expectedCount: 2, + expectedCode: 200, + }, + { + name: "filter by receiver", + filter: nil, + receiver: ptrString("team-X"), + active: ptrBool(true), + silenced: ptrBool(true), + inhibited: ptrBool(true), + muted: ptrBool(true), + dispatchGroups: map[dispatch.Route]map[model.Fingerprint]dispatch.AggregationGroup{ + mockRoute: { + alert1.Fingerprint(): mockAggGroup1, + }, + mockRoute2: { + alert2.Fingerprint(): mockAggGroup2, + }, + }, + expectedCount: 1, + expectedCode: 200, + }, + { + name: "filter by label matcher", + filter: []string{"env=prod"}, + receiver: nil, + active: ptrBool(true), + silenced: ptrBool(true), + inhibited: ptrBool(true), + muted: ptrBool(true), + dispatchGroups: map[dispatch.Route]map[model.Fingerprint]dispatch.AggregationGroup{ + mockRoute: { + alert1.Fingerprint(): mockAggGroup1, + }, + mockRoute2: { + alert2.Fingerprint(): mockAggGroup2, + }, + }, + expectedCount: 1, + expectedCode: 200, + }, + { + name: "filter out resolved alerts when active=false", + filter: nil, + receiver: nil, + active: ptrBool(false), + silenced: ptrBool(true), + inhibited: ptrBool(true), + muted: ptrBool(true), + dispatchGroups: map[dispatch.Route]map[model.Fingerprint]dispatch.AggregationGroup{ + mockRoute: { + alert3.Fingerprint(): mockAggGroup3, + }, + }, + expectedCount: 0, + expectedCode: 200, + }, + { + name: "invalid filter", + filter: []string{"invalid{filter"}, + receiver: nil, + active: ptrBool(true), + silenced: ptrBool(true), + inhibited: ptrBool(true), + muted: ptrBool(true), + dispatchGroups: map[dispatch.Route]map[model.Fingerprint]dispatch.AggregationGroup{}, + expectedCount: 0, + expectedCode: 400, + }, + { + name: "invalid receiver regex", + filter: nil, + receiver: ptrString("[invalid"), + active: ptrBool(true), + silenced: ptrBool(true), + inhibited: ptrBool(true), + muted: ptrBool(true), + dispatchGroups: map[dispatch.Route]map[model.Fingerprint]dispatch.AggregationGroup{}, + expectedCount: 0, + expectedCode: 400, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // Setup API with mock dependencies + api := API{ + uptime: now, + logger: promslog.NewNopLogger(), + dispatchGroups: func(routeFilter func(dispatch.Route) bool, alertFilter func(*types.Alert, time.Time) bool) map[dispatch.Route]map[model.Fingerprint]dispatch.AggregationGroup { + return tc.dispatchGroups + }, + getAlertStatus: func(fp model.Fingerprint) types.AlertStatus { + return types.AlertStatus{State: types.AlertStateActive} + }, + setAlertStatus: func(model.LabelSet) { + // No-op for testing + }, + groupMutedFunc: func(routeID, groupKey string) ([]string, bool) { + return nil, false + }, + } + + // Create HTTP request + r, err := http.NewRequest("GET", "/api/v2/alerts/groups", nil) + require.NoError(t, err) + + // Setup parameters + params := alertgroup_ops.GetAlertGroupsParams{ + HTTPRequest: r, + Filter: tc.filter, + Receiver: tc.receiver, + Active: tc.active, + Silenced: tc.silenced, + Inhibited: tc.inhibited, + Muted: tc.muted, + } + + // Execute handler + w := httptest.NewRecorder() + p := runtime.TextProducer() + responder := api.getAlertGroupsHandler(params) + responder.WriteResponse(w, p) + + body, _ := io.ReadAll(w.Result().Body) + + // Verify response code + require.Equal(t, tc.expectedCode, w.Code, "response: %s", string(body)) + + // If success, verify response body + if tc.expectedCode == 200 { + var resp open_api_models.AlertGroups + err := json.Unmarshal(body, &resp) + require.NoError(t, err) + require.Len(t, resp, tc.expectedCount) + } + }) + } +} + +// Helper types for mocking. +type mockRouteImpl struct { + receiver string + groupKey string + routeID string +} + +func (m *mockRouteImpl) Key() string { + return m.groupKey +} + +func (m *mockRouteImpl) ID() string { + return m.routeID +} + +func (m *mockRouteImpl) Options() dispatch.RouteOpts { + return dispatch.RouteOpts{ + Receiver: m.receiver, + } +} + +func (m *mockRouteImpl) Match(model.LabelSet) []dispatch.Route { + return nil +} + +func (m *mockRouteImpl) Matchers() labels.Matchers { + return nil +} + +func (m *mockRouteImpl) Continues() bool { + return false +} + +func (m *mockRouteImpl) Routes() []dispatch.Route { + return nil +} + +func (m *mockRouteImpl) Walk(func(dispatch.Route)) {} + +type mockAggGroup struct { + alerts []*types.Alert + labels model.LabelSet + groupKey string + routeID string +} + +func (m *mockAggGroup) Alerts() []*types.Alert { + return m.alerts +} + +func (m *mockAggGroup) Labels() model.LabelSet { + return m.labels +} + +func (m *mockAggGroup) GroupKey() string { + return m.groupKey +} + +func (m *mockAggGroup) RouteID() string { + return m.routeID +} + +func (m *mockAggGroup) Receiver() string { + return "" +} + +// Helper functions. +func ptrBool(b bool) *bool { + return &b +} + +func ptrString(s string) *string { + return &s +} diff --git a/api/v2/type.go b/api/v2/type.go new file mode 100644 index 0000000000..8855d91a2d --- /dev/null +++ b/api/v2/type.go @@ -0,0 +1,40 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v2 + +import ( + "github.com/prometheus/common/model" + + "github.com/prometheus/alertmanager/types" +) + +// AlertGroup represents how alerts exist within an aggrGroup. +type AlertGroup struct { + Alerts types.AlertSlice + Labels model.LabelSet + Receiver string + GroupKey string + RouteID string +} + +type AlertGroups []*AlertGroup + +func (ag AlertGroups) Swap(i, j int) { ag[i], ag[j] = ag[j], ag[i] } +func (ag AlertGroups) Less(i, j int) bool { + if ag[i].Labels.Equal(ag[j].Labels) { + return ag[i].Receiver < ag[j].Receiver + } + return ag[i].Labels.Before(ag[j].Labels) +} +func (ag AlertGroups) Len() int { return len(ag) } diff --git a/cli/routing.go b/cli/routing.go index b2f69af3c3..ebbc64372d 100644 --- a/cli/routing.go +++ b/cli/routing.go @@ -75,40 +75,40 @@ func (c *routingShow) routingShowAction(ctx context.Context, _ *kingpin.ParseCon return nil } -func getRouteTreeSlug(route *dispatch.Route, showContinue, showReceiver bool) string { +func getRouteTreeSlug(route dispatch.Route, showContinue, showReceiver bool) string { var branchSlug bytes.Buffer - if route.Matchers.Len() == 0 { + if route.Matchers().Len() == 0 { branchSlug.WriteString("default-route") } else { - branchSlug.WriteString(route.Matchers.String()) + branchSlug.WriteString(route.Matchers().String()) } - if route.Continue && showContinue { + if route.Continues() && showContinue { branchSlug.WriteString(branchSlugSeparator) branchSlug.WriteString("continue: true") } if showReceiver { branchSlug.WriteString(branchSlugSeparator) branchSlug.WriteString("receiver: ") - branchSlug.WriteString(route.RouteOpts.Receiver) + branchSlug.WriteString(route.Options().Receiver) } return branchSlug.String() } -func convertRouteToTree(route *dispatch.Route, tree treeprint.Tree) { +func convertRouteToTree(route dispatch.Route, tree treeprint.Tree) { branch := tree.AddBranch(getRouteTreeSlug(route, true, true)) - for _, r := range route.Routes { + for _, r := range route.Routes() { convertRouteToTree(r, branch) } } -func getMatchingTree(route *dispatch.Route, tree treeprint.Tree, lset models.LabelSet) { +func getMatchingTree(route dispatch.Route, tree treeprint.Tree, lset models.LabelSet) { final := true branch := tree.AddBranch(getRouteTreeSlug(route, false, false)) - for _, r := range route.Routes { - if r.Matchers.Matches(convertClientToCommonLabelSet(lset)) { + for _, r := range route.Routes() { + if r.Matchers().Matches(convertClientToCommonLabelSet(lset)) { getMatchingTree(r, branch, lset) final = false - if !r.Continue { + if !r.Continues() { break } } diff --git a/cli/test_routing.go b/cli/test_routing.go index 163dce391b..9f71602c68 100644 --- a/cli/test_routing.go +++ b/cli/test_routing.go @@ -52,19 +52,19 @@ func configureRoutingTestCmd(cc *kingpin.CmdClause, c *routingShow) { } // resolveAlertReceivers returns list of receiver names which given LabelSet resolves to. -func resolveAlertReceivers(mainRoute *dispatch.Route, labels *models.LabelSet) ([]string, error) { +func resolveAlertReceivers(mainRoute dispatch.Route, labels *models.LabelSet) ([]string, error) { var ( - finalRoutes []*dispatch.Route + finalRoutes []dispatch.Route receivers []string ) finalRoutes = mainRoute.Match(convertClientToCommonLabelSet(*labels)) for _, r := range finalRoutes { - receivers = append(receivers, r.RouteOpts.Receiver) + receivers = append(receivers, r.Options().Receiver) } return receivers, nil } -func printMatchingTree(mainRoute *dispatch.Route, ls models.LabelSet) { +func printMatchingTree(mainRoute dispatch.Route, ls models.LabelSet) { tree := treeprint.New() getMatchingTree(mainRoute, tree, ls) fmt.Println("Matching routes:") diff --git a/cli/test_routing_test.go b/cli/test_routing_test.go index 104d3a36ad..a3772fbdbb 100644 --- a/cli/test_routing_test.go +++ b/cli/test_routing_test.go @@ -30,7 +30,7 @@ type routingTestDefinition struct { configFile string } -func checkResolvedReceivers(mainRoute *dispatch.Route, ls models.LabelSet, expectedReceivers []string) error { +func checkResolvedReceivers(mainRoute dispatch.Route, ls models.LabelSet, expectedReceivers []string) error { resolvedReceivers, err := resolveAlertReceivers(mainRoute, &ls) if err != nil { return err diff --git a/cmd/alertmanager/main.go b/cmd/alertmanager/main.go index 971e74f7a0..5b66c9e227 100644 --- a/cmd/alertmanager/main.go +++ b/cmd/alertmanager/main.go @@ -358,7 +358,7 @@ func run() int { disp.Stop() }() - groupFn := func(routeFilter func(*dispatch.Route) bool, alertFilter func(*types.Alert, time.Time) bool) (dispatch.AlertGroups, map[model.Fingerprint][]string) { + groupFn := func(routeFilter func(dispatch.Route) bool, alertFilter func(*types.Alert, time.Time) bool) map[dispatch.Route]map[model.Fingerprint]dispatch.AggregationGroup { return disp.Groups(routeFilter, alertFilter) } @@ -430,8 +430,8 @@ func run() int { // Build the routing tree and record which receivers are used. routes := dispatch.NewRoute(conf.Route, nil) activeReceivers := make(map[string]struct{}) - routes.Walk(func(r *dispatch.Route) { - activeReceivers[r.RouteOpts.Receiver] = struct{}{} + routes.Walk(func(r dispatch.Route) { + activeReceivers[r.Options().Receiver] = struct{}{} }) // Build the map of receiver to integrations. @@ -499,12 +499,12 @@ func run() int { }) disp = dispatch.NewDispatcher(alerts, routes, pipeline, marker, timeoutFunc, *dispatchMaintenanceInterval, nil, logger, dispMetrics) - routes.Walk(func(r *dispatch.Route) { - if r.RouteOpts.RepeatInterval > *retention { + routes.Walk(func(r dispatch.Route) { + if r.Options().RepeatInterval > *retention { configLogger.Warn( "repeat_interval is greater than the data retention period. It can lead to notifications being repeated more often than expected.", "repeat_interval", - r.RouteOpts.RepeatInterval, + r.Options().RepeatInterval, "retention", *retention, "route", @@ -512,13 +512,13 @@ func run() int { ) } - if r.RouteOpts.RepeatInterval < r.RouteOpts.GroupInterval { + if r.Options().RepeatInterval < r.Options().GroupInterval { configLogger.Warn( "repeat_interval is less than group_interval. Notifications will not repeat until the next group_interval.", "repeat_interval", - r.RouteOpts.RepeatInterval, + r.Options().RepeatInterval, "group_interval", - r.RouteOpts.GroupInterval, + r.Options().GroupInterval, "route", r.Key(), ) diff --git a/dispatch/dispatch.go b/dispatch/dispatch.go index 880bdb0891..58acc13d4f 100644 --- a/dispatch/dispatch.go +++ b/dispatch/dispatch.go @@ -74,7 +74,7 @@ func NewDispatcherMetrics(registerLimitMetrics bool, r prometheus.Registerer) *D // Dispatcher sorts incoming alerts into aggregation groups and // assigns the correct notifiers to each. type Dispatcher struct { - route *Route + route Route alerts provider.Alerts stage notify.Stage marker types.GroupMarker @@ -84,7 +84,7 @@ type Dispatcher struct { timeout func(time.Duration) time.Duration mtx sync.RWMutex - aggrGroupsPerRoute map[*Route]map[model.Fingerprint]*aggrGroup + aggrGroupsPerRoute map[Route]map[model.Fingerprint]*aggrGroup aggrGroupsNum int maintenanceInterval time.Duration @@ -106,7 +106,7 @@ type Limits interface { // NewDispatcher returns a new Dispatcher. func NewDispatcher( ap provider.Alerts, - r *Route, + r Route, s notify.Stage, mk types.GroupMarker, to func(time.Duration) time.Duration, @@ -138,7 +138,7 @@ func (d *Dispatcher) Run() { d.done = make(chan struct{}) d.mtx.Lock() - d.aggrGroupsPerRoute = map[*Route]map[model.Fingerprint]*aggrGroup{} + d.aggrGroupsPerRoute = map[Route]map[model.Fingerprint]*aggrGroup{} d.aggrGroupsNum = 0 d.metrics.aggrGroups.Set(0) d.ctx, d.cancel = context.WithCancel(context.Background()) @@ -223,70 +223,31 @@ func (ag AlertGroups) Less(i, j int) bool { } func (ag AlertGroups) Len() int { return len(ag) } -// Groups returns a slice of AlertGroups from the dispatcher's internal state. -func (d *Dispatcher) Groups(routeFilter func(*Route) bool, alertFilter func(*types.Alert, time.Time) bool) (AlertGroups, map[model.Fingerprint][]string) { - groups := AlertGroups{} - +// Groups returns a snapshot of the dispatcher's internal state. +func (d *Dispatcher) Groups(routeFilter func(Route) bool, alertFilter func(*types.Alert, time.Time) bool) map[Route]map[model.Fingerprint]AggregationGroup { d.mtx.RLock() defer d.mtx.RUnlock() - // Keep a list of receivers for an alert to prevent checking each alert - // again against all routes. The alert has already matched against this - // route on ingestion. - receivers := map[model.Fingerprint][]string{} - - now := time.Now() + // Make a snapshot of the aggrGroupsPerRoute map to use for this function. + // This ensures that we hold the Dispatcher.mtx for as little time as + // possible. + // It also prevents us from holding the any locks in alertFilter or routeFilter + // while we hold the dispatcher lock + aggrGroupsPerRoute := map[Route]map[model.Fingerprint]AggregationGroup{} for route, ags := range d.aggrGroupsPerRoute { - if !routeFilter(route) { - continue - } - - for _, ag := range ags { - receiver := route.RouteOpts.Receiver - alertGroup := &AlertGroup{ - Labels: ag.labels, - Receiver: receiver, - GroupKey: ag.GroupKey(), - RouteID: ag.routeID, - } - - alerts := ag.alerts.List() - filteredAlerts := make([]*types.Alert, 0, len(alerts)) - for _, a := range alerts { - if !alertFilter(a, now) { - continue - } - - fp := a.Fingerprint() - if r, ok := receivers[fp]; ok { - // Receivers slice already exists. Add - // the current receiver to the slice. - receivers[fp] = append(r, receiver) - } else { - // First time we've seen this alert fingerprint. - // Initialize a new receivers slice. - receivers[fp] = []string{receiver} - } - - filteredAlerts = append(filteredAlerts, a) - } - if len(filteredAlerts) == 0 { - continue - } - alertGroup.Alerts = filteredAlerts - - groups = append(groups, alertGroup) + // Since other goroutines could modify d.aggrGroupsPerRoute, we need to + // copy it. We DON'T need to copy the aggrGroup objects because they each + // have a mutex protecting their internal state. + // The aggrGroup methods use the internal lock. It is important to avoid + // accessing internal fields on the aggrGroup objects. + copiedMap := map[model.Fingerprint]AggregationGroup{} + for fp, ag := range ags { + copiedMap[fp] = ag } - } - sort.Sort(groups) - for i := range groups { - sort.Sort(groups[i].Alerts) - } - for i := range receivers { - sort.Strings(receivers[i]) + aggrGroupsPerRoute[route] = copiedMap } - return groups, receivers + return aggrGroupsPerRoute } // Stop the dispatcher. @@ -313,7 +274,7 @@ type notifyFunc func(context.Context, ...*types.Alert) bool // processAlert determines in which aggregation group the alert falls // and inserts it. -func (d *Dispatcher) processAlert(alert *types.Alert, route *Route) { +func (d *Dispatcher) processAlert(alert *types.Alert, route Route) { groupLabels := getGroupLabels(alert, route) fp := groupLabels.Fingerprint() @@ -367,10 +328,11 @@ func (d *Dispatcher) processAlert(alert *types.Alert, route *Route) { }) } -func getGroupLabels(alert *types.Alert, route *Route) model.LabelSet { +func getGroupLabels(alert *types.Alert, route Route) model.LabelSet { groupLabels := model.LabelSet{} + options := route.Options() for ln, lv := range alert.Labels { - if _, ok := route.RouteOpts.GroupBy[ln]; ok || route.RouteOpts.GroupByAll { + if _, ok := options.GroupBy[ln]; ok || options.GroupByAll { groupLabels[ln] = lv } } @@ -383,7 +345,7 @@ func getGroupLabels(alert *types.Alert, route *Route) model.LabelSet { // It emits notifications in the specified intervals. type aggrGroup struct { labels model.LabelSet - opts *RouteOpts + opts RouteOpts logger *slog.Logger routeID string routeKey string @@ -401,7 +363,7 @@ type aggrGroup struct { } // newAggrGroup returns a new aggregation group. -func newAggrGroup(ctx context.Context, labels model.LabelSet, r *Route, to func(time.Duration) time.Duration, marker types.AlertMarker, logger *slog.Logger) *aggrGroup { +func newAggrGroup(ctx context.Context, labels model.LabelSet, r Route, to func(time.Duration) time.Duration, marker types.AlertMarker, logger *slog.Logger) *aggrGroup { if to == nil { to = func(d time.Duration) time.Duration { return d } } @@ -409,7 +371,7 @@ func newAggrGroup(ctx context.Context, labels model.LabelSet, r *Route, to func( labels: labels, routeID: r.ID(), routeKey: r.Key(), - opts: &r.RouteOpts, + opts: r.Options(), timeout: to, alerts: store.NewAlerts(), marker: marker, @@ -430,10 +392,24 @@ func (ag *aggrGroup) fingerprint() model.Fingerprint { return ag.labels.Fingerprint() } +func (ag *aggrGroup) Alerts() []*types.Alert { + ag.mtx.RLock() + defer ag.mtx.RUnlock() + return ag.alerts.List() +} + func (ag *aggrGroup) GroupKey() string { return fmt.Sprintf("%s:%s", ag.routeKey, ag.labels) } +func (ag *aggrGroup) Labels() model.LabelSet { + return ag.labels.Clone() +} + +func (ag *aggrGroup) RouteID() string { + return ag.routeID +} + func (ag *aggrGroup) String() string { return ag.GroupKey() } diff --git a/dispatch/dispatch_test.go b/dispatch/dispatch_test.go index fe4df6d2ef..acbb08da9e 100644 --- a/dispatch/dispatch_test.go +++ b/dispatch/dispatch_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 Prometheus Team +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -24,12 +24,10 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/common/model" "github.com/prometheus/common/promslog" "github.com/stretchr/testify/require" - "github.com/prometheus/alertmanager/config" "github.com/prometheus/alertmanager/notify" "github.com/prometheus/alertmanager/provider/mem" "github.com/prometheus/alertmanager/types" @@ -52,7 +50,7 @@ func TestAggrGroup(t *testing.T) { GroupInterval: 300 * time.Millisecond, RepeatInterval: 1 * time.Hour, } - route := &Route{ + route := &route{ RouteOpts: *opts, } @@ -307,7 +305,7 @@ func TestGroupLabels(t *testing.T) { }, } - route := &Route{ + route := &route{ RouteOpts: RouteOpts{ GroupBy: map[model.LabelName]struct{}{ "a": {}, @@ -340,7 +338,7 @@ func TestGroupByAllLabels(t *testing.T) { }, } - route := &Route{ + route := &route{ RouteOpts: RouteOpts{ GroupBy: map[model.LabelName]struct{}{}, GroupByAll: true, @@ -360,255 +358,6 @@ func TestGroupByAllLabels(t *testing.T) { } } -func TestGroups(t *testing.T) { - confData := `receivers: -- name: 'kafka' -- name: 'prod' -- name: 'testing' - -route: - group_by: ['alertname'] - group_wait: 10ms - group_interval: 10ms - receiver: 'prod' - routes: - - match: - env: 'testing' - receiver: 'testing' - group_by: ['alertname', 'service'] - - match: - env: 'prod' - receiver: 'prod' - group_by: ['alertname', 'service', 'cluster'] - continue: true - - match: - kafka: 'yes' - receiver: 'kafka' - group_by: ['alertname', 'service', 'cluster']` - conf, err := config.Load(confData) - if err != nil { - t.Fatal(err) - } - - logger := promslog.NewNopLogger() - route := NewRoute(conf.Route, nil) - reg := prometheus.NewRegistry() - marker := types.NewMarker(reg) - alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, nil, logger, reg) - if err != nil { - t.Fatal(err) - } - defer alerts.Close() - - timeout := func(d time.Duration) time.Duration { return time.Duration(0) } - recorder := &recordStage{alerts: make(map[string]map[model.Fingerprint]*types.Alert)} - dispatcher := NewDispatcher(alerts, route, recorder, marker, timeout, testMaintenanceInterval, nil, logger, NewDispatcherMetrics(false, reg)) - go dispatcher.Run() - defer dispatcher.Stop() - - // Create alerts. the dispatcher will automatically create the groups. - inputAlerts := []*types.Alert{ - // Matches the parent route. - newAlert(model.LabelSet{"alertname": "OtherAlert", "cluster": "cc", "service": "dd"}), - // Matches the first sub-route. - newAlert(model.LabelSet{"env": "testing", "alertname": "TestingAlert", "service": "api", "instance": "inst1"}), - // Matches the second sub-route. - newAlert(model.LabelSet{"env": "prod", "alertname": "HighErrorRate", "cluster": "aa", "service": "api", "instance": "inst1"}), - newAlert(model.LabelSet{"env": "prod", "alertname": "HighErrorRate", "cluster": "aa", "service": "api", "instance": "inst2"}), - // Matches the second sub-route. - newAlert(model.LabelSet{"env": "prod", "alertname": "HighErrorRate", "cluster": "bb", "service": "api", "instance": "inst1"}), - // Matches the second and third sub-route. - newAlert(model.LabelSet{"env": "prod", "alertname": "HighLatency", "cluster": "bb", "service": "db", "kafka": "yes", "instance": "inst3"}), - } - alerts.Put(inputAlerts...) - - // Let alerts get processed. - for i := 0; len(recorder.Alerts()) != 7 && i < 10; i++ { - time.Sleep(200 * time.Millisecond) - } - require.Len(t, recorder.Alerts(), 7) - - alertGroups, receivers := dispatcher.Groups( - func(*Route) bool { - return true - }, func(*types.Alert, time.Time) bool { - return true - }, - ) - - require.Equal(t, AlertGroups{ - &AlertGroup{ - Alerts: []*types.Alert{inputAlerts[0]}, - Labels: model.LabelSet{ - "alertname": "OtherAlert", - }, - Receiver: "prod", - GroupKey: "{}:{alertname=\"OtherAlert\"}", - RouteID: "{}", - }, - &AlertGroup{ - Alerts: []*types.Alert{inputAlerts[1]}, - Labels: model.LabelSet{ - "alertname": "TestingAlert", - "service": "api", - }, - Receiver: "testing", - GroupKey: "{}/{env=\"testing\"}:{alertname=\"TestingAlert\", service=\"api\"}", - RouteID: "{}/{env=\"testing\"}/0", - }, - &AlertGroup{ - Alerts: []*types.Alert{inputAlerts[2], inputAlerts[3]}, - Labels: model.LabelSet{ - "alertname": "HighErrorRate", - "service": "api", - "cluster": "aa", - }, - Receiver: "prod", - GroupKey: "{}/{env=\"prod\"}:{alertname=\"HighErrorRate\", cluster=\"aa\", service=\"api\"}", - RouteID: "{}/{env=\"prod\"}/1", - }, - &AlertGroup{ - Alerts: []*types.Alert{inputAlerts[4]}, - Labels: model.LabelSet{ - "alertname": "HighErrorRate", - "service": "api", - "cluster": "bb", - }, - Receiver: "prod", - GroupKey: "{}/{env=\"prod\"}:{alertname=\"HighErrorRate\", cluster=\"bb\", service=\"api\"}", - RouteID: "{}/{env=\"prod\"}/1", - }, - &AlertGroup{ - Alerts: []*types.Alert{inputAlerts[5]}, - Labels: model.LabelSet{ - "alertname": "HighLatency", - "service": "db", - "cluster": "bb", - }, - Receiver: "kafka", - GroupKey: "{}/{kafka=\"yes\"}:{alertname=\"HighLatency\", cluster=\"bb\", service=\"db\"}", - RouteID: "{}/{kafka=\"yes\"}/2", - }, - &AlertGroup{ - Alerts: []*types.Alert{inputAlerts[5]}, - Labels: model.LabelSet{ - "alertname": "HighLatency", - "service": "db", - "cluster": "bb", - }, - Receiver: "prod", - GroupKey: "{}/{env=\"prod\"}:{alertname=\"HighLatency\", cluster=\"bb\", service=\"db\"}", - RouteID: "{}/{env=\"prod\"}/1", - }, - }, alertGroups) - require.Equal(t, map[model.Fingerprint][]string{ - inputAlerts[0].Fingerprint(): {"prod"}, - inputAlerts[1].Fingerprint(): {"testing"}, - inputAlerts[2].Fingerprint(): {"prod"}, - inputAlerts[3].Fingerprint(): {"prod"}, - inputAlerts[4].Fingerprint(): {"prod"}, - inputAlerts[5].Fingerprint(): {"kafka", "prod"}, - }, receivers) -} - -func TestGroupsWithLimits(t *testing.T) { - confData := `receivers: -- name: 'kafka' -- name: 'prod' -- name: 'testing' - -route: - group_by: ['alertname'] - group_wait: 10ms - group_interval: 10ms - receiver: 'prod' - routes: - - match: - env: 'testing' - receiver: 'testing' - group_by: ['alertname', 'service'] - - match: - env: 'prod' - receiver: 'prod' - group_by: ['alertname', 'service', 'cluster'] - continue: true - - match: - kafka: 'yes' - receiver: 'kafka' - group_by: ['alertname', 'service', 'cluster']` - conf, err := config.Load(confData) - if err != nil { - t.Fatal(err) - } - - logger := promslog.NewNopLogger() - route := NewRoute(conf.Route, nil) - reg := prometheus.NewRegistry() - marker := types.NewMarker(reg) - alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, nil, logger, reg) - if err != nil { - t.Fatal(err) - } - defer alerts.Close() - - timeout := func(d time.Duration) time.Duration { return time.Duration(0) } - recorder := &recordStage{alerts: make(map[string]map[model.Fingerprint]*types.Alert)} - lim := limits{groups: 6} - m := NewDispatcherMetrics(true, reg) - dispatcher := NewDispatcher(alerts, route, recorder, marker, timeout, testMaintenanceInterval, lim, logger, m) - go dispatcher.Run() - defer dispatcher.Stop() - - // Create alerts. the dispatcher will automatically create the groups. - inputAlerts := []*types.Alert{ - // Matches the parent route. - newAlert(model.LabelSet{"alertname": "OtherAlert", "cluster": "cc", "service": "dd"}), - // Matches the first sub-route. - newAlert(model.LabelSet{"env": "testing", "alertname": "TestingAlert", "service": "api", "instance": "inst1"}), - // Matches the second sub-route. - newAlert(model.LabelSet{"env": "prod", "alertname": "HighErrorRate", "cluster": "aa", "service": "api", "instance": "inst1"}), - newAlert(model.LabelSet{"env": "prod", "alertname": "HighErrorRate", "cluster": "aa", "service": "api", "instance": "inst2"}), - // Matches the second sub-route. - newAlert(model.LabelSet{"env": "prod", "alertname": "HighErrorRate", "cluster": "bb", "service": "api", "instance": "inst1"}), - // Matches the second and third sub-route. - newAlert(model.LabelSet{"env": "prod", "alertname": "HighLatency", "cluster": "bb", "service": "db", "kafka": "yes", "instance": "inst3"}), - } - err = alerts.Put(inputAlerts...) - if err != nil { - t.Fatal(err) - } - - // Let alerts get processed. - for i := 0; len(recorder.Alerts()) != 7 && i < 10; i++ { - time.Sleep(200 * time.Millisecond) - } - require.Len(t, recorder.Alerts(), 7) - - routeFilter := func(*Route) bool { return true } - alertFilter := func(*types.Alert, time.Time) bool { return true } - - alertGroups, _ := dispatcher.Groups(routeFilter, alertFilter) - require.Len(t, alertGroups, 6) - - require.Equal(t, 0.0, testutil.ToFloat64(m.aggrGroupLimitReached)) - - // Try to store new alert. This time, we will hit limit for number of groups. - err = alerts.Put(newAlert(model.LabelSet{"env": "prod", "alertname": "NewAlert", "cluster": "new-cluster", "service": "db"})) - if err != nil { - t.Fatal(err) - } - - // Let alert get processed. - for i := 0; testutil.ToFloat64(m.aggrGroupLimitReached) == 0 && i < 10; i++ { - time.Sleep(200 * time.Millisecond) - } - require.Equal(t, 1.0, testutil.ToFloat64(m.aggrGroupLimitReached)) - - // Verify there are still only 6 groups. - alertGroups, _ = dispatcher.Groups(routeFilter, alertFilter) - require.Len(t, alertGroups, 6) -} - type recordStage struct { mtx sync.RWMutex alerts map[string]map[model.Fingerprint]*types.Alert @@ -691,7 +440,7 @@ func TestDispatcherRaceOnFirstAlertNotDeliveredWhenGroupWaitIsZero(t *testing.T) } defer alerts.Close() - route := &Route{ + route := &route{ RouteOpts: RouteOpts{ Receiver: "default", GroupBy: map[model.LabelName]struct{}{"alertname": {}}, @@ -727,14 +476,6 @@ func TestDispatcherRaceOnFirstAlertNotDeliveredWhenGroupWaitIsZero(t *testing.T) require.Len(t, recorder.Alerts(), numAlerts) } -type limits struct { - groups int -} - -func (l limits) MaxNumberOfAggregationGroups() int { - return l.groups -} - func TestDispatcher_DoMaintenance(t *testing.T) { r := prometheus.NewRegistry() marker := types.NewMarker(r) @@ -744,7 +485,7 @@ func TestDispatcher_DoMaintenance(t *testing.T) { t.Fatal(err) } - route := &Route{ + route := &route{ RouteOpts: RouteOpts{ GroupBy: map[model.LabelName]struct{}{"alertname": {}}, GroupWait: 0, @@ -756,7 +497,7 @@ func TestDispatcher_DoMaintenance(t *testing.T) { ctx := context.Background() dispatcher := NewDispatcher(alerts, route, recorder, marker, timeout, testMaintenanceInterval, nil, promslog.NewNopLogger(), NewDispatcherMetrics(false, r)) - aggrGroups := make(map[*Route]map[model.Fingerprint]*aggrGroup) + aggrGroups := make(map[Route]map[model.Fingerprint]*aggrGroup) aggrGroups[route] = make(map[model.Fingerprint]*aggrGroup) // Insert an aggregation group with no alerts. @@ -785,7 +526,7 @@ func TestDispatcher_DeleteResolvedAlertsFromMarker(t *testing.T) { ctx := context.Background() marker := types.NewMarker(prometheus.NewRegistry()) labels := model.LabelSet{"alertname": "TestAlert"} - route := &Route{ + route := &route{ RouteOpts: RouteOpts{ Receiver: "test", GroupBy: map[model.LabelName]struct{}{"alertname": {}}, @@ -854,7 +595,7 @@ func TestDispatcher_DeleteResolvedAlertsFromMarker(t *testing.T) { ctx := context.Background() marker := types.NewMarker(prometheus.NewRegistry()) labels := model.LabelSet{"alertname": "TestAlert"} - route := &Route{ + route := &route{ RouteOpts: RouteOpts{ Receiver: "test", GroupBy: map[model.LabelName]struct{}{"alertname": {}}, @@ -908,7 +649,7 @@ func TestDispatcher_DeleteResolvedAlertsFromMarker(t *testing.T) { ctx := context.Background() marker := types.NewMarker(prometheus.NewRegistry()) labels := model.LabelSet{"alertname": "TestAlert"} - route := &Route{ + route := &route{ RouteOpts: RouteOpts{ Receiver: "test", GroupBy: map[model.LabelName]struct{}{"alertname": {}}, diff --git a/dispatch/group.go b/dispatch/group.go new file mode 100644 index 0000000000..5ddc46209d --- /dev/null +++ b/dispatch/group.go @@ -0,0 +1,28 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dispatch + +import ( + "github.com/prometheus/common/model" + + "github.com/prometheus/alertmanager/types" +) + +// AggregationGroup interface is used to expose Aggregation Groups within the dispatcher. +type AggregationGroup interface { + Alerts() []*types.Alert + Labels() model.LabelSet + GroupKey() string + RouteID() string +} diff --git a/dispatch/route.go b/dispatch/route.go index e174672d3f..5f2ea303ca 100644 --- a/dispatch/route.go +++ b/dispatch/route.go @@ -1,4 +1,4 @@ -// Copyright 2015 Prometheus Team +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -38,30 +38,50 @@ var DefaultRouteOpts = RouteOpts{ MuteTimeIntervals: []string{}, } +type Route interface { + // Key returns a key for the route. It does not uniquely identify the route in general. + Key() string + // ID returns a unique identifier for the route. + ID() string + // Walk traverses the route tree in depth-first order. + Walk(func(Route)) + // Match does a depth-first left-to-right search through the route tree + // and returns the matching routing nodes. + Match(model.LabelSet) []Route + // Matchers returns the matchers for the route. + Matchers() labels.Matchers + // Options returns the options for the route. + Options() RouteOpts + // Continues returns true if the route will continue matching alerts. + Continues() bool + // Routes returns the child routes of the route. + Routes() []Route +} + // A Route is a node that contains definitions of how to handle alerts. -type Route struct { - parent *Route +type route struct { + parent Route // The configuration parameters for matches of this route. RouteOpts RouteOpts // Matchers an alert has to fulfill to match // this route. - Matchers labels.Matchers + matchers labels.Matchers // If true, an alert matches further routes on the same level. Continue bool // Children routes of this route. - Routes []*Route + routes []Route } // NewRoute returns a new route. -func NewRoute(cr *config.Route, parent *Route) *Route { +func NewRoute(cr *config.Route, parent Route) Route { // Create default and overwrite with configured settings. opts := DefaultRouteOpts if parent != nil { - opts = parent.RouteOpts + opts = parent.Options() } if cr.Receiver != "" { @@ -121,21 +141,21 @@ func NewRoute(cr *config.Route, parent *Route) *Route { opts.MuteTimeIntervals = cr.MuteTimeIntervals opts.ActiveTimeIntervals = cr.ActiveTimeIntervals - route := &Route{ + r := &route{ parent: parent, RouteOpts: opts, - Matchers: matchers, + matchers: matchers, Continue: cr.Continue, } - route.Routes = NewRoutes(cr.Routes, route) + r.routes = NewRoutes(cr.Routes, r) - return route + return r } // NewRoutes returns a slice of routes. -func NewRoutes(croutes []*config.Route, parent *Route) []*Route { - res := []*Route{} +func NewRoutes(croutes []*config.Route, parent Route) []Route { + res := []Route{} for _, cr := range croutes { res = append(res, NewRoute(cr, parent)) } @@ -144,19 +164,19 @@ func NewRoutes(croutes []*config.Route, parent *Route) []*Route { // Match does a depth-first left-to-right search through the route tree // and returns the matching routing nodes. -func (r *Route) Match(lset model.LabelSet) []*Route { - if !r.Matchers.Matches(lset) { +func (r *route) Match(lset model.LabelSet) []Route { + if !r.matchers.Matches(lset) { return nil } - var all []*Route + var all []Route - for _, cr := range r.Routes { + for _, cr := range r.routes { matches := cr.Match(lset) all = append(all, matches...) - if matches != nil && !cr.Continue { + if matches != nil && !cr.Continues() { break } } @@ -169,20 +189,24 @@ func (r *Route) Match(lset model.LabelSet) []*Route { return all } +func (r *route) Matchers() labels.Matchers { + return r.matchers +} + // Key returns a key for the route. It does not uniquely identify the route in general. -func (r *Route) Key() string { +func (r *route) Key() string { b := strings.Builder{} if r.parent != nil { b.WriteString(r.parent.Key()) b.WriteRune('/') } - b.WriteString(r.Matchers.String()) + b.WriteString(r.matchers.String()) return b.String() } // ID returns a unique identifier for the route. -func (r *Route) ID() string { +func (r *route) ID() string { b := strings.Builder{} if r.parent != nil { @@ -190,11 +214,11 @@ func (r *Route) ID() string { b.WriteRune('/') } - b.WriteString(r.Matchers.String()) + b.WriteString(r.matchers.String()) if r.parent != nil { - for i := range r.parent.Routes { - if r == r.parent.Routes[i] { + for i := range r.parent.Routes() { + if r == r.parent.Routes()[i] { b.WriteRune('/') b.WriteString(strconv.Itoa(i)) break @@ -206,13 +230,25 @@ func (r *Route) ID() string { } // Walk traverses the route tree in depth-first order. -func (r *Route) Walk(visit func(*Route)) { +func (r *route) Walk(visit func(Route)) { visit(r) - for i := range r.Routes { - r.Routes[i].Walk(visit) + for i := range r.Routes() { + r.Routes()[i].Walk(visit) } } +func (r *route) Options() RouteOpts { + return r.RouteOpts +} + +func (r *route) Routes() []Route { + return r.routes +} + +func (r *route) Continues() bool { + return r.Continue +} + // RouteOpts holds various routing options necessary for processing alerts // that match a given route. type RouteOpts struct { diff --git a/dispatch/route_test.go b/dispatch/route_test.go index 6a9d7d4588..b7af8faebf 100644 --- a/dispatch/route_test.go +++ b/dispatch/route_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 Prometheus Team +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -254,7 +254,8 @@ routes: var keys []string for _, r := range tree.Match(test.input) { - matches = append(matches, &r.RouteOpts) + options := r.Options() + matches = append(matches, &options) keys = append(keys, r.Key()) } @@ -344,8 +345,8 @@ routes: } var got []string - tree.Walk(func(r *Route) { - got = append(got, r.RouteOpts.Receiver) + tree.Walk(func(r Route) { + got = append(got, r.Options().Receiver) }) if !reflect.DeepEqual(got, expected) { @@ -375,12 +376,12 @@ routes: } tree := NewRoute(&ctree, nil) - parent := tree.Routes[0] - child1 := parent.Routes[0] - child2 := parent.Routes[1] - require.True(t, parent.RouteOpts.GroupByAll) - require.True(t, child1.RouteOpts.GroupByAll) - require.False(t, child2.RouteOpts.GroupByAll) + parent := tree.Routes()[0] + child1 := parent.Routes()[0] + child2 := parent.Routes()[1] + require.True(t, parent.Options().GroupByAll) + require.True(t, child1.Options().GroupByAll) + require.False(t, child2.Options().GroupByAll) } func TestRouteMatchers(t *testing.T) { @@ -604,7 +605,8 @@ routes: var keys []string for _, r := range tree.Match(test.input) { - matches = append(matches, &r.RouteOpts) + options := r.Options() + matches = append(matches, &options) keys = append(keys, r.Key()) } @@ -840,7 +842,8 @@ routes: var keys []string for _, r := range tree.Match(test.input) { - matches = append(matches, &r.RouteOpts) + options := r.Options() + matches = append(matches, &options) keys = append(keys, r.Key()) } @@ -915,7 +918,7 @@ routes: } var actual []string - r.Walk(func(r *Route) { + r.Walk(func(r Route) { actual = append(actual, r.ID()) }) require.ElementsMatch(t, actual, expected)