Merge pull request #36 from thomasjpfan/issue-35

ENH: Adds alert persistent feature

Author: vfarcic

docs/usage.md

@@ -37,6 +37,7 @@ You can find more about scrapeType's on [Scrape Config](https://prometheus.io/do
 |alertLabels    |This parameter is translated to Prometheus alert `LABELS` statement. It allows specifying a set of additional labels to be attached to the alert. Multiple labels can be separated with comma (`,`).<br>**Example:** `severity=high,receiver=system`|No|
 |alertName      |The name of the alert. It is combined with the `serviceName` thus producing an unique identifier.<br>**Example:** `memoryAlert`|Yes|
 |serviceName    |The name of the service. It is combined with the `alertName` thus producing an unique identifier.<br>**Example:** `go-demo`|Yes|
+|alertPersistent|When set to *true*, the alert will persist when the service is scaled to zero replicas.<br>**Example:** `true`|No|
 
 Those parameters can be indexed so that multiple alerts can be defined for a service. Indexing is sequential and starts from 1. An example of indexed `alertName` could be `alertName.1=memload` and `alertName.2=diskload`.
 

prometheus/types.go

@@ -222,6 +222,7 @@ type Alert struct {
 	AlertIf            string            `json:"alertIf,omitempty"`
 	AlertLabels        map[string]string `json:"alertLabels,omitempty"`
 	AlertName          string            `json:"alertName"`
+	AlertPersistent    bool              `json:"alertPersistent"`
 	AlertNameFormatted string
 	ServiceName        string `json:"serviceName"`
 	Replicas           int    `json:"replicas"`

server/server.go

@@ -97,7 +97,7 @@ func (s *serve) ReconfigureHandler(w http.ResponseWriter, req *http.Request) {
 	logPrintf("Processing " + req.URL.String())
 	req.ParseForm()
 	scrape := s.getScrape(req)
-	s.deleteAlerts(scrape.ServiceName)
+	s.deleteAlerts(scrape.ServiceName, false)
 	alerts := s.getAlerts(req)
 	prometheus.WriteConfig(s.configPath, s.scrapes, s.alerts)
 	err := prometheus.Reload()
@@ -115,7 +115,7 @@ func (s *serve) RemoveHandler(w http.ResponseWriter, req *http.Request) {
 	serviceName := req.URL.Query().Get("serviceName")
 	scrape := s.scrapes[serviceName]
 	delete(s.scrapes, serviceName)
-	alerts := s.deleteAlerts(serviceName)
+	alerts := s.deleteAlerts(serviceName, true)
 	prometheus.WriteConfig(s.configPath, s.scrapes, s.alerts)
 	err := prometheus.Reload()
 	statusCode := http.StatusOK
@@ -241,6 +241,8 @@ func (s *serve) getAlerts(req *http.Request) []prometheus.Alert {
 		alertName := req.URL.Query().Get(fmt.Sprintf("alertName.%d", i))
 		annotations := s.getMapFromString(req.URL.Query().Get(fmt.Sprintf("alertAnnotations.%d", i)))
 		labels := s.getMapFromString(req.URL.Query().Get(fmt.Sprintf("alertLabels.%d", i)))
+		persistant := req.URL.Query().Get(fmt.Sprintf("alertPersistent.%d", i)) == "true"
+
 		alert := prometheus.Alert{
 			ServiceName:      alertDecode.ServiceName,
 			AlertName:        alertName,
@@ -249,6 +251,7 @@ func (s *serve) getAlerts(req *http.Request) []prometheus.Alert {
 			AlertAnnotations: annotations,
 			AlertLabels:      labels,
 			Replicas:         replicas,
+			AlertPersistent:  persistant,
 		}
 		s.formatAlert(&alert)
 		if !s.isValidAlert(&alert) {
@@ -322,7 +325,6 @@ func GetShortcuts() map[string]AlertIfShortcut {
 			logPrintf("YAML decoding reading %s, error: %v", path, err)
 			continue
 		}
-		fmt.Println(secretShortcuts)
 
 		for k, v := range secretShortcuts {
 			shortcuts[k] = v
@@ -483,13 +485,16 @@ func (s *serve) isValidAlert(alert *prometheus.Alert) bool {
 	return len(alert.AlertName) > 0 && len(alert.AlertIf) > 0
 }
 
-func (s *serve) deleteAlerts(serviceName string) []prometheus.Alert {
+func (s *serve) deleteAlerts(
+	serviceName string, keepPersistantAlerts bool) []prometheus.Alert {
 	alerts := []prometheus.Alert{}
 	serviceNameFormatted := s.getNameFormatted(serviceName)
 	for k, v := range s.alerts {
 		if strings.HasPrefix(k, serviceNameFormatted) {
-			alerts = append(alerts, v)
-			delete(s.alerts, k)
+			if !keepPersistantAlerts || !v.AlertPersistent {
+				alerts = append(alerts, v)
+				delete(s.alerts, k)
+			}
 		}
 	}
 	return alerts

server/server_test.go

@@ -231,6 +231,57 @@ func (s *ServerTestSuite) Test_ReconfigureHandler_AddsAlert() {
 	s.Equal(expected, serve.alerts[expected.AlertNameFormatted])
 }
 
+func (s *ServerTestSuite) Test_ReconfigureHandler_UpdatesPersistantAlert() {
+	// When service is reconfigured, all alert, including persistant alerts, will be removed
+	// and queried again.
+	expected := prometheus.Alert{
+		ServiceName:        "my-service",
+		AlertName:          "my-alert",
+		AlertIf:            "a>b",
+		AlertFor:           "my-for",
+		AlertNameFormatted: "myservice_myalert",
+		AlertAnnotations:   map[string]string{"a1": "v1", "a2": "v2"},
+		AlertLabels:        map[string]string{"l1": "v1"},
+		AlertPersistent:    true,
+	}
+	rwMock := ResponseWriterMock{}
+	addr := fmt.Sprintf(
+		"/v1/docker-flow-monitor?serviceName=%s&alertName=%s&alertIf=%s&alertFor=%s&alertAnnotations=%s&alertLabels=%s&alertPersistent=%t",
+		expected.ServiceName,
+		expected.AlertName,
+		url.QueryEscape(expected.AlertIf),
+		expected.AlertFor,
+		url.QueryEscape("a1=v1,a2=v2"),
+		url.QueryEscape("l1=v1"),
+		expected.AlertPersistent,
+	)
+	req, _ := http.NewRequest("GET", addr, nil)
+
+	serve := New()
+	serve.ReconfigureHandler(rwMock, req)
+
+	s.Equal(expected, serve.alerts[expected.AlertNameFormatted])
+
+	// Change alert slightly
+	expected.AlertIf = "a<b"
+	rwMock = ResponseWriterMock{}
+	addr = fmt.Sprintf(
+		"/v1/docker-flow-monitor?serviceName=%s&alertName=%s&alertIf=%s&alertFor=%s&alertAnnotations=%s&alertLabels=%s&alertPersistent=%t",
+		expected.ServiceName,
+		expected.AlertName,
+		url.QueryEscape(expected.AlertIf),
+		expected.AlertFor,
+		url.QueryEscape("a1=v1,a2=v2"),
+		url.QueryEscape("l1=v1"),
+		expected.AlertPersistent,
+	)
+	req, _ = http.NewRequest("GET", addr, nil)
+
+	serve.ReconfigureHandler(rwMock, req)
+
+	s.Equal(expected, serve.alerts[expected.AlertNameFormatted])
+}
+
 func (s *ServerTestSuite) Test_ReconfigureHandler_ExpandsShortcuts() {
 	testData := []struct {
 		expected    string
@@ -604,11 +655,12 @@ func (s *ServerTestSuite) Test_ReconfigureHandler_AddsMultipleAlerts() {
 			AlertAnnotations:   map[string]string{"annotation": fmt.Sprintf("annotation-value-%d", i)},
 			AlertLabels:        map[string]string{"label": fmt.Sprintf("label-value-%d", i)},
 			Replicas:           3,
+			AlertPersistent:    i == 2,
 		})
 	}
 	rwMock := ResponseWriterMock{}
 	addr := fmt.Sprintf(
-		"/v1/docker-flow-monitor?serviceName=%s&alertName.1=%s&alertIf.1=%s&alertFor.1=%s&alertName.2=%s&alertIf.2=%s&alertFor.2=%s&alertAnnotations.1=%s&alertAnnotations.2=%s&alertLabels.1=%s&alertLabels.2=%s&replicas=3",
+		"/v1/docker-flow-monitor?serviceName=%s&alertName.1=%s&alertIf.1=%s&alertFor.1=%s&alertName.2=%s&alertIf.2=%s&alertFor.2=%s&alertAnnotations.1=%s&alertAnnotations.2=%s&alertLabels.1=%s&alertLabels.2=%s&replicas=3&alertPersistent.2=true",
 		expected[0].ServiceName,
 		expected[0].AlertName,
 		expected[0].AlertIf,
@@ -1123,6 +1175,21 @@ func (s *ServerTestSuite) Test_RemoveHandler_RemovesAlerts() {
 	s.Len(serve.alerts, 1)
 }
 
+func (s *ServerTestSuite) Test_RemoveHandler_KeepsPersistantAlerts() {
+
+	rwMock := ResponseWriterMock{}
+	addr := "/v1/docker-flow-monitor?serviceName=my-service-1"
+	req, _ := http.NewRequest("DELETE", addr, nil)
+
+	serve := New()
+	serve.alerts["myservice1alert1"] = prometheus.Alert{ServiceName: "my-service-1", AlertName: "my-alert-1", AlertPersistent: true}
+	serve.alerts["myservice1alert2"] = prometheus.Alert{ServiceName: "my-service-1", AlertName: "my-alert-2"}
+	serve.alerts["myservice2alert1"] = prometheus.Alert{ServiceName: "my-service-2", AlertName: "my-alert-1"}
+	serve.RemoveHandler(rwMock, req)
+
+	s.Len(serve.alerts, 2)
+}
+
 func (s *ServerTestSuite) Test_RemoveHandler_ReturnsJson() {
 	reloadOrig := prometheus.Reload
 	defer func() { prometheus.Reload = reloadOrig }()