Merge pull request #45 from integr8ly/add-blackbox-exporter

Add blackbox exporter

Author: david-martin

README.md

@@ -14,6 +14,26 @@ The following resources are supported:
 
 Triggers the installation of the monitoring stack when created. This is achieved by deploying two other operators that install Prometheus and Grafana respectively.
 
+The Application Monitoring CR accepts the following properties in the spec:
+
+* *labelSelector*: The value of the `middleware-monitoring` label that has to be present on all imported resources (prometheus rules, service monitors, grafana dashboards).
+* *blackboxTargets*: A list of targets for the blackbox exporter to probe.
+
+The `blackboxTargets` should be provided as an array in the form of:
+
+```yaml
+  blackboxTargets:
+    - service: example
+      url: https://example.com
+      module: http_extern_2xx
+```
+
+where `service` will be added as a label to the metric, `url` is the URL of the route to probe and `module` can be one of:
+
+* *http_2xx*: Probe http or https targets via GET using the cluster certificates
+* *http_post_2xx*: Probe http or https targets via POST using the cluster certificates
+* *http_extern_2xx*: Probe http or https targets via GET relying on a valid external certificate
+
 ## PrometheusRule
 
 Represents a set of alert rules for Prometheus/Alertmanager. See the [https://github.com/coreos/prometheus-operator/blob/f9bc0aa0fd9aa936f500d9d241098863c60d873d/Documentation/user-guides/alerting.md#alerting](prometheus operator docs) for more details about this resource.

deploy/crds/ApplicationMonitoring.yaml

@@ -18,6 +18,11 @@ spec:
         spec:
           required: ["labelSelector"]
           properties:
+            blackboxTargets:
+              type: array
+              items:
+                type: object
+                description: A target to be probed by the blackbox exporter
             labelSelector:
               type: string
               minimum: 1

deploy/examples/ApplicationMonitoring.yaml

@@ -4,3 +4,7 @@ metadata:
   name: example-applicationmonitoring
 spec:
   labelSelector: "middleware"
+  blackboxTargets:
+    - service: example
+      url: https://example.com
+      module: http_extern_2xx

pkg/apis/applicationmonitoring/v1alpha1/applicationmonitoring_types.go

@@ -7,11 +7,20 @@ import (
 // EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 
+// A target (url, module and service name) to be probed by the
+// blackbox exporter
+type BlackboxTarget struct {
+	Url     string `json:"url"`
+	Service string `json:"service"`
+	Module  string `json:"module"`
+}
+
 // ApplicationMonitoringSpec defines the desired state of ApplicationMonitoring
 type ApplicationMonitoringSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "operator-sdk generate k8s" to regenerate code after modifying this file
-	LabelSelector string `json:"labelSelector"`
+	LabelSelector   string           `json:"labelSelector"`
+	BlackboxTargets []BlackboxTarget `json:"blackboxTargets,omitempty"`
 }
 
 // ApplicationMonitoringStatus defines the observed state of ApplicationMonitoring

pkg/apis/applicationmonitoring/v1alpha1/zz_generated.deepcopy.go

@@ -39,6 +39,7 @@ const (
 	PhaseCreateGrafanaCR
 	PhaseFinalizer
 	PhaseDone
+	PhaseReconcileConfig
 )
 
 // Add creates a new ApplicationMonitoring Controller and adds it to the Manager. The Manager will set fields on the Controller
@@ -144,15 +145,29 @@ func (r *ReconcileApplicationMonitoring) Reconcile(request reconcile.Request) (r
 		return r.setFinalizer(instanceCopy)
 	case PhaseDone:
 		log.Info("Finished installing application monitoring")
+		return r.updatePhase(instanceCopy, PhaseReconcileConfig)
+	case PhaseReconcileConfig:
+		return r.reconcileConfig(instanceCopy)
 	}
 
 	return reconcile.Result{}, nil
 }
 
+func (r *ReconcileApplicationMonitoring) reconcileConfig(cr *applicationmonitoringv1alpha1.ApplicationMonitoring) (reconcile.Result, error) {
+	log.Info("Phase: Reconciling Config")
+	return reconcile.Result{}, r.createOrUpdateAdditionalScrapeConfig(cr)
+}
+
 func (r *ReconcileApplicationMonitoring) installPrometheusOperator(cr *applicationmonitoringv1alpha1.ApplicationMonitoring) (reconcile.Result, error) {
 	log.Info("Phase: Install PrometheusOperator")
 
-	for _, resourceName := range []string{PrometheusOperatorServiceAccountName, PrometheusOperatorName, PrometheusProxySecretsName} {
+	err := r.createOrUpdateAdditionalScrapeConfig(cr)
+	if err != nil {
+		log.Error(err, "Failed to create additional scrape config")
+		return reconcile.Result{}, err
+	}
+
+	for _, resourceName := range []string{PrometheusOperatorServiceAccountName, PrometheusOperatorName, PrometheusProxySecretsName, BlackboxExporterConfigmapName} {
 		if _, err := r.createResource(cr, resourceName); err != nil {
 			log.Info(fmt.Sprintf("Error in InstallPrometheusOperator, resourceName=%s : err=%s", resourceName, err))
 			// Requeue so it can be attempted again
@@ -340,6 +355,54 @@ func (r *ReconcileApplicationMonitoring) getHostFromRoute(namespacedName types.N
 	return host, nil
 }
 
+// Create a secret that contains additional prometheus scrape configurations
+// Used to configure the blackbox exporter
+func (r *ReconcileApplicationMonitoring) createOrUpdateAdditionalScrapeConfig(cr *applicationmonitoringv1alpha1.ApplicationMonitoring) error {
+	t := newTemplateHelper(cr, nil)
+	job, err := t.loadTemplate(BlackboxExporterJobName)
+
+	if err != nil {
+		return errors.Wrap(err, "error loading blackbox exporter template")
+	}
+
+	selector := types.NamespacedName{
+		Namespace: cr.Namespace,
+		Name:      ScrapeConfigSecretName,
+	}
+
+	update := true
+	scrapeConfigSecret := corev1.Secret{}
+
+	// Check if the secret already exists
+	err = r.client.Get(context.TODO(), selector, &scrapeConfigSecret)
+	if err != nil {
+		if !kerrors.IsNotFound(err) {
+			return err
+		}
+		update = false
+	}
+
+	scrapeConfigSecret.ObjectMeta = v1.ObjectMeta{
+		Name:      ScrapeConfigSecretName,
+		Namespace: cr.Namespace,
+	}
+
+	scrapeConfigSecret.Data = map[string][]byte{
+		"blackbox-exporter.yaml": []byte(job),
+	}
+
+	if update {
+		return r.client.Update(context.TODO(), &scrapeConfigSecret)
+	} else {
+		err = controllerutil.SetControllerReference(cr, &scrapeConfigSecret, r.scheme)
+		if err != nil {
+			return errors.Wrap(err, "error setting controller reference")
+		}
+
+		return r.client.Create(context.TODO(), &scrapeConfigSecret)
+	}
+}
+
 func (r *ReconcileApplicationMonitoring) getPrometheusOperatorReady(cr *applicationmonitoringv1alpha1.ApplicationMonitoring) (bool, error) {
 	resource := v1beta1.Deployment{}
 

pkg/controller/applicationmonitoring/applicationmonitoring_controller.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"strings"
 	"text/template"
 
 	applicationmonitoring "github.com/integr8ly/application-monitoring-operator/pkg/apis/applicationmonitoring/v1alpha1"
@@ -39,6 +40,9 @@ const (
 	AlertManagerRouteName                = "alertmanager-route"
 	GrafanaServiceMonitorName            = "grafana-servicemonitor"
 	GrafanaServiceName                   = "grafana-service"
+	BlackboxExporterConfigmapName        = "blackbox-exporter-config"
+	BlackboxExporterJobName              = "blackbox-exporter-job"
+	ScrapeConfigSecretName               = "additional-scrape-configs"
 )
 
 type Parameters struct {
@@ -66,6 +70,9 @@ type Parameters struct {
 	PrometheusServiceMonitorName   string
 	MonitoringKey                  string
 	GrafanaServiceName             string
+	BlackboxExporterConfigmapName  string
+	ScrapeConfigSecretName         string
+	BlackboxTargets                []applicationmonitoring.BlackboxTarget
 	ExtraParams                    map[string]string
 }
 
@@ -103,6 +110,9 @@ func newTemplateHelper(cr *applicationmonitoring.ApplicationMonitoring, extraPar
 		GrafanaServiceMonitorName:      GrafanaServiceMonitorName,
 		PrometheusServiceMonitorName:   PrometheusServiceMonitorName,
 		MonitoringKey:                  cr.Spec.LabelSelector,
+		BlackboxExporterConfigmapName:  BlackboxExporterConfigmapName,
+		BlackboxTargets:                cr.Spec.BlackboxTargets,
+		ScrapeConfigSecretName:         ScrapeConfigSecretName,
 		ExtraParams:                    extraParams,
 	}
 
@@ -131,6 +141,16 @@ func PopulateSessionProxySecret() string {
 	return p
 }
 
+// Takes a list of strings, wraps each string in double quotes and joins them
+// Used for building yaml arrays
+func joinQuote(values []applicationmonitoring.BlackboxTarget) string {
+	var result []string
+	for _, s := range values {
+		result = append(result, fmt.Sprintf("\"%v@%v@%v\"", s.Module, s.Service, s.Url))
+	}
+	return strings.Join(result, ", ")
+}
+
 // load a templates from a given resource name. The templates must be located
 // under ./templates and the filename must be <resource-name>.yaml
 func (h *TemplateHelper) loadTemplate(name string) ([]byte, error) {
@@ -140,7 +160,12 @@ func (h *TemplateHelper) loadTemplate(name string) ([]byte, error) {
 		return nil, err
 	}
 
-	parsed, err := template.New("application-monitoring").Parse(string(tpl))
+	parser := template.New("application-monitoring")
+	parser.Funcs(template.FuncMap{
+		"JoinQuote": joinQuote,
+	})
+
+	parsed, err := parser.Parse(string(tpl))
 	if err != nil {
 		return nil, err
 	}

pkg/controller/applicationmonitoring/templateHelper.go

@@ -7,7 +7,7 @@ spec:
   baseImage: registry.redhat.io/openshift3/prometheus-alertmanager
   tag: v3.11
   externalUrl: https://{{ index .ExtraParams "alertmanagerHost" }}
-  listenLocal: false
+  listenLocal: true
   serviceAccountName: alertmanager
   containers:
     - args:

templates/alertmanager.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+data:
+  blackbox.yml: |-
+    modules:
+      http_extern_2xx:
+        prober: http
+        http:
+          preferred_ip_protocol: ip4
+      http_2xx:
+        prober: http
+        http:
+          preferred_ip_protocol: ip4
+          tls_config:
+            ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+            cert_file: /etc/tls/private/tls.crt
+            key_file: /etc/tls/private/tls.key
+      http_post_2xx:
+        prober: http
+        http:
+          method: POST
+          preferred_ip_protocol: ip4
+          tls_config:
+            ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+            cert_file: /etc/tls/private/tls.crt
+            key_file: /etc/tls/private/tls.key
+kind: ConfigMap
+metadata:
+  name: {{ .BlackboxExporterConfigmapName }}
+  namespace: {{ .Namespace }}

templates/blackbox-exporter-config.yaml

@@ -0,0 +1,21 @@
+- job_name: 'blackbox'
+  metrics_path: /probe
+  static_configs:
+    - targets: [{{ JoinQuote .BlackboxTargets }}]
+  relabel_configs:
+    - source_labels: [__address__]
+      regex: (.*)\@(.*)\@(.*)
+      target_label: __param_module
+      replacement: ${1}
+    - source_labels: [__address__]
+      regex: (.*)\@(.*)\@(.*)
+      target_label: service
+      replacement: ${2}
+    - source_labels: [__address__]
+      regex: (.*)\@(.*)\@(.*)
+      target_label: __param_target
+      replacement: ${3}
+    - source_labels: [__param_target]
+      target_label: instance
+    - target_label: __address__
+      replacement: 127.0.0.1:9115