Initial version of CEL resource expressions.

CEL for Receiver notification filtering

This introduces CEL for filtering CEL resources in a Receiver.

Users can define a CEL expression that is applied as a filter for
resources that are identified for notification.

A CEL expression that returns false means that a resource will not be
annotated.

Signed-off-by: Kevin McDermott <[email protected]>

Author: bigkevmcd

api/v1/receiver_types.go

@@ -67,6 +67,11 @@ type ReceiverSpec struct {
 	// +required
 	Resources []CrossNamespaceObjectReference `json:"resources"`
 
+	// ResourceFilter is an expression that is applied to each Resource
+	// referenced in the Resources. If the expression returns false then the
+	// Resource is discarded and will not be notified.
+	ResourceFilter string `json:"resourceFilter,omitempty"`
+
 	// SecretRef specifies the Secret containing the token used
 	// to validate the payload authenticity.
 	// +required

config/crd/bases/notification.toolkit.fluxcd.io_receivers.yaml

@@ -62,6 +62,12 @@ spec:
                   Secret references.
                 pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$
                 type: string
+              resourceFilter:
+                description: |-
+                  ResourceFilter is an expression that is applied to each Resource
+                  referenced in the Resources. If the expression returns false then the
+                  Resource is discarded and will not be notified.
+                type: string
               resources:
                 description: A list of resources to be notified about changes.
                 items:

docs/api/v1/notification.md

@@ -122,6 +122,19 @@ e.g. &lsquo;push&rsquo; for GitHub or &lsquo;Push Hook&rsquo; for GitLab.</p>
 </tr>
 <tr>
 <td>
+<code>resourceFilter</code><br>
+<em>
+string
+</em>
+</td>
+<td>
+<p>ResourceFilter is an expression that is applied to each Resource
+referenced in the Resources. If the expression returns false then the
+Resource is discarded and will not be notified.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>secretRef</code><br>
 <em>
 <a href="https://pkg.go.dev/github.com/fluxcd/pkg/apis/meta#LocalObjectReference">
@@ -321,6 +334,19 @@ e.g. &lsquo;push&rsquo; for GitHub or &lsquo;Push Hook&rsquo; for GitLab.</p>
 </tr>
 <tr>
 <td>
+<code>resourceFilter</code><br>
+<em>
+string
+</em>
+</td>
+<td>
+<p>ResourceFilter is an expression that is applied to each Resource
+referenced in the Resources. If the expression returns false then the
+Resource is discarded and will not be notified.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>secretRef</code><br>
 <em>
 <a href="https://pkg.go.dev/github.com/fluxcd/pkg/apis/meta#LocalObjectReference">

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/fluxcd/pkg/ssa v0.41.1
 	github.com/getsentry/sentry-go v0.29.0
 	github.com/go-logr/logr v1.4.2
+	github.com/google/cel-go v0.20.1
 	github.com/google/go-github/v64 v64.0.0
 	github.com/hashicorp/go-retryablehttp v0.7.7
 	github.com/ktrysmt/go-bitbucket v0.9.80
@@ -74,6 +75,7 @@ require (
 	github.com/DataDog/zstd v1.5.2 // indirect
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/ProtonMail/go-crypto v1.0.0 // indirect
+	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
@@ -159,6 +161,7 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 // indirect
 	github.com/spf13/cobra v1.8.1 // indirect
+	github.com/stoewer/go-strcase v1.2.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect
 	go.opencensus.io v0.24.0 // indirect

go.sum

@@ -73,6 +73,8 @@ github.com/PagerDuty/go-pagerduty v1.8.0 h1:MTFqTffIcAervB83U7Bx6HERzLbyaSPL/+ox
 github.com/PagerDuty/go-pagerduty v1.8.0/go.mod h1:nzIeAqyFSJAFkjWKvMzug0JtwDg+V+UoCWjFrfFH5mI=
 github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78=
 github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
+github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
+github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -215,6 +217,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
 github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
+github.com/google/cel-go v0.20.1 h1:nDx9r8S3L4pE61eDdt8igGj8rf5kjYR3ILxWIpWNi84=
+github.com/google/cel-go v0.20.1/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg=
 github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
 github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -381,12 +385,15 @@ github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
 github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU=
+github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -591,6 +598,7 @@ gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSP
 gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

internal/server/cel.go

@@ -0,0 +1,172 @@
+package server
+
+import (
+	"encoding/json"
+	"fmt"
+	"mime"
+	"net/http"
+	"strings"
+
+	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/checker/decls"
+	"github.com/google/cel-go/common/types"
+	"github.com/google/cel-go/common/types/ref"
+	"github.com/google/cel-go/common/types/traits"
+	celext "github.com/google/cel-go/ext"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func newCELEvaluator(expr string, req *http.Request) (resourcePredicate, error) {
+	env, err := makeCELEnv()
+	if err != nil {
+		return nil, err
+	}
+	parsed, issues := env.Parse(expr)
+	if issues != nil && issues.Err() != nil {
+		return nil, fmt.Errorf("failed to parse expression %v: %w", expr, issues.Err())
+	}
+
+	checked, issues := env.Check(parsed)
+	if issues != nil && issues.Err() != nil {
+		return nil, fmt.Errorf("expression %v check failed: %w", expr, issues.Err())
+	}
+
+	prg, err := env.Program(checked, cel.EvalOptions(cel.OptOptimize))
+	if err != nil {
+		return nil, fmt.Errorf("expression %v failed to create a Program: %w", expr, err)
+	}
+
+	body := map[string]any{}
+	// Only decodes the body for the expression if the body is JSON.
+	// Technically you could generate several resources without any body.
+	if isJSONContent(req) {
+		if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
+			return nil, fmt.Errorf("failed to parse request body as JSON: %s", err)
+		}
+	}
+
+	return func(obj client.Object) (*bool, error) {
+		data, err := clientObjectToMap(obj)
+		if err != nil {
+			return nil, err
+		}
+
+		out, _, err := prg.Eval(map[string]any{
+			"resource": data,
+			"request":  body,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("expression %v failed to evaluate: %w", expr, err)
+		}
+
+		v, ok := out.(types.Bool)
+		if !ok {
+			return nil, fmt.Errorf("expression %q did not return a boolean value", expr)
+		}
+
+		result := v.Value().(bool)
+
+		return &result, nil
+	}, nil
+}
+
+func makeCELEnv() (*cel.Env, error) {
+	mapStrDyn := decls.NewMapType(decls.String, decls.Dyn)
+	return cel.NewEnv(
+		celext.Strings(),
+		celext.Encoders(),
+		notifications(),
+		cel.Declarations(
+			decls.NewVar("resource", mapStrDyn),
+			decls.NewVar("request", mapStrDyn),
+		))
+}
+
+func isJSONContent(r *http.Request) bool {
+	contentType := r.Header.Get("Content-type")
+	for _, v := range strings.Split(contentType, ",") {
+		t, _, err := mime.ParseMediaType(v)
+		if err != nil {
+			break
+		}
+		if t == "application/json" {
+			return true
+		}
+	}
+
+	return false
+}
+
+func notifications() cel.EnvOption {
+	r, err := types.NewRegistry()
+	if err != nil {
+		panic(err) // TODO: Do something better?
+	}
+
+	return cel.Lib(&notificationsLib{registry: r})
+}
+
+type notificationsLib struct {
+	registry *types.Registry
+}
+
+// LibraryName implements the SingletonLibrary interface method.
+func (*notificationsLib) LibraryName() string {
+	return "flux.notifications.lib"
+}
+
+// CompileOptions implements the Library interface method.
+func (l *notificationsLib) CompileOptions() []cel.EnvOption {
+	listStrDyn := cel.ListType(cel.DynType)
+	opts := []cel.EnvOption{
+		cel.Function("first",
+			cel.MemberOverload("first_list", []*cel.Type{listStrDyn}, cel.DynType,
+				cel.UnaryBinding(listFirst))),
+		cel.Function("last",
+			cel.MemberOverload("last_list", []*cel.Type{listStrDyn}, cel.DynType,
+				cel.UnaryBinding(listLast))),
+	}
+
+	return opts
+}
+
+// ProgramOptions implements the Library interface method.
+func (*notificationsLib) ProgramOptions() []cel.ProgramOption {
+	return []cel.ProgramOption{}
+}
+
+func listLast(val ref.Val) ref.Val {
+	l := val.(traits.Lister)
+	sz := l.Size().Value().(int64)
+
+	if sz == 0 {
+		return types.NullValue
+	}
+
+	return l.Get(types.Int(sz - 1))
+}
+
+func listFirst(val ref.Val) ref.Val {
+	l := val.(traits.Lister)
+	sz := l.Size().Value().(int64)
+
+	if sz == 0 {
+		return types.NullValue
+	}
+
+	return l.Get(types.Int(0))
+}
+
+func clientObjectToMap(v client.Object) (map[string]any, error) {
+	b, err := json.Marshal(v)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal PartialObjectMetadata from resource for CEL expression: %w", err)
+	}
+
+	var result map[string]any
+	if err := json.Unmarshal(b, &result); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal PartialObjectMetadata from resource for CEL expression: %w", err)
+	}
+
+	return result, nil
+}