Add ClickhouseBackup Plugin Support

cmd/mockserver/main.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net"
+	"time"
+
+	pb "github.com/altinity/clickhouse-operator/pkg/plugin/backup"
+
+	"google.golang.org/grpc"
+)
+
+// BackupServer implements pb.BackupServer
+type BackupServer struct {
+	pb.UnimplementedBackupServer
+}
+
+// Backup handles incoming BackupRequest calls
+func (s *BackupServer) Backup(ctx context.Context, req *pb.BackupRequest) (*pb.BackupResult, error) {
+	// Log what we received
+	log.Printf("Received Backup request: chi=%d bytes, backup=%d bytes, params=%v",
+		len(req.ChiDefinition), len(req.BackupDefinition), req.Parameters)
+
+	// Mock result
+	start := time.Now().Unix()
+	time.Sleep(2 * time.Second) // simulate work
+	stop := time.Now().Unix()
+
+	result := &pb.BackupResult{
+		BackupId:   "mock-backup-123",
+		BackupName: "example-backup",
+		StartedAt:  start,
+		StoppedAt:  stop,
+		Metadata: map[string]string{
+			"status": "success",
+			"note":   "this is a mock backup",
+		},
+	}
+
+	return result, nil
+}
+
+func main() {
+	lis, err := net.Listen("tcp", ":50051")
+	if err != nil {
+		log.Fatalf("failed to listen: %v", err)
+	}
+
+	grpcServer := grpc.NewServer()
+	pb.RegisterBackupServer(grpcServer, &BackupServer{})
+
+	fmt.Println("Mock Backup gRPC server running on :50051")
+	if err := grpcServer.Serve(lis); err != nil {
+		log.Fatalf("failed to serve: %v", err)
+	}
+}

cmd/operator/app/main.go

@@ -77,6 +77,7 @@ func Run() {
 	launchClickHouse(ctx, &wg)
 	launchClickHouseReconcilerMetricsExporter(ctx, &wg)
 	launchKeeper(ctx, &wg)
+	launchBackup(ctx, &wg)
 
 	// Wait for completion
 	<-ctx.Done()
@@ -120,6 +121,25 @@ func launchKeeper(ctx context.Context, wg *sync.WaitGroup) {
 	}()
 }
 
+func launchBackup(ctx context.Context, wg *sync.WaitGroup) {
+	backupErr := initBackup(ctx)
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		if backupErr == nil {
+			log.Info("Starting backup")
+			backupErr = runBackup(ctx)
+			if backupErr == nil {
+				log.Info("Starting backup OK")
+			} else {
+				log.Warning("Starting backup FAILED with err: %v", backupErr)
+			}
+		} else {
+			log.Warning("Starting backup skipped due to failed initialization with err: %v", backupErr)
+		}
+	}()
+}
+
 // setupSignalsNotification sets up OS signals
 func setupSignalsNotification(cancel context.CancelFunc) {
 	stopChan := make(chan os.Signal, 2)

cmd/operator/app/thread_backup.go

@@ -0,0 +1,100 @@
+package app
+
+import (
+	"context"
+
+	api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse-backup.altinity.com/v1"
+	"github.com/altinity/clickhouse-operator/pkg/chop"
+	controller "github.com/altinity/clickhouse-operator/pkg/controller/chb"
+	apiMachineryRuntime "k8s.io/apimachinery/pkg/runtime"
+	clientGoScheme "k8s.io/client-go/kubernetes/scheme"
+	ctrl "sigs.k8s.io/controller-runtime"
+	ctrlRuntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+)
+
+func initBackup(ctx context.Context) error {
+	var err error
+
+	ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
+
+	logger = ctrl.Log.WithName("backup-runner")
+
+	scheme = apiMachineryRuntime.NewScheme()
+	if err = clientGoScheme.AddToScheme(scheme); err != nil {
+		logger.Error(err, "init backup - unable to clientGoScheme.AddToScheme")
+		return err
+	}
+	if err = api.AddToScheme(scheme); err != nil {
+		logger.Error(err, "init backup - unable to api.AddToScheme")
+		return err
+	}
+
+	manager, err = ctrlRuntime.NewManager(ctrlRuntime.GetConfigOrDie(), ctrlRuntime.Options{
+		Scheme: scheme,
+		Cache: cache.Options{
+			Namespaces: []string{chop.Config().GetInformerNamespace()},
+		},
+		MetricsBindAddress:     "0",
+		HealthProbeBindAddress: "0",
+	})
+	if err != nil {
+		logger.Error(err, "init backup - unable to ctrlRuntime.NewManager")
+		return err
+	}
+
+	err = ctrlRuntime.
+		NewControllerManagedBy(manager).
+		For(&api.ClickHouseBackup{}, builder.WithPredicates(backupPredicate())).
+		Complete(
+			&controller.Controller{
+				Client: manager.GetClient(),
+				Scheme: manager.GetScheme(),
+			},
+		)
+	if err != nil {
+		logger.Error(err, "init backup - unable to ctrlRuntime.NewControllerManagedBy")
+		return err
+	}
+
+	// Initialization successful
+	return nil
+}
+
+func runBackup(ctx context.Context) error {
+	if err := manager.Start(ctx); err != nil {
+		logger.Error(err, "run backup - unable to manager.Start")
+		return err
+	}
+	// Run successful
+	return nil
+}
+
+func backupPredicate() predicate.Funcs {
+	return predicate.Funcs{
+		CreateFunc: func(e event.CreateEvent) bool {
+			chb, ok := e.Object.(*api.ClickHouseBackup)
+			if !ok {
+				return false
+			}
+			if chb.Status.State == "Completed" {
+				return false
+			}
+
+			return true
+		},
+		DeleteFunc: func(e event.DeleteEvent) bool {
+			return false
+		},
+		UpdateFunc: func(e event.UpdateEvent) bool {
+			return false
+		},
+		GenericFunc: func(e event.GenericEvent) bool {
+			return true
+		},
+	}
+}

deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousebackup.clickhouse-backup.altinity.com.yaml

@@ -0,0 +1,133 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.18.0
+  name: clickhousebackups.clickhouse-backup.altinity.com
+spec:
+  group: clickhouse-backup.altinity.com
+  names:
+    kind: ClickHouseBackup
+    listKind: ClickHouseBackupList
+    plural: clickhousebackups
+    shortNames:
+      - chb
+    singular: clickhousebackup
+  scope: Namespaced
+  versions:
+    - additionalPrinterColumns:
+        - jsonPath: .status.state
+          name: Status
+          type: string
+        - jsonPath: .status.message
+          name: Message
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1
+      schema:
+        openAPIV3Schema:
+          properties:
+            apiVersion:
+              description: |-
+                APIVersion defines the versioned schema of this representation of an object.
+                Servers should convert recognized schemas to the latest internal value, and
+                may reject unrecognized values.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+              type: string
+            kind:
+              description: |-
+                Kind is a string value representing the REST resource this object represents.
+                Servers may infer this from the endpoint the client submits requests to.
+                Cannot be updated.
+                In CamelCase.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+              type: string
+            metadata:
+              type: object
+            spec:
+              properties:
+                ClickhouseInstallation:
+                  description: ClickhouseInstallation represents the CHI (ClickHouseInstallation)
+                    that a backup is tied to
+                  properties:
+                    cluster:
+                      description: ClickhouseCluster represents a single cluster inside
+                        a CHI
+                      properties:
+                        name:
+                          type: string
+                      type: object
+                    name:
+                      type: string
+                  type: object
+                backup:
+                  properties:
+                    dbTable:
+                      properties:
+                        blackList:
+                          items:
+                            type: string
+                          type: array
+                        whiteList:
+                          items:
+                            type: string
+                          type: array
+                      type: object
+                    s3:
+                      properties:
+                        destinationPath:
+                          type: string
+                        endpointURL:
+                          type: string
+                        s3Credentials:
+                          properties:
+                            accessKeyId:
+                              properties:
+                                key:
+                                  type: string
+                                name:
+                                  type: string
+                              type: object
+                            secretAccessKey:
+                              properties:
+                                key:
+                                  type: string
+                                name:
+                                  type: string
+                              type: object
+                          type: object
+                      type: object
+                  type: object
+                method:
+                  type: string
+                pluginConfiguration:
+                  properties:
+                    name:
+                      type: string
+                  type: object
+              type: object
+            status:
+              properties:
+                backupId:
+                  type: string
+                backupName:
+                  type: string
+                message:
+                  type: string
+                startedAt:
+                  format: date-time
+                  type: string
+                state:
+                  type: string
+                stoppedAt:
+                  format: date-time
+                  type: string
+              type: object
+          type: object
+      served: true
+      storage: true
+      subresources:
+        status: {}

deploy/operator/parts/crd.yaml

@@ -6341,4 +6341,4 @@ spec:
                               describe behavior of generated Service
                               More info: https://kubernetes.io/docs/concepts/services-networking/service/
                             # nullable: true
-                            x-kubernetes-preserve-unknown-fields: true
+                            x-kubernetes-preserve-unknown-fields: true

dev/run_code_generator.sh

@@ -58,6 +58,8 @@ bash "${CODE_GENERATOR_DIR}/generate-groups.sh" \
     -o "${GENERATOR_ROOT}" \
     --go-header-file "${SRC_ROOT}/hack/boilerplate.go.txt"
 
+    #TODO need to add generator for clickhuose-backup.altinity.com:v1
+
 echo "Copy generated sources into: ${PKG_ROOT}"
 cp -r "${GENERATOR_ROOT}/${REPO}/pkg/"* "${PKG_ROOT}"
 

docs/chb-examples/01-simple-1.yaml

@@ -0,0 +1,26 @@
+apiVersion: "clickhouse-backup.altinity.com/v1"
+kind: "ClickHouseBackup"
+metadata:
+  name: backup-example
+spec:
+  backup:
+    dbTable:
+      whiteList: ["db1.table1","db1.table2","db2.table1"]
+      blackList: ["db1.test", "db1.logs", "db2.temporary"]
+    s3:
+      destinationPath: s3://backups/
+      endpointURL: http://minio:9000
+      s3Credentials:
+        accessKeyId:
+          name: minio
+          key: ACCESS_KEY_ID
+        secretAccessKey:
+          name: minio
+          key: ACCESS_SECRET_KEY
+  method: plugin
+  pluginConfiguration:
+    name: clickhouse.backup.altinity.com
+  ClickhouseInstallation:
+    name: simple-01
+    cluster:
+      name: chcluster1