K8SPXC-1576: xtrabackup sidecar support

Makefile

@@ -16,7 +16,7 @@ all: build
 help: ## Display this help.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
-generate: controller-gen  ## Generate CRDs and RBAC files
+generate: controller-gen protoc  ## Generate CRDs and RBAC files
 	go generate ./...
 	$(CONTROLLER_GEN) crd:maxDescLen=0,allowDangerousTypes=true,generateEmbeddedObjectMeta=true rbac:roleName=$(NAME) webhook paths="./..." output:crd:artifacts:config=config/crd/bases  ## Generate WebhookConfiguration, Role and CustomResourceDefinition objects.
 	$(CONTROLLER_GEN) object paths="./..." ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
@@ -108,6 +108,27 @@ SWAGGER = $(shell pwd)/bin/swagger
 swagger: ## Download swagger locally if necessary.
 	$(call go-get-tool,$(SWAGGER),github.com/go-swagger/go-swagger/cmd/swagger@latest)
 
+PROTOC_VERSION = 33.1
+PROTOC = $(shell pwd)/bin/protoc
+PROTOC_GEN_GO = $(shell pwd)/bin/protoc-gen-go
+PROTOC_GEN_GO_GRPC = $(shell pwd)/bin/protoc-gen-go-grpc
+protoc: ## Download protoc locally if necessary.
+	os='linux'; \
+	arch='x86_64'; \
+	if [ "$(shell uname)" = "Darwin" ]; then \
+		os='osx'; \
+	fi; \
+	if [ "$(shell uname -m)" = "arm64" ]; then \
+		arch='aarch_64'; \
+	fi; \
+	curl -LO "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-$${os}-$${arch}.zip"; \
+	unzip -o protoc-${PROTOC_VERSION}-$${os}-$${arch}.zip -d protoc-${PROTOC_VERSION}-$${os}-$${arch}; \
+	rm protoc-${PROTOC_VERSION}-$${os}-$${arch}.zip; \
+	mv -f protoc-${PROTOC_VERSION}-$${os}-$${arch}/bin/protoc $(PROTOC); \
+	rm -rf protoc-${PROTOC_VERSION}-$${os}-$${arch}; \
+	$(call go-get-tool,$(PROTOC_GEN_GO),google.golang.org/protobuf/cmd/protoc-gen-go@latest); \
+	$(call go-get-tool,$(PROTOC_GEN_GO_GRPC),google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest)
+
 # Prepare release
 include e2e-tests/release_versions
 CERT_MANAGER_VER := $(shell grep -Eo "cert-manager v.*" go.mod|grep -Eo "[0-9]+\.[0-9]+\.[0-9]+")

build/Dockerfile

@@ -29,6 +29,16 @@ RUN GOOS=$GOOS GOARCH=${TARGETARCH} CGO_ENABLED=$CGO_ENABLED GO_LDFLAGS=$GO_LDFL
        go build -o build/_output/bin/pitr \
                cmd/pitr/main.go \
     && cp -r build/_output/bin/pitr /usr/local/bin/pitr
+
+RUN GOOS=$GOOS GOARCH=${TARGETARCH} CGO_ENABLED=$CGO_ENABLED GO_LDFLAGS=$GO_LDFLAGS \
+       go build -o build/_output/bin/xtrabackup-server-sidecar \
+               cmd/xtrabackup/server/main.go \
+    && cp -r build/_output/bin/xtrabackup-server-sidecar /usr/local/bin/xtrabackup-server-sidecar
+
+RUN GOOS=$GOOS GOARCH=${TARGETARCH} CGO_ENABLED=$CGO_ENABLED GO_LDFLAGS=$GO_LDFLAGS \
+       go build -o build/_output/bin/xtrabackup-run-backup \
+               cmd/xtrabackup/run-backup/main.go \
+    && cp -r build/_output/bin/xtrabackup-run-backup /usr/local/bin/xtrabackup-run-backup
 
 RUN GOOS=$GOOS GOARCH=${TARGETARCH} CGO_ENABLED=$CGO_ENABLED GO_LDFLAGS=$GO_LDFLAGS \
        go build -ldflags "-w -s -X main.GitCommit=$GIT_COMMIT -X main.GitBranch=$GIT_BRANCH -X main.BuildTime=$BUILD_TIME" \
@@ -55,6 +65,8 @@ COPY --from=go_builder /usr/local/bin/percona-xtradb-cluster-operator /usr/local
 COPY --from=go_builder /usr/local/bin/peer-list /peer-list
 COPY --from=go_builder /usr/local/bin/pitr /pitr
 COPY --from=go_builder /usr/local/bin/mysql-state-monitor /mysql-state-monitor
+COPY --from=go_builder /usr/local/bin/xtrabackup-server-sidecar /xtrabackup-server-sidecar
+COPY --from=go_builder /usr/local/bin/xtrabackup-run-backup /xtrabackup-run-backup
 COPY build/pxc-entrypoint.sh /pxc-entrypoint.sh
 COPY build/pxc-init-entrypoint.sh /pxc-init-entrypoint.sh
 COPY build/pitr-init-entrypoint.sh /pitr-init-entrypoint.sh

build/backup-init-entrypoint.sh

@@ -4,6 +4,7 @@ set -o errexit
 set -o xtrace
 
 install -o "$(id -u)" -g "$(id -g)" -m 0755 -D /peer-list /opt/percona/peer-list
+install -o "$(id -u)" -g "$(id -g)" -m 0755 -D /xtrabackup-run-backup /opt/percona/xtrabackup-run-backup
 
 mkdir -p /opt/percona/backup/lib/pxc
 install -o "$(id -u)" -g "$(id -g)" -m 0755 -D /backup/lib/pxc/* /opt/percona/backup/lib/pxc/

build/backup/recovery-cloud.sh

@@ -54,6 +54,23 @@ fi
 # shellcheck disable=SC2086
 xbcloud get --parallel="$(grep -c processor /proc/cpuinfo)" ${XBCLOUD_ARGS} "$(destination)" | xbstream -x -C "${tmp}" --parallel="$(grep -c processor /proc/cpuinfo)" $XBSTREAM_EXTRA_ARGS
 
+set +o xtrace
+if [[ -f "${tmp}/sst_info" ]]; then
+	transition_key=$(vault_get "$tmp/sst_info")
+	if [[ -n $transition_key && $transition_key != null ]]; then
+		MYSQL_VERSION=$(parse_ini 'mysql-version' "$tmp/sst_info")
+		if ! check_for_version "$MYSQL_VERSION" '5.7.29' \
+			&& [[ $MYSQL_VERSION != '5.7.28-31-57.2' ]]; then
+
+			# shellcheck disable=SC2016
+			transition_key='$transition_key'
+		fi
+
+		transition_option="--transition-key=$transition_key"
+		echo transition-key exists
+	fi
+fi
+
 PXB_VAULT_PREPARE_ARGS=""
 PXB_VAULT_MOVEBACK_ARGS=""
 VAULT_CONFIG_FILE=/etc/mysql/vault-keyring-secret/keyring_vault.conf
@@ -65,23 +82,18 @@ if [[ -f ${VAULT_CONFIG_FILE} ]]; then
 		PXB_VAULT_PREPARE_ARGS="${PXB_VAULT_MOVEBACK_ARGS}"
 	else
 		PXB_VAULT_MOVEBACK_ARGS="--keyring-vault-config=${VAULT_CONFIG_FILE} --early-plugin-load=keyring_vault.so"
+
+		# If backup-my.cnf does not contain plugin_load, then --prepare will fail if you pass the --keyring-vault-config option.
+		if [[ -n "$(parse_ini 'plugin_load' "${tmp}/backup-my.cnf")" ]] && [[ -z $transition_key ]]; then
+			PXB_VAULT_PREPARE_ARGS="--keyring-vault-config=${VAULT_CONFIG_FILE}"
+		fi
 	fi
 fi
 
-set +o xtrace
-transition_key=$(vault_get "$tmp/sst_info")
-if [[ -n $transition_key && $transition_key != null ]]; then
-	MYSQL_VERSION=$(parse_ini 'mysql-version' "$tmp/sst_info")
-	if ! check_for_version "$MYSQL_VERSION" '5.7.29' \
-		&& [[ $MYSQL_VERSION != '5.7.28-31-57.2' ]]; then
-
-		# shellcheck disable=SC2016
-		transition_key='$transition_key'
-	fi
 
-	transition_option="--transition-key=$transition_key"
+
+if [ -f "${tmp}/xtrabackup_keys" ]; then
 	master_key_options="--generate-new-master-key"
-	echo transition-key exists
 fi
 
 # Extract --defaults-file from XB_EXTRA_ARGS if present and place it as the first argument
@@ -108,6 +120,13 @@ if ! check_for_version "$XTRABACKUP_VERSION" '8.0.0'; then
 	XB_EXTRA_ARGS="$XB_EXTRA_ARGS --binlog-info=ON"
 fi
 
+DEFAULTS_GROUP="--defaults-group=mysqld"
+if [[ ${XTRABACKUP_ENABLED} == "true" ]]; then
+	# these must not be set for pxb
+	DEFAULTS_GROUP=""
+	DEFAULTS_FILE=""
+fi
+
 echo "+ xtrabackup $DEFAULTS_FILE ${XB_USE_MEMORY+--use-memory=$XB_USE_MEMORY} --prepare \
 	$REMAINING_XB_ARGS --rollback-prepared-trx  --xtrabackup-plugin-dir=/usr/lib64/xtrabackup/plugin \
 	--target-dir=$tmp ${PXB_VAULT_PREPARE_ARGS}"
@@ -122,7 +141,7 @@ echo "+ xtrabackup $DEFAULTS_FILE --defaults-group=mysqld --datadir=/datadir --m
 	${PXB_VAULT_MOVEBACK_ARGS} --xtrabackup-plugin-dir=/usr/lib64/xtrabackup/plugin --target-dir=$tmp"
 
 # shellcheck disable=SC2086
-xtrabackup $DEFAULTS_FILE --defaults-group=mysqld --datadir=/datadir --move-back $REMAINING_XB_ARGS \
+xtrabackup $DEFAULTS_FILE $DEFAULTS_GROUP --datadir=/datadir --move-back $REMAINING_XB_ARGS \
 	--force-non-empty-directories $transition_option $master_key_options \
 	${PXB_VAULT_MOVEBACK_ARGS} --xtrabackup-plugin-dir=/usr/lib64/xtrabackup/plugin --target-dir="$tmp"
 

build/pxc-init-entrypoint.sh

@@ -8,6 +8,7 @@ install -o "$(id -u)" -g "$(id -g)" -m 0755 -D /pxc-configure-pxc.sh /var/lib/my
 install -o "$(id -u)" -g "$(id -g)" -m 0755 -D /liveness-check.sh /var/lib/mysql/liveness-check.sh
 install -o "$(id -u)" -g "$(id -g)" -m 0755 -D /readiness-check.sh /var/lib/mysql/readiness-check.sh
 install -o "$(id -u)" -g "$(id -g)" -m 0755 -D /peer-list /var/lib/mysql/peer-list
+install -o "$(id -u)" -g "$(id -g)" -m 0755 -D /xtrabackup-server-sidecar /var/lib/mysql/xtrabackup-server-sidecar
 install -o "$(id -u)" -g "$(id -g)" -m 0755 -D /get-pxc-state /var/lib/mysql/get-pxc-state
 install -o "$(id -u)" -g "$(id -g)" -m 0755 -D /pmm-prerun.sh /var/lib/mysql/pmm-prerun.sh
 install -o "$(id -u)" -g "$(id -g)" -m 0755 -D /mysql-state-monitor /var/lib/mysql/mysql-state-monitor

cmd/manager/main.go

@@ -30,6 +30,7 @@ import (
 	"github.com/percona/percona-xtradb-cluster-operator/pkg/apis"
 	pxcv1 "github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1"
 	"github.com/percona/percona-xtradb-cluster-operator/pkg/controller"
+	"github.com/percona/percona-xtradb-cluster-operator/pkg/features"
 	"github.com/percona/percona-xtradb-cluster-operator/pkg/k8s"
 	"github.com/percona/percona-xtradb-cluster-operator/pkg/version"
 	"github.com/percona/percona-xtradb-cluster-operator/pkg/webhook"
@@ -89,6 +90,19 @@ func main() {
 		os.Exit(1)
 	}
 
+	fg := features.NewGate()
+	if err := fg.Set(os.Getenv("PXCO_FEATURE_GATES")); err != nil {
+		setupLog.Error(err, "failed to set feature gates")
+		os.Exit(1)
+	}
+	fgCtx := features.NewContextWithGate(context.Background(), fg)
+	setupLog.Info("Feature gates",
+		// These are set by the user
+		"PXCO_FEATURE_GATES", features.ShowAssigned(fgCtx),
+		// These are enabled, including features that are on by default
+		"enabled", features.ShowEnabled(fgCtx),
+	)
+
 	options := ctrl.Options{
 		Scheme: scheme,
 		Metrics: metricsServer.Options{
@@ -100,6 +114,9 @@ func main() {
 		WebhookServer: ctrlWebhook.NewServer(ctrlWebhook.Options{
 			Port: 9443,
 		}),
+		BaseContext: func() context.Context {
+			return features.NewContextWithGate(context.Background(), fg)
+		},
 	}
 
 	err = configureGroupKindConcurrency(&options)