cephfs: move fscrypt lock into the CephFS RADOS namespace

PendingReleaseNotes.md

@@ -37,3 +37,12 @@
    different subvolume.
 
 ## NOTE
+
+- The RADOS lock that serializes fscrypt setup for encrypted CephFS volumes
+  is now taken in the CephFS RADOS namespace instead of the default
+  namespace of the metadata pool. This applies to every deployment with
+  encrypted volumes. `cephFS.radosNamespace` defaults to `csi`, so the lock
+  moves to this namespace even when the option was never directly
+  configured. During a rolling nodeplugin upgrade the locks in the default
+  namespace and `cephFS.radosNamespace` are taken, so that pods which have
+  not been upgraded yet stay serialized against upgraded ones.

e2e/cephfs.go

@@ -647,6 +647,70 @@ var _ = Describe(cephfsType, func() {
 					}
 				})
 			}
+
+			It("verify the fscrypt lock is taken in the CephFS RADOS namespace", func() {
+				scOpts := map[string]string{
+					"encrypted":       "true",
+					"encryptionKMSID": "secrets-metadata-test",
+				}
+				err := createCephfsStorageClass(f.ClientSet, f, true, scOpts)
+				if err != nil {
+					logAndFail("failed to create CephFS storageclass: %v", err)
+				}
+
+				pvc, app, err := createPVCAndAppBinding(pvcPath, appPath, f, deployTimeout)
+				if err != nil {
+					logAndFail("failed to create PVC and application binding: %v", err)
+				}
+
+				imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
+				if err != nil {
+					logAndFail("failed to get image info from PVC: %v", err)
+				}
+
+				// Staging an encrypted volume serializes the fscrypt setup
+				// with a RADOS lock on an object named after the ObjectUUID
+				// of the volume. The lock must live in the RADOS namespace
+				// configured as cephFS.radosNamespace, where cephfsOptions()
+				// points the rados commands.
+				//
+				// The lock is released before NodeStageVolume returns, so it
+				// can not be observed directly. Instead this relies on
+				// cls_lock leaving the object behind on unlock, which holds
+				// for every lock type except LOCK_EXCLUSIVE_EPHEMERAL. If
+				// the fscrypt lock ever becomes ephemeral, this check needs
+				// a new way to observe the namespace of the lock.
+				_, stdErr, err := execCommandInToolBoxPod(f,
+					fmt.Sprintf("rados stat %s %s", imageData.imageID, cephfsOptions(metadataPool)),
+					rookNamespace)
+				if err != nil || stdErr != "" {
+					logAndFail("fscrypt lock object %s missing in the CephFS RADOS namespace: err=%v stdErr=%s",
+						imageData.imageID, err, stdErr)
+				}
+
+				// The transitional lock that serializes against nodeplugins
+				// which have not been upgraded yet is taken in the default
+				// RADOS namespace. Remove this check together with
+				// acquireLegacyEncryptionLock.
+				_, stdErr, err = execCommandInToolBoxPod(f,
+					fmt.Sprintf("rados stat %s --pool=%s", imageData.imageID, metadataPool),
+					rookNamespace)
+				if err != nil || stdErr != "" {
+					logAndFail("legacy fscrypt lock object %s missing in the default RADOS namespace: err=%v stdErr=%s",
+						imageData.imageID, err, stdErr)
+				}
+
+				err = deletePVCAndApp("", f, pvc, app)
+				if err != nil {
+					logAndFail("failed to delete PVC and application: %v", err)
+				}
+				validateOmapCount(f, 0, cephfsType, metadataPool, volumesType)
+
+				err = deleteResource(cephFSExamplePath + "storageclass.yaml")
+				if err != nil {
+					logAndFail("failed to delete CephFS storageclass: %v", err)
+				}
+			})
 		}
 
 		It("create a PVC and check PVC/PV metadata on CephFS subvolume", func() {

internal/cephfs/nodeserver.go

@@ -167,6 +167,15 @@ func maybeUnlockFileEncryption(
 
 	log.DebugLog(ctx, "Creating lock for the following volume ID %s", volID)
 
+	releaseLegacyLock, err := acquireLegacyEncryptionLock(ctx, volOptions, objectUUID,
+		lockName, lockCookie, lockDesc, lockDuration)
+	if err != nil {
+		log.ErrorLog(ctx, "failed to create the legacy lock for volume ID %s: %v", volID, err)
+
+		return err
+	}
+	defer releaseLegacyLock()
+
 	ioctx, err := volOptions.GetConnection().GetIoctx(volOptions.MetadataPool)
 	if err != nil {
 		log.ErrorLog(ctx, "Failed to create ioctx: %s", err)
@@ -175,6 +184,8 @@ func maybeUnlockFileEncryption(
 	}
 	defer ioctx.Destroy()
 
+	ioctx.SetNamespace(volOptions.RadosNamespace)
+
 	lock := iolock.NewLock(ioctx, objectUUID, lockName, lockCookie, lockDesc, lockDuration)
 	err = lock.LockExclusive(ctx)
 	if err != nil {
@@ -194,6 +205,57 @@ func maybeUnlockFileEncryption(
 	return nil
 }
 
+// acquireLegacyEncryptionLock takes the fscrypt lock in the default RADOS
+// namespace of the metadata pool, where releases before the lock moved to the
+// volume's RADOS namespace took it.
+//
+// The returned function releases the lock and must always be called.
+//
+// TODO: remove once upgrades from releases that lock in the default RADOS
+// namespace are no longer supported.
+func acquireLegacyEncryptionLock(
+	ctx context.Context,
+	volOptions *store.VolumeOptions,
+	objectUUID, lockName, lockCookie, lockDesc string,
+	lockDuration time.Duration,
+) (func(), error) {
+	noop := func() {}
+
+	// Skip when the volume has no RADOS namespace. In that case the caller
+	// already takes its lock in the default namespace of the pool, which is
+	// exactly where the legacy lock lives, and locking the same object twice
+	// with the same cookie fails with EEXIST.
+	if volOptions.RadosNamespace == "" {
+		return noop, nil
+	}
+
+	ioctx, err := volOptions.GetConnection().GetIoctx(volOptions.MetadataPool)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create ioctx in the default namespace of pool %q: %w",
+			volOptions.MetadataPool, err)
+	}
+
+	lock := iolock.NewLock(ioctx, objectUUID, lockName, lockCookie, lockDesc, lockDuration)
+	if err = lock.LockExclusive(ctx); err != nil {
+		ioctx.Destroy()
+
+		if errors.Is(err, iolock.ErrLockNotPermitted) {
+			log.DebugLog(ctx, "not allowed to lock in the default namespace of pool %q, "+
+				"skipping the legacy lock for object %s: %v",
+				volOptions.MetadataPool, objectUUID, err)
+
+			return noop, nil
+		}
+
+		return nil, err
+	}
+
+	return func() {
+		lock.Unlock(ctx)
+		ioctx.Destroy()
+	}, nil
+}
+
 // generateLockCookie generates a consistent lock cookie for the client.
 func generateLockCookie() string {
 	hostname, err := os.Hostname()

internal/util/lock/lock.go

@@ -18,6 +18,7 @@ package lock
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"syscall"
 	"time"
@@ -27,6 +28,12 @@ import (
 	"github.com/ceph/ceph-csi/internal/util/log"
 )
 
+// ErrLockNotPermitted is returned when the CephX credentials are not allowed to
+// perform the lock operation. Locking is a RADOS class operation, so it needs
+// the class-exec permission on the pool, and caps that cover the RADOS
+// namespace of the IO context.
+var ErrLockNotPermitted = errors.New("not allowed to lock")
+
 // IOCtxLock provides methods for acquiring and releasing exclusive locks on a volume.
 // using rados IO context locks.
 type IOCtxLock interface {
@@ -82,6 +89,8 @@ func (lck *lock) LockExclusive(ctx context.Context) error {
 		case -int(syscall.EEXIST):
 			return fmt.Errorf("lock is already held by the same client and cookie pair for %v volume",
 				lck.volID)
+		case -int(syscall.EPERM), -int(syscall.EACCES):
+			return fmt.Errorf("%w volume ID %v", ErrLockNotPermitted, lck.volID)
 		default:
 			return fmt.Errorf("failed to lock volume ID %v: %w", lck.volID, err)
 		}