From 6ecf0ff0950124f0ca74f9b923b714407b1a0ae3 Mon Sep 17 00:00:00 2001 From: Sneha Aradhey Date: Wed, 12 Feb 2025 22:31:05 +0000 Subject: [PATCH 01/25] Add support for data cache --- Dockerfile | 46 ++- cmd/gce-pd-csi-driver/main.go | 64 ++- .../base/controller/cluster_setup.yaml | 3 + deploy/kubernetes/base/node_linux/node.yaml | 13 + initialize-driver.sh | 9 + pkg/common/constants.go | 12 + pkg/common/parameters.go | 79 +++- pkg/common/parameters_test.go | 74 +++- pkg/common/runcmd.go | 71 ++++ pkg/common/utils.go | 42 ++ pkg/common/utils_test.go | 166 ++++++++ pkg/deviceutils/device-utils.go | 10 +- pkg/gce-pd-csi-driver/cache.go | 369 ++++++++++++++++++ pkg/gce-pd-csi-driver/controller.go | 46 ++- pkg/gce-pd-csi-driver/gce-pd-driver.go | 6 +- pkg/gce-pd-csi-driver/gce-pd-driver_test.go | 12 +- pkg/gce-pd-csi-driver/node.go | 36 +- pkg/gce-pd-csi-driver/node_test.go | 8 +- test/e2e/tests/multi_zone_e2e_test.go | 158 +++++--- test/e2e/tests/resize_e2e_test.go | 8 +- test/e2e/tests/setup_e2e_test.go | 85 ++-- test/e2e/tests/single_zone_e2e_test.go | 147 ++++++- test/e2e/utils/utils.go | 30 ++ .../k8s-integration/config/data-cache-sc.yaml | 11 + test/remote/client-wrappers.go | 29 +- test/remote/instance.go | 161 ++++---- test/remote/runner.go | 8 +- test/remote/setup-teardown.go | 19 +- test/run-e2e.sh | 7 +- test/sanity/sanity_test.go | 8 +- 30 files changed, 1487 insertions(+), 250 deletions(-) create mode 100755 initialize-driver.sh create mode 100644 pkg/common/runcmd.go create mode 100644 pkg/gce-pd-csi-driver/cache.go create mode 100644 test/k8s-integration/config/data-cache-sc.yaml diff --git a/Dockerfile b/Dockerfile index 50e4b1ffc..5136146b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ FROM gke.gcr.io/debian-base:bookworm-v1.0.4-gke.2 as debian # Install necessary dependencies # google_nvme_id script depends on the following packages: nvme-cli, xxd, bash -RUN clean-install util-linux e2fsprogs mount ca-certificates udev xfsprogs nvme-cli xxd bash +RUN clean-install util-linux e2fsprogs mount ca-certificates udev xfsprogs nvme-cli xxd bash kmod lvm2 mdadm # Since we're leveraging apt to pull in dependencies, we use `gcr.io/distroless/base` because it includes glibc. FROM gcr.io/distroless/base-debian12 as distroless-base @@ -56,6 +56,35 @@ COPY --from=debian /sbin/e2fsck /sbin/e2fsck COPY --from=debian /sbin/fsck /sbin/fsck COPY --from=debian /sbin/fsck* /sbin/ COPY --from=debian /sbin/fsck.xfs /sbin/fsck.xfs +# Add dependencies for LVM +COPY --from=debian /etc/lvm /lvm-tmp/lvm +COPY --from=debian /lib/systemd/system/blk-availability.service /lib/systemd/system/blk-availability.service +COPY --from=debian /lib/systemd/system/lvm2-lvmpolld.service /lib/systemd/system/lvm2-lvmpolld.service +COPY --from=debian /lib/systemd/system/lvm2-lvmpolld.socket /lib/systemd/system/lvm2-lvmpolld.socket +COPY --from=debian /lib/systemd/system/lvm2-monitor.service /lib/systemd/system/lvm2-monitor.service +COPY --from=debian /lib/udev/rules.d/56-lvm.rules /lib/udev/rules.d/56-lvm.rules +COPY --from=debian /sbin/fsadm /sbin/fsadm +COPY --from=debian /sbin/lvm /sbin/lvm +COPY --from=debian /sbin/lvmdump /sbin/lvmdump +COPY --from=debian /sbin/lvmpolld /sbin/lvmpolld +COPY --from=debian /usr/lib/tmpfiles.d /usr/lib/tmpfiles.d +COPY --from=debian /usr/lib/tmpfiles.d/lvm2.conf /usr/lib/tmpfiles.d/lvm2.conf +COPY --from=debian /sbin/lv* /sbin/ +COPY --from=debian /sbin/pv* /sbin/ +COPY --from=debian /sbin/vg* /sbin/ +COPY --from=debian /bin/lsblk /bin/lsblk +COPY --from=debian /sbin/modprobe /sbin/modprobe +COPY --from=debian /lib/udev /lib/udev +COPY --from=debian /lib/udev/rules.d /lib/udev/rules.d +COPY --from=debian /lib/udev/rules.d/55-dm.rules /lib/udev/rules.d/55-dm.rules +COPY --from=debian /lib/udev/rules.d/60-persistent-storage-dm.rules /lib/udev/rules.d/60-persistent-storage-dm.rules +COPY --from=debian /lib/udev/rules.d/95-dm-notify.rules /lib/udev/rules.d/95-dm-notify.rules +COPY --from=debian /sbin/blkdeactivate /sbin/blkdeactivate +COPY --from=debian /sbin/dmsetup /sbin/dmsetup +COPY --from=debian /sbin/dmstats /sbin/dmstats +COPY --from=debian /bin/ls /bin/ls +# End of dependencies for LVM +COPY --from=debian /sbin/mdadm /sbin/mdadm COPY --from=debian /sbin/mke2fs /sbin/mke2fs COPY --from=debian /sbin/mkfs* /sbin/ COPY --from=debian /sbin/resize2fs /sbin/resize2fs @@ -71,14 +100,20 @@ COPY --from=debian /bin/date /bin/date COPY --from=debian /bin/grep /bin/grep COPY --from=debian /bin/sed /bin/sed COPY --from=debian /bin/ln /bin/ln +COPY --from=debian /bin/cp /bin/cp COPY --from=debian /bin/udevadm /bin/udevadm # Copy shared libraries into distroless base. COPY --from=debian /lib/${LIB_DIR_PREFIX}-linux-gnu/libselinux.so.1 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libdl.so.2 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libpthread.so.0 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libtinfo.so.6 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libe2p.so.2 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libcom_err.so.2 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libdevmapper.so.1.02.1 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libm.so.6 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libc.so.6 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libdevmapper-event.so.1.02.1 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libext2fs.so.2 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libgcc_s.so.1 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/liblzma.so.5 \ @@ -99,11 +134,17 @@ COPY --from=debian /lib/${LIB_DIR_PREFIX}-linux-gnu/libselinux.so.1 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libzstd.so.1 /lib/${LIB_DIR_PREFIX}-linux-gnu/ COPY --from=debian /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libblkid.so.1 \ + /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libsmartcols.so.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libbsd.so.0 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libinih.so.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libmount.so.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libudev.so.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libuuid.so.1 \ + /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libzstd.so.1 \ + /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libaio.so.1 \ + /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libgcrypt.so.20 \ + /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libsystemd.so.0 \ + /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/liblz4.so.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libacl.so.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libattr.so.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libedit.so.2 \ @@ -130,4 +171,5 @@ RUN /print-missing-deps.sh # Final build stage, create the real Docker image with ENTRYPOINT FROM output-image -ENTRYPOINT ["/gce-pd-csi-driver"] +COPY --from=builder /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/initialize-driver.sh /initialize-driver.sh +ENTRYPOINT ["/initialize-driver.sh"] diff --git a/cmd/gce-pd-csi-driver/main.go b/cmd/gce-pd-csi-driver/main.go index 3c7808498..d638d0bd2 100644 --- a/cmd/gce-pd-csi-driver/main.go +++ b/cmd/gce-pd-csi-driver/main.go @@ -27,9 +27,12 @@ import ( "strings" "time" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/klog/v2" "k8s.io/utils/strings/slices" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/deviceutils" gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute" @@ -68,10 +71,13 @@ var ( maxConcurrentFormat = flag.Int("max-concurrent-format", 1, "The maximum number of concurrent format exec calls") concurrentFormatTimeout = flag.Duration("concurrent-format-timeout", 1*time.Minute, "The maximum duration of a format operation before its concurrency token is released") - maxConcurrentFormatAndMount = flag.Int("max-concurrent-format-and-mount", 1, "If set then format and mount operations are serialized on each node. This is stronger than max-concurrent-format as it includes fsck and other mount operations") - formatAndMountTimeout = flag.Duration("format-and-mount-timeout", 1*time.Minute, "The maximum duration of a format and mount operation before another such operation will be started. Used only if --serialize-format-and-mount") - fallbackRequisiteZonesFlag = flag.String("fallback-requisite-zones", "", "Comma separated list of requisite zones that will be used if there are not sufficient zones present in requisite topologies when provisioning a disk") - enableStoragePoolsFlag = flag.Bool("enable-storage-pools", false, "If set to true, the CSI Driver will allow volumes to be provisioned in Storage Pools") + maxConcurrentFormatAndMount = flag.Int("max-concurrent-format-and-mount", 1, "If set then format and mount operations are serialized on each node. This is stronger than max-concurrent-format as it includes fsck and other mount operations") + formatAndMountTimeout = flag.Duration("format-and-mount-timeout", 1*time.Minute, "The maximum duration of a format and mount operation before another such operation will be started. Used only if --serialize-format-and-mount") + fallbackRequisiteZonesFlag = flag.String("fallback-requisite-zones", "", "Comma separated list of requisite zones that will be used if there are not sufficient zones present in requisite topologies when provisioning a disk") + enableStoragePoolsFlag = flag.Bool("enable-storage-pools", false, "If set to true, the CSI Driver will allow volumes to be provisioned in Storage Pools") + enableControllerDataCacheFlag = flag.Bool("enable-controller-data-cache", false, "If set to true, the CSI Driver will allow volumes to be provisioned with data cache configuration") + enableNodeDataCacheFlag = flag.Bool("enable-node-data-cache", false, "If set to true, the CSI Driver will allow volumes to be provisioned with data cache configuration") + nodeName = flag.String("node-name", "", "The node this driver is running on") multiZoneVolumeHandleDiskTypesFlag = flag.String("multi-zone-volume-handle-disk-types", "", "Comma separated list of allowed disk types that can use the multi-zone volumeHandle. Used only if --multi-zone-volume-handle-enable") multiZoneVolumeHandleEnableFlag = flag.Bool("multi-zone-volume-handle-enable", false, "If set to true, the multi-zone volumeHandle feature will be enabled") @@ -90,7 +96,9 @@ var ( ) const ( - driverName = "pd.csi.storage.gke.io" + driverName = "pd.csi.storage.gke.io" + dataCacheLabel = "datacache-storage-gke-io" + dataCacheLabelValue = "enabled" ) func init() { @@ -209,7 +217,7 @@ func handle() { } initialBackoffDuration := time.Duration(*errorBackoffInitialDurationMs) * time.Millisecond maxBackoffDuration := time.Duration(*errorBackoffMaxDurationMs) * time.Millisecond - controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration, fallbackRequisiteZones, *enableStoragePoolsFlag, multiZoneVolumeHandleConfig, listVolumesConfig) + controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration, fallbackRequisiteZones, *enableStoragePoolsFlag, *enableControllerDataCacheFlag, multiZoneVolumeHandleConfig, listVolumesConfig) } else if *cloudConfigFilePath != "" { klog.Warningf("controller service is disabled but cloud config given - it has no effect") } @@ -227,12 +235,24 @@ func handle() { if err != nil { klog.Fatalf("Failed to set up metadata service: %v", err.Error()) } - nodeServer = driver.NewNodeServer(gceDriver, mounter, deviceUtils, meta, statter) + nsArgs := driver.NodeServerArgs{ + EnableDataCache: *enableNodeDataCacheFlag, + } + nodeServer = driver.NewNodeServer(gceDriver, mounter, deviceUtils, meta, statter, nsArgs) if *maxConcurrentFormatAndMount > 0 { nodeServer = nodeServer.WithSerializedFormatAndMount(*formatAndMountTimeout, *maxConcurrentFormatAndMount) } } + if *enableNodeDataCacheFlag { + if nodeName == nil || *nodeName == "" { + klog.Errorf("Data cache enabled, but --node-name not passed") + } + if err := setupDataCache(ctx, *nodeName); err != nil { + klog.Errorf("DataCache setup failed: %v", err) + } + } + err = gceDriver.SetupGCEDriver(driverName, version, extraVolumeLabels, extraTags, identityServer, controllerServer, nodeServer) if err != nil { klog.Fatalf("Failed to initialize GCE CSI Driver: %v", err.Error()) @@ -311,3 +331,33 @@ func urlFlag(target **url.URL, name string, usage string) { return err }) } + +func setupDataCache(ctx context.Context, nodeName string) error { + klog.V(2).Infof("Seting up data cache for node %s", nodeName) + if nodeName != common.TestNode { + cfg, err := rest.InClusterConfig() + if err != nil { + return err + } + kubeClient, err := kubernetes.NewForConfig(cfg) + if err != nil { + return err + } + node, err := kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + // We could retry, but this error will also crashloop the driver which may be as good a way to retry as any. + return err + } + if val, found := node.GetLabels()[dataCacheLabel]; !found || val != dataCacheLabelValue { + klog.V(2).Infof("Datacache not enabled for node %s; node label %s=%s and not %s", nodeName, dataCacheLabel, val, dataCacheLabelValue) + return nil + } + } + klog.V(2).Info("Raiding local ssds to setup data cache") + if err := driver.RaidLocalSsds(); err != nil { + return fmt.Errorf("Failed to Raid local SSDs, unable to setup data caching, got error %v", err) + } + + klog.V(2).Infof("Datacache enabled for node %s", nodeName) + return nil +} diff --git a/deploy/kubernetes/base/controller/cluster_setup.yaml b/deploy/kubernetes/base/controller/cluster_setup.yaml index e22c46b55..804cc5e48 100644 --- a/deploy/kubernetes/base/controller/cluster_setup.yaml +++ b/deploy/kubernetes/base/controller/cluster_setup.yaml @@ -199,6 +199,9 @@ rules: verbs: ['use'] resourceNames: - csi-gce-pd-node-psp + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list"] --- kind: ClusterRole diff --git a/deploy/kubernetes/base/node_linux/node.yaml b/deploy/kubernetes/base/node_linux/node.yaml index ebf7779cf..211963118 100644 --- a/deploy/kubernetes/base/node_linux/node.yaml +++ b/deploy/kubernetes/base/node_linux/node.yaml @@ -46,8 +46,15 @@ spec: - "--v=5" - "--endpoint=unix:/csi/csi.sock" - "--run-controller-service=false" + - "--enable-node-data-cache" + - "--node-name=$(KUBE_NODE_NAME)" securityContext: privileged: true + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName volumeMounts: - name: kubelet-dir mountPath: /var/lib/kubelet @@ -66,6 +73,8 @@ spec: mountPath: /run/udev - name: sys mountPath: /sys + - name: lib-modules + mountPath: /lib/modules volumes: - name: registration-dir hostPath: @@ -101,6 +110,10 @@ spec: hostPath: path: /sys type: Directory + - name: lib-modules + hostPath: + path: /lib/modules + type: Directory # https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ # See "special case". This will tolerate everything. Node component should # be scheduled on all nodes. diff --git a/initialize-driver.sh b/initialize-driver.sh new file mode 100755 index 000000000..fe5a615c8 --- /dev/null +++ b/initialize-driver.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +/bin/cp -r /lvm-tmp/lvm /etc/ +/bin/sed -i -e "s/.*allow_mixed_block_sizes = 0.*/ allow_mixed_block_sizes = 1/" /etc/lvm/lvm.conf +/bin/sed -i -e "s/.*udev_sync = 1.*/ udev_sync = 0/" /etc/lvm/lvm.conf +/bin/sed -i -e "s/.*udev_rules = 1.*/ udev_rules = 0/" /etc/lvm/lvm.conf +/bin/sed -i -e "s/.*locking_dir = .*/ locking_dir = \"\/tmp\"/" /etc/lvm/lvm.conf + +/gce-pd-csi-driver "$@" \ No newline at end of file diff --git a/pkg/common/constants.go b/pkg/common/constants.go index 278063b19..0279dd3cc 100644 --- a/pkg/common/constants.go +++ b/pkg/common/constants.go @@ -32,6 +32,18 @@ const ( // Label that is set on a disk when it is used by a 'multi-zone' VolumeHandle MultiZoneLabel = "goog-gke-multi-zone" + + // Data cache mode + DataCacheModeWriteBack = "writeback" + DataCacheModeWriteThrough = "writethrough" + + ContextDataCacheSize = "data-cache-size" + ContextDataCacheMode = "data-cache-mode" + + // Keys in the publish context + ContexLocalSsdCacheSize = "local-ssd-cache-size" + // Node name for E2E tests + TestNode = "test-node-csi-e2e" ) // doc https://cloud.google.com/compute/docs/disks/hyperdisks#max-total-disks-per-vm diff --git a/pkg/common/parameters.go b/pkg/common/parameters.go index 2a1fa4d13..5bd8bd6a8 100644 --- a/pkg/common/parameters.go +++ b/pkg/common/parameters.go @@ -18,6 +18,7 @@ package common import ( "fmt" + "strconv" "strings" ) @@ -32,8 +33,13 @@ const ( ParameterAvailabilityClass = "availability-class" ParameterKeyEnableConfidentialCompute = "enable-confidential-storage" ParameterKeyStoragePools = "storage-pools" - ParameterKeyResourceTags = "resource-tags" - ParameterKeyEnableMultiZoneProvisioning = "enable-multi-zone-provisioning" + + // Parameters for Data Cache + ParameterKeyDataCacheSize = "data-cache-size" + ParameterKeyDataCacheMode = "data-cache-mode" + ParameterKeyResourceTags = "resource-tags" + ParameterKeyEnableMultiZoneProvisioning = "enable-multi-zone-provisioning" + ParameterHdHADiskType = "hyperdisk-balanced-high-availability" // Parameters for VolumeSnapshotClass ParameterKeyStorageLocations = "storage-locations" @@ -69,6 +75,15 @@ const ( tagKeyCreatedForSnapshotContentName = "kubernetes.io/created-for/volumesnapshotcontent/name" ) +type DataCacheParameters struct { + // Values: {string} in int64 form + // Default: "" + DataCacheSize string + // Values: writethrough, writeback + // Default: writethrough + DataCacheMode string +} + // DiskParameters contains normalized and defaulted disk parameters type DiskParameters struct { // Values: pd-standard, pd-balanced, pd-ssd, or any other PD disk type. Not validated. @@ -135,7 +150,8 @@ type ParameterProcessor struct { // put them into a well defined struct making sure to default unspecified fields. // extraVolumeLabels are added as labels; if there are also labels specified in // parameters, any matching extraVolumeLabels will be overridden. -func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string]string, extraVolumeLabels map[string]string, extraTags map[string]string) (DiskParameters, error) { +func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string]string, extraVolumeLabels map[string]string, enableDataCache bool, extraTags map[string]string) (DiskParameters, DataCacheParameters, error) { + p := DiskParameters{ DiskType: "pd-standard", // Default ReplicationType: replicationTypeNone, // Default @@ -145,6 +161,12 @@ func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string] ResourceTags: make(map[string]string), // Default } + // Set data cache mode default + d := DataCacheParameters{} + if enableDataCache && parameters[ParameterKeyDataCacheSize] != "" { + d.DataCacheMode = DataCacheModeWriteThrough + } + for k, v := range extraVolumeLabels { p.Labels[k] = v } @@ -179,7 +201,7 @@ func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string] case ParameterKeyLabels: paramLabels, err := ConvertLabelsStringToMap(v) if err != nil { - return p, fmt.Errorf("parameters contain invalid labels parameter: %w", err) + return p, d, fmt.Errorf("parameters contain invalid labels parameter: %w", err) } // Override any existing labels with those from this parameter. for labelKey, labelValue := range paramLabels { @@ -188,19 +210,25 @@ func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string] case ParameterKeyProvisionedIOPSOnCreate: paramProvisionedIOPSOnCreate, err := ConvertStringToInt64(v) if err != nil { - return p, fmt.Errorf("parameters contain invalid provisionedIOPSOnCreate parameter: %w", err) + return p, d, fmt.Errorf("parameters contain invalid provisionedIOPSOnCreate parameter: %w", err) } p.ProvisionedIOPSOnCreate = paramProvisionedIOPSOnCreate case ParameterKeyProvisionedThroughputOnCreate: paramProvisionedThroughputOnCreate, err := ConvertMiStringToInt64(v) if err != nil { - return p, fmt.Errorf("parameters contain invalid provisionedThroughputOnCreate parameter: %w", err) + return p, d, fmt.Errorf("parameters contain invalid provisionedThroughputOnCreate parameter: %w", err) + } + if paramProvisionedThroughputOnCreate < 0 { + return p, d, fmt.Errorf("parameter provisionedThroughputOnCreate cannot be negative") + } + if paramProvisionedThroughputOnCreate < 0 { + return p, d, fmt.Errorf("parameter provisionedThroughputOnCreate cannot be negative") } p.ProvisionedThroughputOnCreate = paramProvisionedThroughputOnCreate case ParameterAvailabilityClass: paramAvailabilityClass, err := ConvertStringToAvailabilityClass(v) if err != nil { - return p, fmt.Errorf("parameters contain invalid availability class parameter: %w", err) + return p, d, fmt.Errorf("parameters contain invalid availability class parameter: %w", err) } if paramAvailabilityClass == ParameterRegionalHardFailoverClass { p.ForceAttach = true @@ -208,37 +236,56 @@ func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string] case ParameterKeyEnableConfidentialCompute: paramEnableConfidentialCompute, err := ConvertStringToBool(v) if err != nil { - return p, fmt.Errorf("parameters contain invalid value for enable-confidential-storage parameter: %w", err) + return p, d, fmt.Errorf("parameters contain invalid value for enable-confidential-storage parameter: %w", err) } if paramEnableConfidentialCompute { // DiskEncryptionKmsKey is needed to enable confidentialStorage if val, ok := parameters[ParameterKeyDiskEncryptionKmsKey]; !ok || !isValidDiskEncryptionKmsKey(val) { - return p, fmt.Errorf("Valid %v is required to enable ConfidentialStorage", ParameterKeyDiskEncryptionKmsKey) + return p, d, fmt.Errorf("Valid %v is required to enable ConfidentialStorage", ParameterKeyDiskEncryptionKmsKey) } } p.EnableConfidentialCompute = paramEnableConfidentialCompute case ParameterKeyStoragePools: if !pp.EnableStoragePools { - return p, fmt.Errorf("parameters contains invalid option %q", ParameterKeyStoragePools) + return p, d, fmt.Errorf("parameters contains invalid option %q", ParameterKeyStoragePools) } storagePools, err := ParseStoragePools(v) if err != nil { - return p, fmt.Errorf("parameters contains invalid value for %s parameter %q: %w", ParameterKeyStoragePools, v, err) + return p, d, fmt.Errorf("parameters contains invalid value for %s parameter %q: %w", ParameterKeyStoragePools, v, err) } p.StoragePools = storagePools + case ParameterKeyDataCacheSize: + if !enableDataCache { + return p, d, fmt.Errorf("data caching enabled: %v; parameters contains invalid option %q", enableDataCache, ParameterKeyDataCacheSize) + } + // TODO: need to parse or validate the string + + paramDataCacheSize, err := ConvertGiStringToInt64(v) + if err != nil { + return p, d, fmt.Errorf("parameters contain invalid dataCacheSize parameter: %w", err) + } + d.DataCacheSize = strconv.FormatInt(paramDataCacheSize, 10) + case ParameterKeyDataCacheMode: + if !enableDataCache { + return p, d, fmt.Errorf("data caching enabled %v; parameters contains invalid option %q", enableDataCache, ParameterKeyDataCacheSize) + } + if err := ValidateDataCacheMode(v); err != nil { + return p, d, fmt.Errorf("parameters contains invalid option: %w", err) + } + d.DataCacheMode = v case ParameterKeyResourceTags: if err := extractResourceTagsParameter(v, p.ResourceTags); err != nil { - return p, err + return p, d, err } case ParameterKeyEnableMultiZoneProvisioning: if !pp.EnableMultiZone { - return p, fmt.Errorf("parameters contains invalid option %q", ParameterKeyEnableMultiZoneProvisioning) + return p, d, fmt.Errorf("parameters contains invalid option %q", ParameterKeyEnableMultiZoneProvisioning) } paramEnableMultiZoneProvisioning, err := ConvertStringToBool(v) if err != nil { - return p, fmt.Errorf("parameters contain invalid value for %s parameter: %w", ParameterKeyEnableMultiZoneProvisioning, err) + return p, d, fmt.Errorf("parameters contain invalid value for %s parameter: %w", ParameterKeyEnableMultiZoneProvisioning, err) } p.MultiZoneProvisioning = paramEnableMultiZoneProvisioning @@ -246,13 +293,13 @@ func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string] p.Labels[MultiZoneLabel] = "true" } default: - return p, fmt.Errorf("parameters contains invalid option %q", k) + return p, d, fmt.Errorf("parameters contains invalid option %q", k) } } if len(p.Tags) > 0 { p.Tags[tagKeyCreatedBy] = pp.DriverName } - return p, nil + return p, d, nil } func ExtractAndDefaultSnapshotParameters(parameters map[string]string, driverName string, extraTags map[string]string) (SnapshotParameters, error) { diff --git a/pkg/common/parameters_test.go b/pkg/common/parameters_test.go index 75f1c2a16..64cd4da08 100644 --- a/pkg/common/parameters_test.go +++ b/pkg/common/parameters_test.go @@ -25,14 +25,17 @@ import ( func TestExtractAndDefaultParameters(t *testing.T) { tests := []struct { - name string - parameters map[string]string - labels map[string]string - enableStoragePools bool - enableMultiZone bool - extraTags map[string]string - expectParams DiskParameters - expectErr bool + name string + parameters map[string]string + labels map[string]string + enableStoragePools bool + enableDataCache bool + enableMultiZone bool + enableHdHA bool + extraTags map[string]string + expectParams DiskParameters + expectDataCacheParams DataCacheParameters + expectErr bool }{ { name: "defaults", @@ -351,6 +354,55 @@ func TestExtractAndDefaultParameters(t *testing.T) { labels: map[string]string{}, expectErr: true, }, + { + name: "data cache parameters - set default cache mode", + enableDataCache: true, + parameters: map[string]string{ParameterKeyType: "pd-balanced", ParameterKeyReplicationType: "none", ParameterKeyDiskEncryptionKmsKey: "foo/key", ParameterKeyLabels: "key1=value1,key2=value2", ParameterKeyDataCacheSize: "1234Gi"}, + labels: map[string]string{}, + expectParams: DiskParameters{ + DiskType: "pd-balanced", + ReplicationType: "none", + DiskEncryptionKMSKey: "foo/key", + Tags: map[string]string{}, + Labels: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + ResourceTags: map[string]string{}, + }, + expectDataCacheParams: DataCacheParameters{ + DataCacheMode: DataCacheModeWriteThrough, + DataCacheSize: "1234", + }, + }, + { + name: "data cache parameters", + enableDataCache: true, + parameters: map[string]string{ParameterKeyType: "pd-balanced", ParameterKeyReplicationType: "none", ParameterKeyDiskEncryptionKmsKey: "foo/key", ParameterKeyLabels: "key1=value1,key2=value2", ParameterKeyDataCacheSize: "1234Gi", ParameterKeyDataCacheMode: DataCacheModeWriteBack}, + labels: map[string]string{}, + expectParams: DiskParameters{ + DiskType: "pd-balanced", + ReplicationType: "none", + DiskEncryptionKMSKey: "foo/key", + Tags: map[string]string{}, + Labels: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + ResourceTags: map[string]string{}, + }, + expectDataCacheParams: DataCacheParameters{ + DataCacheMode: DataCacheModeWriteBack, + DataCacheSize: "1234", + }, + }, + { + name: "data cache parameters - enableDataCache is false", + enableDataCache: false, + parameters: map[string]string{ParameterKeyType: "pd-balanced", ParameterKeyReplicationType: "none", ParameterKeyDiskEncryptionKmsKey: "foo/key", ParameterKeyLabels: "key1=value1,key2=value2", ParameterKeyDataCacheSize: "1234Gi", ParameterKeyDataCacheMode: DataCacheModeWriteBack}, + labels: map[string]string{}, + expectErr: true, + }, { name: "multi-zone-enable parameters, multi-zone label is set, multi-zone feature enabled", parameters: map[string]string{ParameterKeyType: "hyperdisk-ml", ParameterKeyEnableMultiZoneProvisioning: "true"}, @@ -397,7 +449,7 @@ func TestExtractAndDefaultParameters(t *testing.T) { EnableStoragePools: tc.enableStoragePools, EnableMultiZone: tc.enableMultiZone, } - p, err := pp.ExtractAndDefaultParameters(tc.parameters, tc.labels, tc.extraTags) + p, d, err := pp.ExtractAndDefaultParameters(tc.parameters, tc.labels, tc.enableDataCache, tc.extraTags) if gotErr := err != nil; gotErr != tc.expectErr { t.Fatalf("ExtractAndDefaultParameters(%+v) = %v; expectedErr: %v", tc.parameters, err, tc.expectErr) } @@ -408,6 +460,10 @@ func TestExtractAndDefaultParameters(t *testing.T) { if diff := cmp.Diff(tc.expectParams, p); diff != "" { t.Errorf("ExtractAndDefaultParameters(%+v): -want, +got \n%s", tc.parameters, diff) } + + if diff := cmp.Diff(tc.expectDataCacheParams, d); diff != "" { + t.Errorf("ExtractAndDefaultParameters(%+v) for data cache params: -want, +got \n%s", tc.parameters, diff) + } }) } } diff --git a/pkg/common/runcmd.go b/pkg/common/runcmd.go new file mode 100644 index 000000000..71240d2a9 --- /dev/null +++ b/pkg/common/runcmd.go @@ -0,0 +1,71 @@ +package common + +import ( + "fmt" + "os/exec" + "strings" + + "k8s.io/klog/v2" +) + +const ( + // Error thrown by exec cmd.Run() when process spawned by cmd.Start() completes before cmd.Wait() is called (see - k/k issue #103753) + errNoChildProcesses = "wait: no child processes" +) + +// RunCommand wraps a k8s exec to deal with the no child process error. Same as exec.CombinedOutput. +// On error, the output is included so callers don't need to echo it again. + +func RunCommand(pipeCmd string, pipeCmdArg string, cmd1 string, execCmdArgs ...string) ([]byte, error) { + execCmd1 := exec.Command(cmd1, execCmdArgs...) + + if pipeCmd != "" { + output, err := execPipeCommand(pipeCmd, pipeCmdArg, execCmd1) + if err != nil { + return nil, fmt.Errorf("%s %s failed here: %w; output: %s", pipeCmd, pipeCmdArg, err, string(output)) + } + return output, nil + } + output, err := execCmd1.CombinedOutput() + if err != nil { + err = checkError(err, *execCmd1) + return nil, fmt.Errorf("%s %s failed here 2: %w; output: %s", cmd1, strings.Join(execCmdArgs, " "), err, string(output)) + } + + return output, nil +} + +func checkError(err error, execCmd exec.Cmd) error { + if err.Error() == errNoChildProcesses { + if execCmd.ProcessState.Success() { + // If the process succeeded, this can be ignored, see k/k issue #103753 + return nil + } + // Get actual error + klog.Infof("Errored here") + err = &exec.ExitError{ProcessState: execCmd.ProcessState} + } + return err +} +func execPipeCommand(pipeCmd string, pipeCmdArg string, execCmd1 *exec.Cmd) ([]byte, error) { + + execPipeCmd := exec.Command(pipeCmd, pipeCmdArg) + stdoutPipe, err := execCmd1.StdoutPipe() + if err != nil { + klog.Errorf("failed command %v: got error:%v", execCmd1, err) + } + err = execCmd1.Start() + if err != nil { + klog.Infof("errored running command %v; error %v; ", execCmd1, err) + } + defer stdoutPipe.Close() + + execPipeCmd.Stdin = stdoutPipe + output, err := execPipeCmd.CombinedOutput() + if err != nil { + err = checkError(err, *execPipeCmd) + return nil, fmt.Errorf("%s failed: %w; output: %s", pipeCmd, err, string(output)) + } + + return output, nil +} diff --git a/pkg/common/utils.go b/pkg/common/utils.go index 1789dfd0c..73fa32243 100644 --- a/pkg/common/utils.go +++ b/pkg/common/utils.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "hash/fnv" "net/http" "regexp" "slices" @@ -78,6 +79,7 @@ const ( // Full or partial URL of the zone resource, in the format: // projects/{project}/zones/{zone} zoneURIPattern = "projects/[^/]+/zones/([^/]+)$" + alphanums = "bcdfghjklmnpqrstvwxz2456789" ) var ( @@ -101,6 +103,8 @@ var ( http.StatusConflict: codes.FailedPrecondition, } + validDataCacheMode = []string{DataCacheModeWriteBack, DataCacheModeWriteThrough} + // Regular expressions for validating parent_id, key and value of a resource tag. regexParent = regexp.MustCompile(`(^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$)`) regexKey = regexp.MustCompile(`^[a-zA-Z0-9]([0-9A-Za-z_.-]{0,61}[a-zA-Z0-9])?$`) @@ -392,6 +396,15 @@ func ConvertMiStringToInt64(str string) (int64, error) { return volumehelpers.RoundUpToMiB(quantity) } +// ConvertGiStringToInt64 converts a GiB string to int64 +func ConvertGiStringToInt64(str string) (int64, error) { + quantity, err := resource.ParseQuantity(str) + if err != nil { + return -1, err + } + return volumehelpers.RoundUpToGiB(quantity) +} + // ConvertStringToBool converts a string to a boolean. func ConvertStringToBool(str string) (bool, error) { switch strings.ToLower(str) { @@ -684,6 +697,22 @@ func VolumeIdAsMultiZone(volumeId string) (string, error) { return strings.Join(splitId, "/"), nil } +func StringInSlice(s string, list []string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} + +func ValidateDataCacheMode(s string) error { + if StringInSlice(s, validDataCacheMode) { + return nil + } + return fmt.Errorf("invalid data-cache-mode %s. Only \"writeback\" and \"writethrough\" is a valid input", s) +} + // NewLimiter returns a token bucket based request rate limiter after initializing // the passed values for limit, burst (or token bucket) size. If opted for emptyBucket // all initial tokens are reserved for the first burst. @@ -710,3 +739,16 @@ func MapNumber(num int64) int64 { } return 0 } + +// shortString is inspired by k8s.io/apimachinery/pkg/util/rand.SafeEncodeString, but takes data from a hash. +func ShortString(s string) string { + hasher := fnv.New128a() + hasher.Write([]byte(s)) + sum := hasher.Sum([]byte{}) + const sz = 8 + short := make([]byte, sz) + for i := 0; i < sz; i++ { + short[i] = alphanums[int(sum[i])%len(alphanums)] + } + return string(short) +} diff --git a/pkg/common/utils_test.go b/pkg/common/utils_test.go index 0df0ed1d1..e3323905d 100644 --- a/pkg/common/utils_test.go +++ b/pkg/common/utils_test.go @@ -1044,6 +1044,108 @@ func TestConvertMiStringToInt64(t *testing.T) { } } +func TestConvertGiStringToInt64(t *testing.T) { + tests := []struct { + desc string + inputStr string + expInt64 int64 + expectError bool + }{ + { + desc: "valid number string", + inputStr: "10000", + expInt64: 1, + expectError: false, + }, + { + desc: "round Ki to GiB", + inputStr: "1000000Ki", + expInt64: 1, + expectError: false, + }, + { + desc: "round k to GiB", + inputStr: "1000000k", + expInt64: 1, + expectError: false, + }, + { + desc: "round Mi to GiB", + inputStr: "1000Mi", + expInt64: 1, + expectError: false, + }, + { + desc: "round M to GiB", + inputStr: "1000M", + expInt64: 1, + expectError: false, + }, + { + desc: "round G to GiB", + inputStr: "1000G", + expInt64: 932, + expectError: false, + }, + { + desc: "round Gi to GiB - most common case", + inputStr: "1234Gi", + expInt64: 1234, + expectError: false, + }, + { + desc: "round decimal to GiB", + inputStr: "1.2Gi", + expInt64: 2, + expectError: false, + }, + { + desc: "round big value to GiB", + inputStr: "8191Pi", + expInt64: 8588886016, + expectError: false, + }, + { + desc: "invalid empty string", + inputStr: "", + expInt64: 0, + expectError: true, + }, + { + desc: "invalid KiB string", + inputStr: "10KiB", + expInt64: 10000, + expectError: true, + }, + { + desc: "invalid GB string", + inputStr: "10GB", + expInt64: 0, + expectError: true, + }, + { + desc: "invalid string", + inputStr: "ew%65", + expInt64: 0, + expectError: true, + }, + } + for _, tc := range tests { + t.Run(tc.desc, func(t *testing.T) { + actualInt64, err := ConvertGiStringToInt64(tc.inputStr) + if err != nil && !tc.expectError { + t.Errorf("Got error %v converting string to int64 %s; expect no error", err, tc.inputStr) + } + if err == nil && tc.expectError { + t.Errorf("Got no error converting string to int64 %s; expect an error", tc.inputStr) + } + if err == nil && actualInt64 != tc.expInt64 { + t.Errorf("Got %d for converting string to int64; expect %d", actualInt64, tc.expInt64) + } + }) + } +} + func TestConvertStringToBool(t *testing.T) { tests := []struct { desc string @@ -1657,6 +1759,70 @@ func TestUnorderedSlicesEqual(t *testing.T) { } } +func TestStringInSlice(t *testing.T) { + testCases := []struct { + name string + inputStr string + inputSlice []string + expectedInSlice bool + }{ + { + name: "string is in the slice", + inputStr: "in slice", + inputSlice: []string{"in slice", "other string"}, + expectedInSlice: true, + }, + { + name: "string is NOT in the slice", + inputStr: "not in slice", + inputSlice: []string{"other string"}, + }, + } + + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + actualResult := StringInSlice(tc.inputStr, tc.inputSlice) + if actualResult != tc.expectedInSlice { + t.Errorf("Expect value is %v but got %v. inputStr is %s, inputSlice is %v", tc.expectedInSlice, actualResult, tc.inputStr, tc.inputSlice) + } + } +} + +func TestValidateDataCacheMode(t *testing.T) { + testCases := []struct { + name string + inputStr string + expectError bool + }{ + { + name: "valid input - writethrough", + inputStr: "writethrough", + }, + { + name: "valid input - writeback", + inputStr: "writeback", + }, + { + name: "invalid input", + inputStr: "write-back not valid", + expectError: true, + }, + } + + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + err := ValidateDataCacheMode(tc.inputStr) + if err != nil && !tc.expectError { + t.Errorf("Got error %v validate data cache mode %s; expect no error", err, tc.inputStr) + } + + if err == nil && tc.expectError { + t.Errorf("Got no error validate data cache mode %s; expect an error", tc.inputStr) + } + } + +} + func TestParseZoneFromURI(t *testing.T) { testcases := []struct { name string diff --git a/pkg/deviceutils/device-utils.go b/pkg/deviceutils/device-utils.go index c9ce89885..096e157a3 100644 --- a/pkg/deviceutils/device-utils.go +++ b/pkg/deviceutils/device-utils.go @@ -231,7 +231,7 @@ func (m *deviceUtils) VerifyDevicePath(devicePaths []string, deviceName string) devicePath, innerErr = existingDevicePath(devicePaths) if innerErr != nil { e := fmt.Errorf("for disk %s failed to check for existing device path: %w", deviceName, innerErr) - klog.Errorf(e.Error()) + klog.Errorf("Error: %s", e.Error()) return false, e } @@ -243,7 +243,7 @@ func (m *deviceUtils) VerifyDevicePath(devicePaths []string, deviceName string) innerErr := udevadmTriggerForDiskIfExists(deviceName) if innerErr != nil { e := fmt.Errorf("for disk %s failed to trigger udevadm fix of non existent device path: %w", deviceName, innerErr) - klog.Errorf(e.Error()) + klog.Errorf("Error: %s", e.Error()) return false, e } // Go to next retry loop to get the deviceName again after @@ -256,7 +256,7 @@ func (m *deviceUtils) VerifyDevicePath(devicePaths []string, deviceName string) devFsPath, innerErr := filepath.EvalSymlinks(devicePath) if innerErr != nil { e := fmt.Errorf("filepath.EvalSymlinks(%q) failed: %w", devicePath, innerErr) - klog.Errorf(e.Error()) + klog.Errorf("Error: %s", e.Error()) return false, e } klog.V(4).Infof("For disk %s the /dev/* path is %s for disk/by-id path %s", deviceName, devFsPath, devicePath) @@ -264,7 +264,7 @@ func (m *deviceUtils) VerifyDevicePath(devicePaths []string, deviceName string) devFsSerial, innerErr := getDevFsSerial(devFsPath) if innerErr != nil { e := fmt.Errorf("couldn't get serial number for disk %s at device path %s: %w", deviceName, devFsPath, innerErr) - klog.Errorf(e.Error()) + klog.Errorf("Error: %s", e.Error()) return false, e } klog.V(4).Infof("For disk %s, device path %s, found serial number %s", deviceName, devFsPath, devFsSerial) @@ -281,7 +281,7 @@ func (m *deviceUtils) VerifyDevicePath(devicePaths []string, deviceName string) innerErr = udevadmTriggerForDiskIfExists(deviceName) if innerErr != nil { e := fmt.Errorf("failed to trigger udevadm fix of misconfigured disk for %q: %w", deviceName, innerErr) - klog.Errorf(e.Error()) + klog.Errorf("Error: %s", e.Error()) return false, e } // Go to next retry loop to get the deviceName again after diff --git a/pkg/gce-pd-csi-driver/cache.go b/pkg/gce-pd-csi-driver/cache.go new file mode 100644 index 000000000..7d0f65c29 --- /dev/null +++ b/pkg/gce-pd-csi-driver/cache.go @@ -0,0 +1,369 @@ +package gceGCEDriver + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + csi "github.com/container-storage-interface/spec/lib/go/csi" + + "k8s.io/klog/v2" + + "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" +) + +const ( + cacheSuffix = "csi-fast" + mainLvSuffix = "csi-main" + raidedLocalSsdName = "csi-driver-data-cache" + raidMode = "0" + raidedLssdPrefix = "/dev/md/" +) + +var raidedLocalSsdPath = raidedLssdPrefix + raidedLocalSsdName + +func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId string) (string, error) { + volumeId := req.GetVolumeId() + volumeGroupName := getVolumeGroupName(nodeId) + mainDevicePath := "/dev/" + volumeGroupName + "/" + getLvName(mainLvSuffix, volumeId) + mainLvName := getLvName(mainLvSuffix, volumeId) + klog.V(2).Infof("Volume group available on node %v ", volumeGroupName) + + info, err := common.RunCommand("grep", raidedLocalSsdName, "ls", raidedLssdPrefix) + if err != nil { + klog.Errorf("failed while listing raided devices, err: %v, output:%v", err, info) + } + infoString := strings.TrimSpace(string(info)) + raidedLocalSsdPath = raidedLssdPrefix + infoString + + vgExists := checkVgExists(volumeGroupName) + if vgExists { + // Clean up Volume Group before adding the PD + reduceVolumeGroup(volumeGroupName, true) + } else { + err := createVg(volumeGroupName, devicePath, raidedLocalSsdPath) + if err != nil { + return mainDevicePath, err + } + } + + // Check if the Physical Volume(PV) is part of some other volume group + args := []string{ + "--select", + "pv_name=" + devicePath, + "-o", + "vg_name", + } + info, err = common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "pvs", args...) + if err != nil { + klog.Errorf("errored while checking physical volume details %v: %s", err, info) + // On error info contains the error message which we cannot use for further steps + info = nil + } + + infoString = strings.TrimSpace(strings.ReplaceAll(string(info), "\n", " ")) + infoString = strings.ReplaceAll(infoString, ".", "") + infoString = strings.ReplaceAll(infoString, "\"", "") + infoSlice := strings.Split(strings.TrimSpace(infoString), " ") + vgNameForPv := strings.TrimSpace(infoSlice[(len(infoSlice) - 1)]) + if vgNameForPv == volumeGroupName { + klog.V(2).Infof("Physical Volume(PV) already exists in the Volume Group %v", volumeGroupName) + } else if vgNameForPv != "VG" && vgNameForPv != "" { + info, err = common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "vgchange", []string{"-an", vgNameForPv}...) + if err != nil { + klog.Errorf("Errored while deactivating VG %v: err: %v: %s", vgNameForPv, err, info) + } + // CLean up volume group to remove any dangling PV refrences + reduceVolumeGroup(vgNameForPv, false) + _, isCached := isCachingSetup(mainLvName) + // We will continue to uncache even if it errors to check caching as it is not a terminal issue. + if isCached { + // Uncache LV + args = []string{ + "--uncache", + vgNameForPv + "/" + mainLvName, + "--force", + "-y", // force remove cache without flushing data + } + info, err = common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "lvconvert", args...) + if err != nil { + return "", fmt.Errorf("errored while uncaching main LV. %v: %s", err, info) + } + // CLean up volume group to remove any dangling PV refrences + reduceVolumeGroup(vgNameForPv, false) + } + info, err = common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "vgmerge", []string{volumeGroupName, vgNameForPv}...) + if err != nil { + return "", fmt.Errorf("Errored while merging the PV Volume group %s into %s %v: %s", vgNameForPv, volumeGroupName, err, info) + } + + } else { + info, err := common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "vgextend", []string{volumeGroupName, devicePath}...) + if err != nil { + return "", fmt.Errorf("Errored while extending Volume group to add PV %v, error: %v: %s", devicePath, err, info) + } + } + + // Create LV if not already created + args = []string{ + "--select", + "vg_name=" + volumeGroupName, + "-o", + "lv_name", + } + lvList, err := common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "lvs", args...) + if err != nil { + return mainDevicePath, fmt.Errorf("Errored while checking logical volume for the device %s %w: %s", devicePath, err, info) + } + if !strings.Contains(string(lvList), mainLvName) { + args = []string{ + "--yes", + "-n", + mainLvName, + "-l", + "100%PVS", // Use 100% of the PV + volumeGroupName, + devicePath, + } + info, err = common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "lvcreate", args...) + if err != nil { + return mainDevicePath, fmt.Errorf("Errored setting up logical volume for the volume %s %w: %s", devicePath, err, info) + } + + } + err, isCached := isCachingSetup(mainLvName) + if err != nil { + klog.Errorf("faild to check if caching ius setup for LV, continuing to setup caching.") + } + cacheLvName := getLvName(cacheSuffix, volumeId) + if isCached { + // Validate that cache is setup for required size + klog.V(2).Infof("Assuming valid data cache size and mode, resizing cache is not supported") + } else { + fastCacheSize := req.GetPublishContext()[common.ContexLocalSsdCacheSize] + chunkSize := "960" // Cannot use default chunk size(64KiB) as it errors on maxChunksAllowed. Unit - KiB + args = []string{ + "--yes", + "-n", + cacheLvName, + "-L", + // ConvertGiStringToInt64 converts the input size to GiB so default to "g" for cache size - LVM g|G is GiB. + fastCacheSize + "g", + volumeGroupName, + raidedLocalSsdPath, + } + info, err = common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "lvcreate", args...) + if err != nil { + return mainDevicePath, fmt.Errorf("Errored while creating cache %w: %s", err, info) + } + + // Once caching is setup, link the PD to cache + args = []string{ + "--type", + "cache", + "--cachevol", + cacheLvName, + "--zero", + "y", + "--cachemode", + req.GetPublishContext()[common.ContextDataCacheMode], + volumeGroupName + "/" + mainLvName, + "--chunksize", + string(chunkSize), + "--force", + "-y", + } + info, err = common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "lvconvert", args...) + if err != nil { + return mainDevicePath, fmt.Errorf("Errored while setting up caching for volume %s %w: %s", devicePath, err, info) + } + } + + // activate all the LVs in the Volume group + info, err = common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "vgchange", []string{"-ay", volumeGroupName}...) + if err != nil { + // The logical volumes would not be accessible if the group is not activated + return mainDevicePath, fmt.Errorf("Failed to activate volume group %v %v:%s", volumeGroupName, err, info) + } + return mainDevicePath, nil +} + +func checkVgExists(volumeGroupName string) bool { + args := []string{} + info, err := common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "vgscan", args...) + if err != nil { + klog.Errorf("Errored while checking if volume group exists %v: %s", err, info) + return false + } + // Check if the required volume group already exists + return strings.Contains(string(info), volumeGroupName) +} + +func cleanupCache(volumeId string, nodeId string) error { + + volumeGroupName := getVolumeGroupName(nodeId) + if !checkVgExists(volumeGroupName) { + // If volume group doesn't exist then there's nothing to uncache + return nil + } + mainLvName := getLvName(mainLvSuffix, volumeId) + args := []string{ + "-an", + "/dev/" + volumeGroupName + "/" + mainLvName, + } + info, err := common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "lvchange", args...) + if err != nil { + return fmt.Errorf("Failed to deactivate volume for uncaching %s %v: %s", volumeId, err, info) + } + args = []string{ + "--uncache", + volumeGroupName + "/" + mainLvName, + "-y", + } + info, err = common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "lvconvert", args...) + if err != nil { + return fmt.Errorf("Failed to uncache volume %s %w: %s", volumeId, err, info) + } + return nil +} + +func getVolumeGroupName(nodePath string) string { + nodeSlice := strings.Split(nodePath, "/") + nodeId := nodeSlice[len(nodeSlice)-1] + nodeHash := common.ShortString(nodeId) + return fmt.Sprintf("csi-vg-%s", nodeHash) +} + +func getLvName(suffix string, volumeId string) string { + pvcNameStringSlice := strings.Split(volumeId, "/") + pvcName := pvcNameStringSlice[len(pvcNameStringSlice)-1] + return fmt.Sprintf("%s-%s", suffix, pvcName) +} + +func createVg(volumeGroupName string, devicePath string, raidedLocalSsds string) error { + klog.V(2).Infof(" vgcreate=") + args := []string{ + "--zero", + "y", + volumeGroupName, + raidedLocalSsds, + "-v", + } + info, err := common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "vgcreate", args...) + if err != nil { + return fmt.Errorf("Volume group creation failed %w: %s", err, info) + } + klog.Infof("Volume group creation succeeded for %v", volumeGroupName) + + args = []string{} + info, err = common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "vgscan", args...) + if err != nil { + klog.Errorf("Failed to scan for volume group post creation, continuing: %v: %s", err, info) + } + return nil +} + +func reduceVolumeGroup(volumeGroupName string, force bool) { + args := []string{ + "--removemissing", + volumeGroupName, + } + if force { + args = append(args, "--force") + } + info, err := common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "vgreduce", args...) + if err != nil { + klog.Errorf("Errored while cleaning up volume group %v: %s", err, info) + } +} + +func RaidLocalSsds() error { + isAlreadyRaided, err := isRaided() + if err != nil { + klog.V(2).Infof("Errored while scanning for available LocalSSDs err:%v; continuing Raiding", err) + } else if isAlreadyRaided { + klog.V(2).Infof("Local SSDs are already RAIDed, no further action needed here") + return nil + } + diskList := []string{} + info, err := common.RunCommand("" /* pipedCmd */, "" /* pipeCmdArg */, "lsblk", []string{"-o", "NAME,MODEL", "-p", "-d", "-n"}...) + if err != nil { + return fmt.Errorf("Failed to fetch LSSD info: %v; err:%v", info, err) + } + infoList := strings.Split(strings.TrimSpace(string(info)), "\n") + re, err := regexp.Compile("nvme_card([0-9]+)?$") + if err != nil { + return fmt.Errorf("Errored while compiling to check PD or LSSD %s", err) + } + for _, ssd := range infoList { + ssd = strings.TrimSpace(ssd) + if strings.HasPrefix(ssd, "/dev/nvme") { + ssdDetails := strings.Split(ssd, " ") + lssd := re.MatchString(ssdDetails[1]) + if lssd { + diskList = append(diskList, strings.TrimSpace(ssdDetails[0])) + } + } + } + nvmeDiskCount := len(diskList) + if nvmeDiskCount == 0 { + return fmt.Errorf("No local SSDs found for raiding") + } + args := []string{ + "--create", + raidedLssdPrefix + raidedLocalSsdName, + "-l" + raidMode, + // Force RAIDing as sometime it might fail for caution if there is just 1 LSSD present as 1 LSSD need not be RAIDed + "--force", + "-n", + strconv.Itoa(nvmeDiskCount), + } + args = append(args, diskList...) + info, err = common.RunCommand("" /* pipedCmd */, "" /* pipeCmdArg */, "mdadm", args...) + if err != nil { + return fmt.Errorf("errored while RAIDing LSSDs info: %v; err:%v", info, err) + } + // Validate if Raided successfully + isAlreadyRaided, err = isRaided() + if err != nil { + klog.V(2).Infof("Errored while scanning for available raided LocalSSDs err:%v=", err) + } + if !isAlreadyRaided { + return fmt.Errorf("failed raiding, raided device not found on scanning") + } + return nil +} + +func isRaided() (bool, error) { + args := []string{ + "--detail", + "--scan", + } + info, err := common.RunCommand("" /* pipedCmd */, "" /* pipeCmdArg */, "mdadm", args...) + if err != nil { + return false, fmt.Errorf("errored while scanning for raided LSSD %v: %s", err, info) + } + if info != nil && strings.Contains(string(info), raidedLocalSsdName) { + return true, nil + } + return false, nil +} + +func isCachingSetup(mainLvName string) (error, bool) { + // Verify caching is setup for PD + args := []string{ + "--select", + "lv_name=" + mainLvName, + "-o", + "pool_lv", + } + poolName, err := common.RunCommand("" /* pipedCmd */, "" /* pipeCmdArg */, "lvs", args...) + if err != nil { + return fmt.Errorf("Failed to check if caching is setup %w", err), false + } + if strings.Contains(string(poolName), "csi-fast") { + return nil, true + } + return nil, false +} diff --git a/pkg/gce-pd-csi-driver/controller.go b/pkg/gce-pd-csi-driver/controller.go index a2d57000d..a2172bef7 100644 --- a/pkg/gce-pd-csi-driver/controller.go +++ b/pkg/gce-pd-csi-driver/controller.go @@ -105,6 +105,9 @@ type GCEControllerServer struct { // If set to true, the CSI Driver will allow volumes to be provisioned in Storage Pools. enableStoragePools bool + // If set to true, the CSI Driver will allow volumes to be provisioned with data cache configuration + enableDataCache bool + multiZoneVolumeHandleConfig MultiZoneVolumeHandleConfig listVolumesConfig ListVolumesConfig @@ -298,6 +301,14 @@ func (gceCS *GCEControllerServer) createVolumeInternal(ctx context.Context, req defer func() { gceCS.Metrics.RecordOperationErrorMetrics("CreateVolume", err, diskTypeForMetric, enableConfidentialCompute, enableStoragePools) }() + + // Apply Parameters (case-insensitive). We leave validation of + // the values to the cloud provider. + params, dataCacheParams, err := gceCS.parameterProcessor().ExtractAndDefaultParameters(req.GetParameters(), gceCS.Driver.extraVolumeLabels, gceCS.enableDataCache, gceCS.Driver.extraTags) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "failed to extract parameters: %v", err.Error()) + } + // Validate arguments volumeCapabilities := req.GetVolumeCapabilities() capacityRange := req.GetCapacityRange() @@ -320,7 +331,6 @@ func (gceCS *GCEControllerServer) createVolumeInternal(ctx context.Context, req // Apply Parameters (case-insensitive). We leave validation of // the values to the cloud provider. - params, err := gceCS.parameterProcessor().ExtractAndDefaultParameters(req.GetParameters(), gceCS.Driver.extraVolumeLabels, gceCS.Driver.extraTags) diskTypeForMetric = params.DiskType enableConfidentialCompute = strconv.FormatBool(params.EnableConfidentialCompute) hasStoragePools := len(params.StoragePools) > 0 @@ -359,11 +369,11 @@ func (gceCS *GCEControllerServer) createVolumeInternal(ctx context.Context, req if gceCS.multiZoneVolumeHandleConfig.Enable && params.MultiZoneProvisioning { // Create multi-zone disk, that may have up to N disks. - return gceCS.createMultiZoneDisk(ctx, req, params) + return gceCS.createMultiZoneDisk(ctx, req, params, dataCacheParams, gceCS.enableDataCache) } // Create single device zonal or regional disk - return gceCS.createSingleDeviceDisk(ctx, req, params) + return gceCS.createSingleDeviceDisk(ctx, req, params, dataCacheParams, gceCS.enableDataCache) } func (gceCS *GCEControllerServer) getSupportedZonesForPDType(ctx context.Context, zones []string, diskType string) ([]string, error) { @@ -418,7 +428,7 @@ func (gceCS *GCEControllerServer) getMultiZoneProvisioningZones(ctx context.Cont return combinedZones.List(), nil } -func (gceCS *GCEControllerServer) createMultiZoneDisk(ctx context.Context, req *csi.CreateVolumeRequest, params common.DiskParameters) (*csi.CreateVolumeResponse, error) { +func (gceCS *GCEControllerServer) createMultiZoneDisk(ctx context.Context, req *csi.CreateVolumeRequest, params common.DiskParameters, dataCacheParams common.DataCacheParameters, enableDataCache bool) (*csi.CreateVolumeResponse, error) { // Determine the zones that are needed. var err error @@ -464,7 +474,7 @@ func (gceCS *GCEControllerServer) createMultiZoneDisk(ctx context.Context, req * // Use the first response as a template volumeId := fmt.Sprintf("projects/%s/zones/%s/disks/%s", gceCS.CloudProvider.GetDefaultProject(), common.MultiZoneValue, req.GetName()) klog.V(4).Infof("CreateVolume succeeded for multi-zone disks in zones %s: %v", zones, multiZoneVolKey) - return generateCreateVolumeResponseWithVolumeId(createdDisks[0], zones, params, volumeId), nil + return generateCreateVolumeResponseWithVolumeId(createdDisks[0], zones, params, dataCacheParams, enableDataCache, volumeId), nil } func (gceCS *GCEControllerServer) getZonesWithDiskNameAndType(ctx context.Context, name string, diskType string) ([]string, error) { @@ -508,7 +518,7 @@ func (gceCS *GCEControllerServer) updateAccessModeIfNecessary(ctx context.Contex return gceCS.CloudProvider.SetDiskAccessMode(ctx, project, volKey, readOnlyManyAccessMode) } -func (gceCS *GCEControllerServer) createSingleDeviceDisk(ctx context.Context, req *csi.CreateVolumeRequest, params common.DiskParameters) (*csi.CreateVolumeResponse, error) { +func (gceCS *GCEControllerServer) createSingleDeviceDisk(ctx context.Context, req *csi.CreateVolumeRequest, params common.DiskParameters, dataCacheParams common.DataCacheParameters, enableDataCache bool) (*csi.CreateVolumeResponse, error) { var err error var locationTopReq *locationRequirements if useVolumeCloning(req) { @@ -560,7 +570,7 @@ func (gceCS *GCEControllerServer) createSingleDeviceDisk(ctx context.Context, re return nil, common.LoggedError("CreateVolume failed: %v", err) } - return generateCreateVolumeResponseWithVolumeId(disk, zones, params, volumeID), err + return generateCreateVolumeResponseWithVolumeId(disk, zones, params, dataCacheParams, enableDataCache, volumeID), err } func (gceCS *GCEControllerServer) createSingleDisk(ctx context.Context, req *csi.CreateVolumeRequest, params common.DiskParameters, volKey *meta.Key, zones []string) (*gce.CloudDisk, error) { @@ -962,6 +972,15 @@ func (gceCS *GCEControllerServer) executeControllerPublishVolume(ctx context.Con PublishContext: nil, } + // Set data cache publish context + if gceCS.enableDataCache && req.GetVolumeContext() != nil { + if req.GetVolumeContext()[common.ContextDataCacheSize] != "" { + pubVolResp.PublishContext = map[string]string{} + pubVolResp.PublishContext[common.ContexLocalSsdCacheSize] = req.GetVolumeContext()[common.ContextDataCacheSize] + pubVolResp.PublishContext[common.ContextDataCacheMode] = req.GetVolumeContext()[common.ContextDataCacheMode] + } + } + instanceZone, instanceName, err := common.NodeIDToZoneAndName(nodeID) if err != nil { return nil, status.Errorf(codes.NotFound, "could not split nodeID: %v", err.Error()), nil @@ -1232,10 +1251,11 @@ func (gceCS *GCEControllerServer) ValidateVolumeCapabilities(ctx context.Context } // Validate the disk parameters match the disk we GET - params, err := gceCS.parameterProcessor().ExtractAndDefaultParameters(req.GetParameters(), gceCS.Driver.extraVolumeLabels, gceCS.Driver.extraTags) + params, _, err := gceCS.parameterProcessor().ExtractAndDefaultParameters(req.GetParameters(), gceCS.Driver.extraVolumeLabels, gceCS.enableDataCache, gceCS.Driver.extraTags) if err != nil { return nil, status.Errorf(codes.InvalidArgument, "failed to extract parameters: %v", err.Error()) } + if err := gce.ValidateDiskParameters(disk, params); err != nil { return generateFailedValidationMessage("Parameters %v do not match given disk %s: %v", req.GetParameters(), disk.GetName(), err.Error()), nil } @@ -2270,7 +2290,7 @@ func extractVolumeContext(context map[string]string) (*PDCSIContext, error) { return info, nil } -func generateCreateVolumeResponseWithVolumeId(disk *gce.CloudDisk, zones []string, params common.DiskParameters, volumeId string) *csi.CreateVolumeResponse { +func generateCreateVolumeResponseWithVolumeId(disk *gce.CloudDisk, zones []string, params common.DiskParameters, dataCacheParams common.DataCacheParameters, enableDataCache bool, volumeId string) *csi.CreateVolumeResponse { tops := []*csi.Topology{} for _, zone := range zones { tops = append(tops, &csi.Topology{ @@ -2286,6 +2306,14 @@ func generateCreateVolumeResponseWithVolumeId(disk *gce.CloudDisk, zones []strin AccessibleTopology: tops, }, } + // Set data cache volume context + if enableDataCache && dataCacheParams != (common.DataCacheParameters{}) { + if createResp.Volume.VolumeContext == nil { + createResp.Volume.VolumeContext = map[string]string{} + } + createResp.Volume.VolumeContext[common.ContextDataCacheMode] = dataCacheParams.DataCacheMode + createResp.Volume.VolumeContext[common.ContextDataCacheSize] = dataCacheParams.DataCacheSize + } snapshotID := disk.GetSnapshotId() imageID := disk.GetImageId() diskID := disk.GetSourceDiskId() diff --git a/pkg/gce-pd-csi-driver/gce-pd-driver.go b/pkg/gce-pd-csi-driver/gce-pd-driver.go index ba1a0c0ba..28f6a56c9 100644 --- a/pkg/gce-pd-csi-driver/gce-pd-driver.go +++ b/pkg/gce-pd-csi-driver/gce-pd-driver.go @@ -143,7 +143,7 @@ func NewIdentityServer(gceDriver *GCEDriver) *GCEIdentityServer { } } -func NewNodeServer(gceDriver *GCEDriver, mounter *mount.SafeFormatAndMount, deviceUtils deviceutils.DeviceUtils, meta metadataservice.MetadataService, statter mountmanager.Statter) *GCENodeServer { +func NewNodeServer(gceDriver *GCEDriver, mounter *mount.SafeFormatAndMount, deviceUtils deviceutils.DeviceUtils, meta metadataservice.MetadataService, statter mountmanager.Statter, args NodeServerArgs) *GCENodeServer { return &GCENodeServer{ Driver: gceDriver, Mounter: mounter, @@ -151,10 +151,11 @@ func NewNodeServer(gceDriver *GCEDriver, mounter *mount.SafeFormatAndMount, devi MetadataService: meta, volumeLocks: common.NewVolumeLocks(), VolumeStatter: statter, + EnableDataCache: args.EnableDataCache, } } -func NewControllerServer(gceDriver *GCEDriver, cloudProvider gce.GCECompute, errorBackoffInitialDuration, errorBackoffMaxDuration time.Duration, fallbackRequisiteZones []string, enableStoragePools bool, multiZoneVolumeHandleConfig MultiZoneVolumeHandleConfig, listVolumesConfig ListVolumesConfig) *GCEControllerServer { +func NewControllerServer(gceDriver *GCEDriver, cloudProvider gce.GCECompute, errorBackoffInitialDuration, errorBackoffMaxDuration time.Duration, fallbackRequisiteZones []string, enableStoragePools bool, enableDataCache bool, multiZoneVolumeHandleConfig MultiZoneVolumeHandleConfig, listVolumesConfig ListVolumesConfig) *GCEControllerServer { return &GCEControllerServer{ Driver: gceDriver, CloudProvider: cloudProvider, @@ -163,6 +164,7 @@ func NewControllerServer(gceDriver *GCEDriver, cloudProvider gce.GCECompute, err errorBackoff: newCsiErrorBackoff(errorBackoffInitialDuration, errorBackoffMaxDuration), fallbackRequisiteZones: fallbackRequisiteZones, enableStoragePools: enableStoragePools, + enableDataCache: enableDataCache, multiZoneVolumeHandleConfig: multiZoneVolumeHandleConfig, listVolumesConfig: listVolumesConfig, } diff --git a/pkg/gce-pd-csi-driver/gce-pd-driver_test.go b/pkg/gce-pd-csi-driver/gce-pd-driver_test.go index 15306c141..3789b7bc6 100644 --- a/pkg/gce-pd-csi-driver/gce-pd-driver_test.go +++ b/pkg/gce-pd-csi-driver/gce-pd-driver_test.go @@ -41,17 +41,23 @@ func initBlockingGCEDriver(t *testing.T, cloudDisks []*gce.CloudDisk, readyToExe return initGCEDriverWithCloudProvider(t, fakeBlockingBlockProvider) } -func initGCEDriverWithCloudProvider(t *testing.T, cloudProvider gce.GCECompute) *GCEDriver { - vendorVersion := "test-vendor" +func controllerServerForTest(cloudProvider gce.GCECompute) *GCEControllerServer { gceDriver := GetGCEDriver() errorBackoffInitialDuration := 200 * time.Millisecond errorBackoffMaxDuration := 5 * time.Minute fallbackRequisiteZones := []string{} enableStoragePools := false + enableDataCache := false multiZoneVolumeHandleConfig := MultiZoneVolumeHandleConfig{} listVolumesConfig := ListVolumesConfig{} - controllerServer := NewControllerServer(gceDriver, cloudProvider, errorBackoffInitialDuration, errorBackoffMaxDuration, fallbackRequisiteZones, enableStoragePools, multiZoneVolumeHandleConfig, listVolumesConfig) + return NewControllerServer(gceDriver, cloudProvider, errorBackoffInitialDuration, errorBackoffMaxDuration, fallbackRequisiteZones, enableStoragePools, enableDataCache, multiZoneVolumeHandleConfig, listVolumesConfig) +} + +func initGCEDriverWithCloudProvider(t *testing.T, cloudProvider gce.GCECompute) *GCEDriver { + vendorVersion := "test-vendor" + gceDriver := GetGCEDriver() + controllerServer := controllerServerForTest(cloudProvider) err := gceDriver.SetupGCEDriver(driver, vendorVersion, nil, nil, nil, controllerServer, nil) if err != nil { t.Fatalf("Failed to setup GCE Driver: %v", err) diff --git a/pkg/gce-pd-csi-driver/node.go b/pkg/gce-pd-csi-driver/node.go index b10471e3d..c8170b27e 100644 --- a/pkg/gce-pd-csi-driver/node.go +++ b/pkg/gce-pd-csi-driver/node.go @@ -47,6 +47,7 @@ type GCENodeServer struct { DeviceUtils deviceutils.DeviceUtils VolumeStatter mountmanager.Statter MetadataService metadataservice.MetadataService + EnableDataCache bool // A map storing all volumes with ongoing operations so that additional operations // for that same volume (as defined by VolumeID) return an Aborted error @@ -60,6 +61,14 @@ type GCENodeServer struct { // been observed). formatAndMountSemaphore chan any formatAndMountTimeout time.Duration + + // Embed UnimplementedNodeServer to ensure the driver returns Unimplemented for any + // new RPC methods that might be introduced in future versions of the spec. + csi.UnimplementedNodeServer +} + +type NodeServerArgs struct { + EnableDataCache bool } var _ csi.NodeServer = &GCENodeServer{} @@ -279,6 +288,7 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage volumeID := req.GetVolumeId() stagingTargetPath := req.GetStagingTargetPath() volumeCapability := req.GetVolumeCapability() + nodeId := ns.MetadataService.GetName() if len(volumeID) == 0 { return nil, status.Error(codes.InvalidArgument, "NodeStageVolume Volume ID must be provided") } @@ -312,12 +322,25 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage partition = part } devicePath, err := getDevicePath(ns, volumeID, partition) - if err != nil { return nil, status.Error(codes.Internal, fmt.Sprintf("Error when getting device path: %v", err.Error())) } - klog.V(4).Infof("Successfully found attached GCE PD %q at device path %s.", volumeKey.Name, devicePath) + klog.Infof("Successfully found attached GCE PD %q at device path %s.", volumeKey.Name, devicePath) + + if ns.EnableDataCache && req.GetPublishContext()[common.ContexLocalSsdCacheSize] != "" { + if len(nodeId) == 0 { + return nil, status.Error(codes.InvalidArgument, "NodeStageVolume Node ID must be provided") + } + devFsPath, err := filepath.EvalSymlinks(devicePath) + if err != nil { + klog.Errorf("filepath.EvalSymlinks(%q) failed when trying to create volume group: %v", devicePath, err) + } + devicePath, err = setupCaching(devFsPath, req, nodeId) + if err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("Error setting up cache: %v", err.Error())) + } + } // Part 2: Check if mount already exists at stagingTargetPath if ns.isVolumePathMounted(stagingTargetPath) { @@ -465,6 +488,15 @@ func (ns *GCENodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUns } } + // The NodeUnstageVolume does not have any volume or publish context, we need to get the info from LVM locally + // Check if cache group cache-{volumeID} exist in LVM + if ns.EnableDataCache { + nodeId := ns.MetadataService.GetName() + err := cleanupCache(volumeID, nodeId) + if err != nil { + klog.Errorf("Failed to cleanup cache for volume %s: %v", volumeID, err) + } + } klog.V(4).Infof("NodeUnstageVolume succeeded on %v from %s", volumeID, stagingTargetPath) return &csi.NodeUnstageVolumeResponse{}, nil } diff --git a/pkg/gce-pd-csi-driver/node_test.go b/pkg/gce-pd-csi-driver/node_test.go index 7e59ea7e0..80e5c28ec 100644 --- a/pkg/gce-pd-csi-driver/node_test.go +++ b/pkg/gce-pd-csi-driver/node_test.go @@ -51,7 +51,8 @@ func getTestGCEDriverWithCustomMounter(t *testing.T, mounter *mount.SafeFormatAn func getCustomTestGCEDriver(t *testing.T, mounter *mount.SafeFormatAndMount, deviceUtils deviceutils.DeviceUtils, metaService metadataservice.MetadataService) *GCEDriver { gceDriver := GetGCEDriver() - nodeServer := NewNodeServer(gceDriver, mounter, deviceUtils, metaService, mountmanager.NewFakeStatter(mounter)) + enableDataCache := false + nodeServer := NewNodeServer(gceDriver, mounter, deviceUtils, metaService, mountmanager.NewFakeStatter(mounter), NodeServerArgs{enableDataCache}) err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, nil, nil, nodeServer) if err != nil { t.Fatalf("Failed to setup GCE Driver: %v", err) @@ -62,7 +63,7 @@ func getCustomTestGCEDriver(t *testing.T, mounter *mount.SafeFormatAndMount, dev func getTestBlockingMountGCEDriver(t *testing.T, readyToExecute chan chan struct{}) *GCEDriver { gceDriver := GetGCEDriver() mounter := mountmanager.NewFakeSafeBlockingMounter(readyToExecute) - nodeServer := NewNodeServer(gceDriver, mounter, deviceutils.NewFakeDeviceUtils(false), metadataservice.NewFakeService(), mountmanager.NewFakeStatter(mounter)) + nodeServer := NewNodeServer(gceDriver, mounter, deviceutils.NewFakeDeviceUtils(false), metadataservice.NewFakeService(), mountmanager.NewFakeStatter(mounter), NodeServerArgs{true}) err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, nil, nil, nodeServer) if err != nil { t.Fatalf("Failed to setup GCE Driver: %v", err) @@ -72,8 +73,9 @@ func getTestBlockingMountGCEDriver(t *testing.T, readyToExecute chan chan struct func getTestBlockingFormatAndMountGCEDriver(t *testing.T, readyToExecute chan chan struct{}) *GCEDriver { gceDriver := GetGCEDriver() + enableDataCache := true mounter := mountmanager.NewFakeSafeBlockingMounter(readyToExecute) - nodeServer := NewNodeServer(gceDriver, mounter, deviceutils.NewFakeDeviceUtils(false), metadataservice.NewFakeService(), mountmanager.NewFakeStatter(mounter)).WithSerializedFormatAndMount(5*time.Second, 1) + nodeServer := NewNodeServer(gceDriver, mounter, deviceutils.NewFakeDeviceUtils(false), metadataservice.NewFakeService(), mountmanager.NewFakeStatter(mounter), NodeServerArgs{enableDataCache}).WithSerializedFormatAndMount(5*time.Second, 1) err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, nil, nil, nodeServer) if err != nil { diff --git a/test/e2e/tests/multi_zone_e2e_test.go b/test/e2e/tests/multi_zone_e2e_test.go index fb1ea32dd..456c781e6 100644 --- a/test/e2e/tests/multi_zone_e2e_test.go +++ b/test/e2e/tests/multi_zone_e2e_test.go @@ -332,7 +332,7 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { }() // Attach Disk - err := testAttachWriteReadDetach(underSpecifiedID, snapshotVolName, tc0.Instance, controllerClient, false /* readOnly */) + err := testAttachWriteReadDetach(underSpecifiedID, snapshotVolName, tc0.Instance, controllerClient, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle") // Create Snapshot @@ -413,11 +413,11 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { Expect(disk2.AccessMode).To(Equal("READ_ONLY_MANY")) // Attach Disk to node1 and validate contents - err = testAttachWriteReadDetach(volID, volName, tc0.Instance, tc0.Client, true /* readonly */) + err = testAttachWriteReadDetach(volID, volName, tc0.Instance, tc0.Client, true /* readonly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to attach/read/detach on vol1") // Attach Disk to node1 and validate contents - err = testAttachWriteReadDetach(volID, volName, tc1.Instance, tc1.Client, true /* readonly */) + err = testAttachWriteReadDetach(volID, volName, tc1.Instance, tc1.Client, true /* readonly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to attach/read/detach on vol2") disk1, err = computeService.Disks.Get(p, zones[0], volName).Do() @@ -479,7 +479,7 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { }() // Attach Disk - err := testAttachWriteReadDetach(underSpecifiedID, snapshotVolName, tc0.Instance, controllerClient, false /* readOnly */) + err := testAttachWriteReadDetach(underSpecifiedID, snapshotVolName, tc0.Instance, controllerClient, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle") // Create Snapshot @@ -551,11 +551,11 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { Expect(disk2.AccessMode).To(Equal("READ_ONLY_MANY")) // Attach Disk to node1 and validate contents - err = testAttachWriteReadDetach(volID, volName, tc0.Instance, tc0.Client, true /* readonly */) + err = testAttachWriteReadDetach(volID, volName, tc0.Instance, tc0.Client, true /* readonly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to attach/read/detach on vol1") // Attach Disk to node1 and validate contents - err = testAttachWriteReadDetach(volID, volName, tc1.Instance, tc1.Client, true /* readonly */) + err = testAttachWriteReadDetach(volID, volName, tc1.Instance, tc1.Client, true /* readonly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to attach/read/detach on vol2") disk1, err = computeService.Disks.Get(p, zones[0], volName).Do() @@ -617,7 +617,7 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { }() // Attach Disk - err := testAttachWriteReadDetach(underSpecifiedID, snapshotVolName, tc0.Instance, controllerClient, false /* readOnly */) + err := testAttachWriteReadDetach(underSpecifiedID, snapshotVolName, tc0.Instance, controllerClient, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle") // Create Disk Image @@ -707,11 +707,11 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { Expect(disk2.AccessMode).To(Equal("READ_ONLY_MANY")) // Attach Disk to node1 - err = testAttachWriteReadDetach(volID, volName, tc0.Instance, tc0.Client, true /* readonly */) + err = testAttachWriteReadDetach(volID, volName, tc0.Instance, tc0.Client, true /* readonly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to attach/read/detach on vol1") // Attach Disk to node1 - err = testAttachWriteReadDetach(volID, volName, tc1.Instance, tc1.Client, true /* readonly */) + err = testAttachWriteReadDetach(volID, volName, tc1.Instance, tc1.Client, true /* readonly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to attach/read/detach on vol2") }) @@ -806,18 +806,18 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { volID0 := fmt.Sprintf("projects/%s/zones/%s/disks/%s", p, zones[0], volName) volID1 := fmt.Sprintf("projects/%s/zones/%s/disks/%s", p, zones[1], volName) - err = testAttachWriteReadDetach(volID0, volName, tc0.Instance, tc0.Client, false /* readonly */) + err = testAttachWriteReadDetach(volID0, volName, tc0.Instance, tc0.Client, false /* readonly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to attach/write/read/detach on vol1") - err = testAttachWriteReadDetach(volID1, volName, tc1.Instance, tc1.Client, false /* readonly */) + err = testAttachWriteReadDetach(volID1, volName, tc1.Instance, tc1.Client, false /* readonly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to attach/write/read/detach on vol2") // Validate disks can be used in multi-zone mode on both nodes volIDMultiZone := fmt.Sprintf("projects/%s/zones/multi-zone/disks/%s", p, volName) - err = testAttachWriteReadDetach(volIDMultiZone, volName, tc0.Instance, tc0.Client, true /* readonly */) + err = testAttachWriteReadDetach(volIDMultiZone, volName, tc0.Instance, tc0.Client, true /* readonly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to attach/read/detach on vol1") - err = testAttachWriteReadDetach(volIDMultiZone, volName, tc1.Instance, tc1.Client, true /* readonly */) + err = testAttachWriteReadDetach(volIDMultiZone, volName, tc1.Instance, tc1.Client, true /* readonly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to attach/read/detach on vol2") // Validate disks are ROX now @@ -910,7 +910,7 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { if i >= 1 { readOnly = true } - err = testAttachWriteReadDetach(volume.VolumeId, volName, testContext.Instance, testContext.Client, readOnly) + err = testAttachWriteReadDetach(volume.VolumeId, volName, testContext.Instance, testContext.Client, readOnly, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "failed volume lifecycle checks") i = i + 1 } @@ -998,9 +998,10 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { // Attach disk to instance in the first zone. tc0 := zoneToContext[zones[0]] err, detacher, args := testAttachAndMount(volume.VolumeId, volName, tc0.Instance, tc0.Client, attachAndMountArgs{ - readOnly: false, - useBlock: false, - forceAttach: false, + readOnly: false, + useBlock: false, + forceAttach: false, + setupDataCache: false, }) detachers = append(detachers, detacher) Expect(err).To(BeNil(), "failed attach in zone 0") @@ -1018,9 +1019,10 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { // Now force attach to the second instance without detaching. tc1 := zoneToContext[zones[1]] err, detacher, _ = testAttachAndMount(volume.VolumeId, volName, tc1.Instance, tc1.Client, attachAndMountArgs{ - readOnly: false, - useBlock: false, - forceAttach: true, + readOnly: false, + useBlock: false, + forceAttach: true, + setupDataCache: false, }) detachers = append(detachers, detacher) Expect(err).To(BeNil(), "failed force attach in zone 1") @@ -1040,7 +1042,12 @@ func deleteDisk(controllerClient *remote.CsiClient, p, zone, volID, volName stri Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") } -func testAttachWriteReadDetach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool) error { +func testAttachWriteReadDetach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool, detachAndReattach bool, setupDataCache bool) error { + writeFile, verifyReadFile := testWriteAndReadFile(instance, readOnly) + return testLifecycleWithVerify(volID, volName, instance, client, readOnly, false /* fs */, writeFile, verifyReadFile, detachAndReattach, setupDataCache) +} + +func testWriteAndReadFile(instance *remote.InstanceInfo, readOnly bool) (verifyFunc, verifyFunc) { var testFileContents = "test" writeFile := func(a *verifyArgs) error { if !readOnly { @@ -1066,46 +1073,61 @@ func testAttachWriteReadDetach(volID string, volName string, instance *remote.In } return nil } - return testLifecycleWithVerify(volID, volName, instance, client, readOnly, false /* fs */, writeFile, verifyReadFile) + return writeFile, verifyReadFile } type attachAndMountArgs struct { - readOnly bool - useBlock bool - forceAttach bool + readOnly bool + useBlock bool + forceAttach bool + setupDataCache bool } func testAttachAndMount(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, args attachAndMountArgs) (error, func(), *verifyArgs) { + klog.Infof("Starting testAttachAndMount with volume %v node %v \n", volID, instance.GetNodeID()) + err, unstageAndDetach, stageDir := testAttach(volID, volName, instance, client, args) + if err != nil { + return err, nil, nil + } + // Mount Disk + err, unpublish, returnArgs := testMount(volID, volName, instance, client, args, stageDir) + if err != nil { + unstageAndDetach() + return err, nil, nil + } + unpublishUnstageAndDetach := func() { + unpublish() + unstageAndDetach() + } + return nil, unpublishUnstageAndDetach, returnArgs +} + +func testAttach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, args attachAndMountArgs) (error, func(), string) { + klog.Infof("Starting testAttach with volume %v node %v \n", volID, instance.GetNodeID()) // Attach Disk var err error + var stageDir string if args.readOnly { err = client.ControllerPublishVolumeReadOnly(volID, instance.GetNodeID()) } else { err = client.ControllerPublishVolumeReadWrite(volID, instance.GetNodeID(), args.forceAttach) } if err != nil { - return fmt.Errorf("ControllerPublishVolume failed with error for disk %v on node %v: %v", volID, instance.GetNodeID(), err.Error()), nil, nil - } - - detach := func() { - // Detach Disk - err = client.ControllerUnpublishVolume(volID, instance.GetNodeID()) - if err != nil { - klog.Errorf("Failed to detach disk: %v", err) - } + return fmt.Errorf("ControllerPublishVolume failed with error for disk %v on node %v: %v", volID, instance.GetNodeID(), err.Error()), nil, stageDir } // Stage Disk - stageDir := filepath.Join("/tmp/", volName, "stage") + stageDir = filepath.Join("/tmp/", volName, "stage") if args.useBlock { - err = client.NodeStageBlockVolume(volID, stageDir) + err = client.NodeStageBlockVolume(volID, stageDir, args.setupDataCache) + } else { - err = client.NodeStageExt4Volume(volID, stageDir) + err = client.NodeStageExt4Volume(volID, stageDir, args.setupDataCache) } if err != nil { - detach() - return fmt.Errorf("NodeStageExt4Volume failed with error: %w", err), nil, nil + _ = detach(volID, instance, client) + return fmt.Errorf("NodeStageExt4Volume failed with error: %w for node: %v", err, instance.GetNodeID()), nil, stageDir } unstageAndDetach := func() { @@ -1120,9 +1142,23 @@ func testAttachAndMount(volID string, volName string, instance *remote.InstanceI klog.Errorf("Failed to rm file path %s: %v", fp, err) } - detach() + detach(volID, instance, client) + } + return nil, unstageAndDetach, stageDir +} + +func detach(volID string, instance *remote.InstanceInfo, client *remote.CsiClient) error { + // Detach Disk + err := client.ControllerUnpublishVolume(volID, instance.GetNodeID()) + if err != nil { + klog.Errorf("Failed to detach disk %v", err) + return fmt.Errorf("Failed to detach disk: %v", err) } + return nil +} +func testMount(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, args attachAndMountArgs, stageDir string) (error, func(), *verifyArgs) { + var err error // Mount Disk publishDir := filepath.Join("/tmp/", volName, "mount") @@ -1133,7 +1169,6 @@ func testAttachAndMount(volID string, volName string, instance *remote.InstanceI } if err != nil { - unstageAndDetach() return fmt.Errorf("NodePublishVolume failed with error: %v", err.Error()), nil, nil } @@ -1144,14 +1179,10 @@ func testAttachAndMount(volID string, volName string, instance *remote.InstanceI klog.Errorf("Failed to unpublish volume: %v", err) } } - unpublishUnstageAndDetach := func() { - unpublish() - unstageAndDetach() - } err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777", !args.readOnly /* recursive */) if err != nil { - unpublishUnstageAndDetach() + unpublish() return fmt.Errorf("Chmod failed with error: %v", err.Error()), nil, nil } @@ -1160,16 +1191,18 @@ func testAttachAndMount(volID string, volName string, instance *remote.InstanceI stageDir: stageDir, } - return nil, unpublishUnstageAndDetach, returnArgs + return nil, unpublish, returnArgs } -func testLifecycleWithVerify(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly, useBlock bool, firstMountVerify, secondMountVerify verifyFunc) error { +func testLifecycleWithVerify(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly, useBlock bool, firstMountVerify, secondMountVerify verifyFunc, detachAndReattach bool, setupDataCache bool) error { klog.Infof("Starting testAttachWriteReadDetach with volume %v node %v with readonly %v\n", volID, instance.GetNodeID(), readOnly) - err, detacher, args := testAttachAndMount(volID, volName, instance, client, attachAndMountArgs{ - readOnly: readOnly, - useBlock: useBlock, - forceAttach: false, - }) + attachArgs := attachAndMountArgs{ + readOnly: readOnly, + useBlock: useBlock, + forceAttach: false, + setupDataCache: setupDataCache, + } + err, detacher, args := testAttachAndMount(volID, volName, instance, client, attachArgs) if err != nil { return fmt.Errorf("failed to attach and mount: %w", err) } @@ -1186,13 +1219,28 @@ func testLifecycleWithVerify(volID string, volName string, instance *remote.Inst return fmt.Errorf("NodeUnpublishVolume failed with error: %v", err.Error()) } + stageDir := args.stageDir + if detachAndReattach { + // Unstage and detach + err = client.NodeUnstageVolume(volID, stageDir) + if err != nil { + klog.Errorf("Failed to unstage volume: %v", err) + } + detach(volID, instance, client) + // Reattach the volume + err, _, stageDir = testAttach(volID, volName, instance, client, attachArgs) + if err != nil { + return err + } + } + if secondMountVerify != nil { // Mount disk somewhere else secondPublishDir := filepath.Join("/tmp/", volName, "secondmount") if useBlock { - err = client.NodePublishBlockVolume(volID, args.stageDir, secondPublishDir) + err = client.NodePublishBlockVolume(volID, stageDir, secondPublishDir) } else { - err = client.NodePublishVolume(volID, args.stageDir, secondPublishDir) + err = client.NodePublishVolume(volID, stageDir, secondPublishDir) } if err != nil { return fmt.Errorf("NodePublishVolume failed with error: %v", err.Error()) diff --git a/test/e2e/tests/resize_e2e_test.go b/test/e2e/tests/resize_e2e_test.go index fbe8af868..fc4d87558 100644 --- a/test/e2e/tests/resize_e2e_test.go +++ b/test/e2e/tests/resize_e2e_test.go @@ -80,7 +80,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Stage Disk stageDir := filepath.Join("/tmp/", volName, "stage") - err = client.NodeStageExt4Volume(volume.VolumeId, stageDir) + err = client.NodeStageExt4Volume(volume.VolumeId, stageDir, false /* setupDataCache */) Expect(err).To(BeNil(), "Node Stage volume failed") defer func() { @@ -174,7 +174,7 @@ var _ = Describe("GCE PD CSI Driver", func() { }() // Volume should be attached/formatted/mounted/unmounted/detached - err = testAttachWriteReadDetach(volume.VolumeId, volName, instance, client, false /* readOnly */) + err = testAttachWriteReadDetach(volume.VolumeId, volName, instance, client, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle") // Resize controller @@ -203,7 +203,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Stage Disk stageDir := filepath.Join("/tmp/", volName, "stage") - err = client.NodeStageExt4Volume(volume.VolumeId, stageDir) + err = client.NodeStageExt4Volume(volume.VolumeId, stageDir, false /* setupDataCache */) Expect(err).To(BeNil(), "Node Stage volume failed") defer func() { @@ -295,7 +295,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Stage Disk stageDir := filepath.Join("/tmp/", volName, "stage") - err = client.NodeStageBlockVolume(volume.VolumeId, stageDir) + err = client.NodeStageBlockVolume(volume.VolumeId, stageDir, false /* setupDataCache */) Expect(err).To(BeNil(), "Node Stage volume failed") defer func() { diff --git a/test/e2e/tests/setup_e2e_test.go b/test/e2e/tests/setup_e2e_test.go index 7fd12a52d..1e3dc1beb 100644 --- a/test/e2e/tests/setup_e2e_test.go +++ b/test/e2e/tests/setup_e2e_test.go @@ -19,6 +19,7 @@ import ( "flag" "fmt" "math/rand" + "strconv" "strings" "testing" "time" @@ -36,16 +37,19 @@ import ( ) var ( - project = flag.String("project", "", "Project to run tests in") - serviceAccount = flag.String("service-account", "", "Service account to bring up instance with") - architecture = flag.String("arch", "amd64", "Architecture pd csi driver build on") - zones = flag.String("zones", "us-east4-a,us-east4-c", "Zones to run tests in. If there are multiple zones, separate each by comma") - machineType = flag.String("machine-type", "n2-standard-2", "Type of machine to provision instance on") - imageURL = flag.String("image-url", "projects/debian-cloud/global/images/family/debian-11", "OS image url to get image from") - runInProw = flag.Bool("run-in-prow", false, "If true, use a Boskos loaned project and special CI service accounts and ssh keys") - deleteInstances = flag.Bool("delete-instances", false, "Delete the instances after tests run") - cloudtopHost = flag.Bool("cloudtop-host", false, "The local host is cloudtop, a kind of googler machine with special requirements to access GCP") - extraDriverFlags = flag.String("extra-driver-flags", "", "Extra flags to pass to the driver") + project = flag.String("project", "", "Project to run tests in") + serviceAccount = flag.String("service-account", "", "Service account to bring up instance with") + vmNamePrefix = flag.String("vm-name-prefix", "gce-pd-csi-e2e", "VM name prefix") + architecture = flag.String("arch", "amd64", "Architecture pd csi driver build on") + minCpuPlatform = flag.String("min-cpu-platform", "rome", "Minimum CPU architecture") + zones = flag.String("zones", "us-east4-a,us-east4-c", "Zones to run tests in. If there are multiple zones, separate each by comma") + machineType = flag.String("machine-type", "n2-standard-2", "Type of machine to provision instance on") + imageURL = flag.String("image-url", "projects/debian-cloud/global/images/family/debian-11", "OS image url to get image from") + runInProw = flag.Bool("run-in-prow", false, "If true, use a Boskos loaned project and special CI service accounts and ssh keys") + deleteInstances = flag.Bool("delete-instances", false, "Delete the instances after tests run") + cloudtopHost = flag.Bool("cloudtop-host", false, "The local host is cloudtop, a kind of googler machine with special requirements to access GCP") + extraDriverFlags = flag.String("extra-driver-flags", "", "Extra flags to pass to the driver") + enableConfidentialCompute = flag.Bool("enable-confidential-compute", false, "Create VMs with confidential compute mode. This uses NVMe devices") testContexts = []*remote.TestContext{} computeService *compute.Service @@ -54,6 +58,8 @@ var ( kmsClient *cloudkms.KeyManagementClient ) +const localSSDCount int64 = 2 + func init() { klog.InitFlags(flag.CommandLine) } @@ -70,6 +76,7 @@ var _ = BeforeSuite(func() { defer close(tcc) zones := strings.Split(*zones, ",") + // Create 2 instances for each zone as we need 2 instances each zone for certain test cases rand.Seed(time.Now().UnixNano()) @@ -95,14 +102,21 @@ var _ = BeforeSuite(func() { klog.Infof("Running in project %v with service account %v", *project, *serviceAccount) - for _, zone := range zones { - go func(curZone string) { - defer GinkgoRecover() - tcc <- NewTestContext(curZone) - }(zone) + numberOfInstancesPerZone := 2 + + setupContext := func(zones []string, randInt int) { + for _, zone := range zones { + go func(curZone string) { + defer GinkgoRecover() + tcc <- NewDefaultTestContext(curZone, strconv.Itoa(randInt)) + }(zone) + } + } + for j := 0; j < numberOfInstancesPerZone; j++ { + setupContext(zones, j) } - for i := 0; i < len(zones); i++ { + for i := 0; i < len(zones)*numberOfInstancesPerZone; i++ { tc := <-tcc testContexts = append(testContexts, tc) klog.Infof("Added TestContext for node %s", tc.Instance.GetName()) @@ -130,21 +144,29 @@ func getDriverConfig() testutils.DriverConfig { } } -func getRemoteInstanceConfig() *remote.InstanceConfig { - return &remote.InstanceConfig{ - Project: *project, - Architecture: *architecture, - MachineType: *machineType, - ServiceAccount: *serviceAccount, - ImageURL: *imageURL, - CloudtopHost: *cloudtopHost} +func NewDefaultTestContext(zone string, instanceNumber string) *remote.TestContext { + return NewTestContext(zone, *minCpuPlatform, *machineType, instanceNumber) } -func NewTestContext(zone string) *remote.TestContext { - nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", zone) +func NewTestContext(zone, minCpuPlatform, machineType string, instanceNumber string) *remote.TestContext { + nodeID := fmt.Sprintf("%s-%s-%s-%s", *vmNamePrefix, zone, machineType, instanceNumber) klog.Infof("Setting up node %s", nodeID) - i, err := remote.SetupInstance(getRemoteInstanceConfig(), zone, nodeID, computeService) + instanceConfig := remote.InstanceConfig{ + Project: *project, + Architecture: *architecture, + MinCpuPlatform: minCpuPlatform, + Zone: zone, + Name: nodeID, + MachineType: machineType, + ServiceAccount: *serviceAccount, + ImageURL: *imageURL, + CloudtopHost: *cloudtopHost, + EnableConfidentialCompute: *enableConfidentialCompute, + ComputeService: computeService, + LocalSSDCount: localSSDCount, + } + i, err := remote.SetupInstance(instanceConfig) if err != nil { klog.Fatalf("Failed to setup instance %v: %v", nodeID, err) } @@ -163,7 +185,16 @@ func NewTestContext(zone string) *remote.TestContext { if err != nil { klog.Fatalf("Failed to copy google_nvme_id to containerized directory: %v", err) } + pkgs := []string{"lvm2", "mdadm", "grep", "coreutils"} + err = testutils.InstallDependencies(i, pkgs) + if err != nil { + klog.Fatalf("Failed to install dependency package on node %v: error : %v", i.GetNodeID(), err) + } + err = testutils.SetupDataCachingConfig(i) + if err != nil { + klog.Fatalf("Failed to setup data cache required config error %v", err) + } klog.Infof("Creating new driver and client for node %s", i.GetName()) tc, err := testutils.GCEClientAndDriverSetup(i, getDriverConfig()) if err != nil { diff --git a/test/e2e/tests/single_zone_e2e_test.go b/test/e2e/tests/single_zone_e2e_test.go index 5bb989441..307675036 100644 --- a/test/e2e/tests/single_zone_e2e_test.go +++ b/test/e2e/tests/single_zone_e2e_test.go @@ -97,7 +97,7 @@ var _ = Describe("GCE PD CSI Driver", func() { }() // Attach Disk - err := testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */) + err := testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle") }) @@ -145,7 +145,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Stage Disk stageDir := filepath.Join("/tmp/", volName, "stage") - err = client.NodeStageExt4Volume(volID, stageDir) + err = client.NodeStageExt4Volume(volID, stageDir, false /* setupDataCache */) Expect(err).To(BeNil(), "failed to repair /dev/by-id symlink and stage volume") // Validate that the link is correct @@ -215,7 +215,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Stage Disk stageDir := filepath.Join("/tmp/", volName, "stage") - err = client.NodeStageExt4Volume(volID, stageDir) + err = client.NodeStageExt4Volume(volID, stageDir, false /* setupDataCache */) Expect(err).To(BeNil(), "failed to repair /dev/by-id symlink and stage volume") // Validate that the link is correct @@ -322,7 +322,7 @@ var _ = Describe("GCE PD CSI Driver", func() { }() // Attach Disk - err := testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */) + err := testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache*/) Expect(err).To(BeNil(), "Failed to go through volume lifecycle") }, Entry("on pd-standard", standardDiskType), @@ -579,7 +579,14 @@ var _ = Describe("GCE PD CSI Driver", func() { defer func() { // Delete Disk - err := client.DeleteVolume(volID) + err = wait.Poll(5*time.Second, 1*time.Minute, func() (bool, error) { + err := client.DeleteVolume(volID) + if err == nil { + return true, err + } + return false, err + }) + Expect(err).To(BeNil(), "DeleteVolume failed") // Validate Disk Deleted @@ -666,7 +673,7 @@ var _ = Describe("GCE PD CSI Driver", func() { }() // Test disk works - err = testAttachWriteReadDetach(volume.VolumeId, volName, controllerInstance, controllerClient, false /* readOnly */) + err = testAttachWriteReadDetach(volume.VolumeId, volName, controllerInstance, controllerClient, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle before revoking CMEK key") // Revoke CMEK key @@ -687,7 +694,7 @@ var _ = Describe("GCE PD CSI Driver", func() { } // Make sure attach of PD fails - err = testAttachWriteReadDetach(volume.VolumeId, volName, controllerInstance, controllerClient, false /* readOnly */) + err = testAttachWriteReadDetach(volume.VolumeId, volName, controllerInstance, controllerClient, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).ToNot(BeNil(), "Volume lifecycle should have failed, but succeeded") // Restore CMEK key @@ -708,7 +715,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // The controller publish failure in above step would set a backoff condition on the node. Wait suffcient amount of time for the driver to accept new controller publish requests. time.Sleep(time.Second) // Make sure attach of PD succeeds - err = testAttachWriteReadDetach(volume.VolumeId, volName, controllerInstance, controllerClient, false /* readOnly */) + err = testAttachWriteReadDetach(volume.VolumeId, volName, controllerInstance, controllerClient, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle after restoring CMEK key") }, Entry("on pd-standard", standardDiskType), @@ -851,7 +858,7 @@ var _ = Describe("GCE PD CSI Driver", func() { } // Attach Disk - err := testLifecycleWithVerify(volID, volName, instance, client, false /* readOnly */, true /* block */, verifyVolumeStats, nil) + err := testLifecycleWithVerify(volID, volName, instance, client, false /* readOnly */, true /* block */, verifyVolumeStats, nil, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle") }) @@ -888,7 +895,7 @@ var _ = Describe("GCE PD CSI Driver", func() { } // Attach Disk - err := testLifecycleWithVerify(volID, volName, instance, client, false /* readOnly */, false /* fs */, verifyVolumeStats, nil) + err := testLifecycleWithVerify(volID, volName, instance, client, false /* readOnly */, false /* fs */, verifyVolumeStats, nil, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle") }) @@ -957,7 +964,7 @@ var _ = Describe("GCE PD CSI Driver", func() { } return nil } - err := testLifecycleWithVerify(volID, volName, instance, client, false /* readOnly */, true /* block */, writeFunc, verifyReadFunc) + err := testLifecycleWithVerify(volID, volName, instance, client, false /* readOnly */, true /* block */, writeFunc, verifyReadFunc, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle") }) @@ -1317,13 +1324,7 @@ var _ = Describe("GCE PD CSI Driver", func() { _, err := getRandomTestContext().Client.ListVolumes() Expect(err).To(BeNil(), "no error expected when passed valid compute url") - zone := "us-central1-c" - nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", zone) - i, err := remote.SetupInstance(getRemoteInstanceConfig(), zone, nodeID, computeService) - - if err != nil { - klog.Fatalf("Failed to setup instance %v: %v", nodeID, err) - } + i := getRandomTestContext().Instance klog.Infof("Creating new driver and client for node %s\n", i.GetName()) @@ -1395,7 +1396,7 @@ var _ = Describe("GCE PD CSI Driver", func() { Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, }, } - err = client.NodeStageVolume(volID, stageDir, volCap) + err = client.NodeStageVolume(volID, stageDir, volCap, false /* setupDataCache */) Expect(err).To(BeNil(), "failed to stage volume: %v", err) // Validate that the link is correct @@ -1432,6 +1433,112 @@ var _ = Describe("GCE PD CSI Driver", func() { } }() }) + It("Should create disks, attach them to instance with local ssd, setup caching between LSSD->detach->reattach to same instance", func() { + Expect(testContexts).ToNot(BeEmpty()) + testContext := getRandomTestContext() + + p, z, _ := testContext.Instance.GetIdentity() + client := testContext.Client + instance := testContext.Instance + volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) + defer deleteVolumeOrError(client, volID) + + // Attach Disk + err := testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */, true /* detachAndReattach */, true /* setupDataCache */) + Expect(err).To(BeNil(), "Failed to go through volume lifecycle") + + }) + It("Should create->attach->setup caching->write->detach->attach to different node->mount->read", func() { + Expect(testContexts).ToNot(BeEmpty()) + zoneToContext := map[string][]*remote.TestContext{} + testZoneContexts := []*remote.TestContext{} + for _, tc := range testContexts { + _, z, _ := tc.Instance.GetIdentity() + // Zone hasn't been seen before + if _, ok := zoneToContext[z]; !ok { + zoneToContext[z] = []*remote.TestContext{tc} + } else { + zoneToContext[z] = append(zoneToContext[z], tc) + } + if len(zoneToContext[z]) >= 2 { + testZoneContexts = zoneToContext[z] + break + } + } + if len(testZoneContexts) < 2 { + klog.Fatalf("No test contexts setup %v", testZoneContexts) + } + testContextForVm1 := testZoneContexts[0] + p, z, _ := testContextForVm1.Instance.GetIdentity() + + client := testContextForVm1.Client + firstInstance := testContextForVm1.Instance + + volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) + defer deleteVolumeOrError(client, volID) + + testContextForVm2 := testZoneContexts[1] + secondClient := testContextForVm2.Client + secondInstance := testContextForVm2.Instance + unmountDisk := func(client *remote.CsiClient, volID string, args *verifyArgs) { + err := client.NodeUnpublishVolume(volID, args.publishDir) + if err != nil { + klog.Errorf("NodeUnpublishVolume failed with error: %v", err) + } + } + attachMountArgs := attachAndMountArgs{ + readOnly: false, + useBlock: false, + forceAttach: false, + setupDataCache: true, + } + // Controller Publish (Attach) - Node Stage - Node Publish(Mount) Volume + err, _, args := testAttachAndMount(volID, volName, firstInstance, client, attachMountArgs) + if err != nil { + klog.Errorf("Failed to attach and mount: %v", err.Error()) + } + // Write file in the volume + firstMountVerify, _ := testWriteAndReadFile(firstInstance, false /* readOnly */) + err = firstMountVerify(args) + if err != nil { + klog.Errorf("failed to verify after first mount to %s: %v", args, err) + } + Expect(err).To(BeNil(), "Failed to write data to volume %s on instance %s", volName, firstInstance.GetName()) + // Unmount Disk + unmountDisk(client, volID, args) + + // Node Unstage + err = client.NodeUnstageVolume(volID, args.stageDir) + if err != nil { + klog.Errorf("Failed to unstage volume: %v", err) + } + detach(volID, firstInstance, client) + + // Attach Disk to secondInstance + err, detacher, stageDir := testAttach(volID, volName, secondInstance, secondClient, attachMountArgs) + if err != nil { + klog.Errorf("Failed to attach disk %v", err) + } + defer func() { + detacher() + deleteVolumeOrError(secondClient, volID) + }() + + // Mount disk + err, _, args = testMount(volID, volName, secondInstance, secondClient, attachMountArgs, stageDir) + if err != nil { + klog.Fatalf("Failed to mount disk %v", err) + } + _, secondMountRead := testWriteAndReadFile(secondInstance, false /* readOnly */) + err = secondMountRead(args) + if err != nil { + klog.Errorf("failed to verify after second mount to %s: %v", args, err) + } + // Unmount disk for cleanup + unmountDisk(secondClient, volID, args) + Expect(err).To(BeNil(), "Failed to read data from volume %s on instance %s", volName, secondInstance.GetName()) + + }) It("Should block unstage if filesystem mounted", func() { testContext := getRandomTestContext() @@ -1467,7 +1574,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Stage Disk stageDir := filepath.Join("/tmp/", volName, "stage") - err = client.NodeStageExt4Volume(volID, stageDir) + err = client.NodeStageExt4Volume(volID, stageDir, false) Expect(err).To(BeNil(), "failed to stage volume: %v", err) // Create private bind mount diff --git a/test/e2e/utils/utils.go b/test/e2e/utils/utils.go index 995b114b0..3ffdfdd40 100644 --- a/test/e2e/utils/utils.go +++ b/test/e2e/utils/utils.go @@ -67,6 +67,9 @@ func GCEClientAndDriverSetup(instance *remote.InstanceInfo, driverConfig DriverC "--use-instance-api-to-poll-attachment-disk-types=pd-ssd", "--use-instance-api-to-list-volumes-published-nodes", fmt.Sprintf("--fallback-requisite-zones=%s", strings.Join(driverConfig.Zones, ",")), + "--enable-controller-data-cache", + "--enable-node-data-cache", + fmt.Sprintf("--node-name=%s", utilcommon.TestNode), } extra_flags = append(extra_flags, fmt.Sprintf("--compute-endpoint=%s", driverConfig.ComputeEndpoint)) extra_flags = append(extra_flags, driverConfig.ExtraFlags...) @@ -274,6 +277,33 @@ func CopyFile(instance *remote.InstanceInfo, src, dest string) error { return nil } +func InstallDependencies(instance *remote.InstanceInfo, pkgs []string) error { + _, _ = instance.SSH("apt-get", "update") + for _, pkg := range pkgs { + output, err := instance.SSH("apt-get", "install", "-y", pkg) + if err != nil { + return fmt.Errorf("failed to install package %s. Output: %v, errror: %v", pkg, output, err.Error()) + } + } + return nil +} + +func SetupDataCachingConfig(instance *remote.InstanceInfo) error { + output, err := instance.SSH("/bin/sed", "-i", "-e", "\"s/.*allow_mixed_block_sizes = 0.*/ allow_mixed_block_sizes = 1/\"", "/etc/lvm/lvm.conf") + if err != nil { + return fmt.Errorf("failed to update field allow_mixed_block_sizes, error:%v; output: %v", err, output) + } + output, err = instance.SSH("/bin/sed", "-i", "-e", "\"s/.*udev_sync = 1.*/ udev_sync = 0/\"", "/etc/lvm/lvm.conf") + if err != nil { + return fmt.Errorf("failed to update field udev_sync, error:%v; output: %v", err, output) + } + output, err = instance.SSH("/bin/sed", "-i", "-e", "\"s/.*udev_rules = 1.*/ udev_rules = 0/\"", "/etc/lvm/lvm.conf") + if err != nil { + return fmt.Errorf("failed to update field udev_rules, error:%v; output: %v", err, output) + } + return nil +} + // ValidateLogicalLinkIsDisk takes a symlink location at "link" and finds the // link location - it then finds the backing PD using either scsi_id or // google_nvme_id (depending on the /dev path) and validates that it is the diff --git a/test/k8s-integration/config/data-cache-sc.yaml b/test/k8s-integration/config/data-cache-sc.yaml new file mode 100644 index 000000000..7e911eb36 --- /dev/null +++ b/test/k8s-integration/config/data-cache-sc.yaml @@ -0,0 +1,11 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: csi-gcepd-balanced +provisioner: pd.csi.storage.gke.io +parameters: + type: pd-balanced + data-cache-mode: writeback + data-cache-size: "50Gi" +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true \ No newline at end of file diff --git a/test/remote/client-wrappers.go b/test/remote/client-wrappers.go index 671cbb1df..efb0035f3 100644 --- a/test/remote/client-wrappers.go +++ b/test/remote/client-wrappers.go @@ -52,6 +52,19 @@ var ( } ) +const ( + // Keys in the volume context. + contextForceAttach = "force-attach" + contextDataCacheSize = "data-cache-size" + contextDataCacheMode = "data-cache-mode" + + // Keys in the publish context + contexLocalSsdCacheSize = "local-ssd-cache-size" + + defaultLocalSsdCacheSize = "200Gi" + defaultDataCacheMode = common.DataCacheModeWriteThrough +) + type CsiClient struct { conn *grpc.ClientConn idClient csipb.IdentityClient @@ -179,19 +192,25 @@ func (c *CsiClient) ControllerUnpublishVolume(volId, nodeId string) error { return err } -func (c *CsiClient) NodeStageExt4Volume(volId, stageDir string) error { - return c.NodeStageVolume(volId, stageDir, stdVolCap) +func (c *CsiClient) NodeStageExt4Volume(volId, stageDir string, setupDataCache bool) error { + return c.NodeStageVolume(volId, stageDir, stdVolCap, setupDataCache) } -func (c *CsiClient) NodeStageBlockVolume(volId, stageDir string) error { - return c.NodeStageVolume(volId, stageDir, blockVolCap) +func (c *CsiClient) NodeStageBlockVolume(volId, stageDir string, setupDataCache bool) error { + return c.NodeStageVolume(volId, stageDir, blockVolCap, setupDataCache) } -func (c *CsiClient) NodeStageVolume(volId, stageDir string, volumeCap *csipb.VolumeCapability) error { +func (c *CsiClient) NodeStageVolume(volId string, stageDir string, volumeCap *csipb.VolumeCapability, setupDataCache bool) error { + publishContext := map[string]string{} + if setupDataCache { + publishContext[contexLocalSsdCacheSize] = defaultLocalSsdCacheSize + publishContext[contextDataCacheMode] = defaultDataCacheMode + } nodeStageReq := &csipb.NodeStageVolumeRequest{ VolumeId: volId, StagingTargetPath: stageDir, VolumeCapability: volumeCap, + PublishContext: publishContext, } _, err := c.nodeClient.NodeStageVolume(context.Background(), nodeStageReq) return err diff --git a/test/remote/instance.go b/test/remote/instance.go index 6fb1b5233..0ba1b989d 100644 --- a/test/remote/instance.go +++ b/test/remote/instance.go @@ -47,61 +47,68 @@ const ( // InstanceConfig is the common bundle of options used for instance creation. type InstanceConfig struct { - Project string - Architecture string - MachineType string - ServiceAccount string - ImageURL string - CloudtopHost bool + Project string + Architecture string + Zone string + Name string + MachineType string + ServiceAccount string + ImageURL string + CloudtopHost bool + MinCpuPlatform string + ComputeService *compute.Service + EnableConfidentialCompute bool + LocalSSDCount int64 + EnableDataCache bool } type InstanceInfo struct { - project string - architecture string - zone string - name string - machineType string - serviceAccount string - imageURL string - cloudtopHost bool - + cfg InstanceConfig // External IP is filled in after instance creation externalIP string - - computeService *compute.Service } func (i *InstanceInfo) GetIdentity() (string, string, string) { - return i.project, i.zone, i.name + return i.cfg.Project, i.cfg.Zone, i.cfg.Name } func (i *InstanceInfo) GetName() string { - return i.name + return i.cfg.Name } func (i *InstanceInfo) GetNodeID() string { - return common.CreateNodeID(i.project, i.zone, i.name) + return common.CreateNodeID(i.cfg.Project, i.cfg.Zone, i.cfg.Name) } -func CreateInstanceInfo(config *InstanceConfig, zone, name string, cs *compute.Service) (*InstanceInfo, error) { - return &InstanceInfo{ - project: config.Project, - architecture: config.Architecture, - zone: zone, - name: name, - machineType: config.MachineType, - cloudtopHost: config.CloudtopHost, - serviceAccount: config.ServiceAccount, - imageURL: config.ImageURL, - computeService: cs, - }, nil +func machineTypeMismatch(curInst *compute.Instance, newInst *compute.Instance) bool { + if !strings.Contains(curInst.MachineType, newInst.MachineType) { + klog.Infof("Machine type mismatch") + return true + } + // Ideally we could compare to see if the new instance has a greater minCpuPlatfor + // For now we just check it was set and it's different. + if curInst.MinCpuPlatform != "" && curInst.MinCpuPlatform != newInst.MinCpuPlatform { + klog.Infof("CPU Platform mismatch") + return true + } + if (curInst.ConfidentialInstanceConfig != nil && newInst.ConfidentialInstanceConfig == nil) || + (curInst.ConfidentialInstanceConfig == nil && newInst.ConfidentialInstanceConfig != nil) || + (curInst.ConfidentialInstanceConfig != nil && newInst.ConfidentialInstanceConfig != nil && curInst.ConfidentialInstanceConfig.EnableConfidentialCompute != newInst.ConfidentialInstanceConfig.EnableConfidentialCompute) { + klog.Infof("Confidential compute mismatch") + return true + } + if curInst.SourceMachineImage != newInst.SourceMachineImage { + klog.Infof("Source Machine Mismatch") + return true + } + return false } // Provision a gce instance using image -func (i *InstanceInfo) CreateOrGetInstance() error { +func (i *InstanceInfo) CreateOrGetInstance(localSSDCount int) error { var err error var instance *compute.Instance - klog.V(4).Infof("Creating instance: %v", i.name) + klog.V(4).Infof("Creating instance: %v", i.cfg.Name) myuuid := string(uuid.NewUUID()) @@ -111,8 +118,8 @@ func (i *InstanceInfo) CreateOrGetInstance() error { } newInst := &compute.Instance{ - Name: i.name, - MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", i.zone, i.machineType), + Name: i.cfg.Name, + MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", i.cfg.Zone, i.cfg.MachineType), NetworkInterfaces: []*compute.NetworkInterface{ { AccessConfigs: []*compute.AccessConfig{ @@ -129,14 +136,34 @@ func (i *InstanceInfo) CreateOrGetInstance() error { Type: "PERSISTENT", InitializeParams: &compute.AttachedDiskInitializeParams{ DiskName: "my-root-pd-" + myuuid, - SourceImage: i.imageURL, + SourceImage: i.cfg.ImageURL, }, }, }, + MinCpuPlatform: i.cfg.MinCpuPlatform, } + if i.cfg.EnableConfidentialCompute { + newInst.ConfidentialInstanceConfig = &compute.ConfidentialInstanceConfig{ + EnableConfidentialCompute: true, + } + } + klog.Infof("=======Adding LocalSSD %v=============", localSSDCount) + + localSSDConfig := &compute.AttachedDisk{ + Type: "SCRATCH", + InitializeParams: &compute.AttachedDiskInitializeParams{ + DiskType: fmt.Sprintf("zones/%s/diskTypes/local-ssd", i.cfg.Zone), + }, + AutoDelete: true, + Interface: "NVME", + } + + for i := 0; i < localSSDCount; i++ { + newInst.Disks = append(newInst.Disks, localSSDConfig) + } saObj := &compute.ServiceAccount{ - Email: i.serviceAccount, + Email: i.cfg.ServiceAccount, Scopes: []string{"https://www.googleapis.com/auth/cloud-platform"}, } newInst.ServiceAccounts = []*compute.ServiceAccount{saObj} @@ -150,19 +177,19 @@ func (i *InstanceInfo) CreateOrGetInstance() error { newInst.Metadata = meta } - // If instance exists but machine-type doesn't match, delete instance - curInst, _ := i.computeService.Instances.Get(i.project, i.zone, newInst.Name).Do() + // If instance exists but settings differ, delete instance + curInst, _ := i.cfg.ComputeService.Instances.Get(i.cfg.Project, i.cfg.Zone, newInst.Name).Do() if curInst != nil { - if !strings.Contains(curInst.MachineType, newInst.MachineType) { + if machineTypeMismatch(curInst, newInst) { klog.V(4).Infof("Instance machine type doesn't match the required one. Delete instance.") - if _, err := i.computeService.Instances.Delete(i.project, i.zone, i.name).Do(); err != nil { + if _, err := i.cfg.ComputeService.Instances.Delete(i.cfg.Project, i.cfg.Zone, i.cfg.Name).Do(); err != nil { return err } start := time.Now() err := wait.Poll(15*time.Second, 5*time.Minute, func() (bool, error) { klog.V(2).Infof("Waiting for instance to be deleted. %v elapsed", time.Since(start)) - if curInst, _ = i.computeService.Instances.Get(i.project, i.zone, i.name).Do(); curInst != nil { + if curInst, _ = i.cfg.ComputeService.Instances.Get(i.cfg.Project, i.cfg.Zone, i.cfg.Name).Do(); curInst != nil { return false, nil } return true, nil @@ -174,16 +201,16 @@ func (i *InstanceInfo) CreateOrGetInstance() error { } if curInst == nil { - op, err := i.computeService.Instances.Insert(i.project, i.zone, newInst).Do() - klog.V(4).Infof("Inserted instance %v in project: %v, zone: %v", newInst.Name, i.project, i.zone) + op, err := i.cfg.ComputeService.Instances.Insert(i.cfg.Project, i.cfg.Zone, newInst).Do() + klog.V(4).Infof("Inserted instance %v in project: %v, zone: %v", newInst.Name, i.cfg.Project, i.cfg.Zone) if err != nil { - ret := fmt.Sprintf("could not create instance %s: API error: %v", i.name, err.Error()) + ret := fmt.Sprintf("could not create instance %s: API error: %v", i.cfg.Name, err.Error()) if op != nil { ret = fmt.Sprintf("%s. op error: %v", ret, op.Error) } return errors.New(ret) } else if op.Error != nil { - return fmt.Errorf("could not create instance %s: %+v", i.name, op.Error) + return fmt.Errorf("could not create instance %s: %+v", i.cfg.Name, op.Error) } } else { klog.V(4).Infof("Compute service GOT instance %v, skipping instance creation", newInst.Name) @@ -191,26 +218,26 @@ func (i *InstanceInfo) CreateOrGetInstance() error { start := time.Now() err = wait.Poll(15*time.Second, 5*time.Minute, func() (bool, error) { - klog.V(2).Infof("Waiting for instance %v to come up. %v elapsed", i.name, time.Since(start)) + klog.V(2).Infof("Waiting for instance %v to come up. %v elapsed", i.cfg.Name, time.Since(start)) - instance, err = i.computeService.Instances.Get(i.project, i.zone, i.name).Do() + instance, err = i.cfg.ComputeService.Instances.Get(i.cfg.Project, i.cfg.Zone, i.cfg.Name).Do() if err != nil { - klog.Errorf("Failed to get instance %q: %v", i.name, err) + klog.Errorf("Failed to get instance %q: %v", i.cfg.Name, err) return false, nil } if strings.ToUpper(instance.Status) != "RUNNING" { - klog.Warningf("instance %s not in state RUNNING, was %s", i.name, instance.Status) + klog.Warningf("instance %s not in state RUNNING, was %s", i.cfg.Name, instance.Status) return false, nil } - if i.cloudtopHost { - output, err := exec.Command("gcloud", "compute", "ssh", i.name, "--zone", i.zone, "--project", i.project, "--", "-o", "ProxyCommand=corp-ssh-helper %h %p", "--", "echo").CombinedOutput() + if i.cfg.CloudtopHost { + output, err := exec.Command("gcloud", "compute", "ssh", i.cfg.Name, "--zone", i.cfg.Zone, "--project", i.cfg.Project, "--", "-o", "ProxyCommand=corp-ssh-helper %h %p", "--", "echo").CombinedOutput() if err != nil { klog.Errorf("Failed to bootstrap ssh (%v): %s", err, string(output)) return false, nil } - klog.V(4).Infof("Bootstrapped cloudtop ssh for instance %v", i.name) + klog.V(4).Infof("Bootstrapped cloudtop ssh for instance %v", i.cfg.Name) } externalIP := getexternalIP(instance) @@ -219,11 +246,11 @@ func (i *InstanceInfo) CreateOrGetInstance() error { } if sshOut, err := i.SSHCheckAlive(); err != nil { - err = fmt.Errorf("Instance %v in state RUNNING but not available by SSH: %v", i.name, err.Error()) + err = fmt.Errorf("Instance %v in state RUNNING but not available by SSH: %v", i.cfg.Name, err.Error()) klog.Warningf("SSH encountered an error: %v, output: %v", err, sshOut) return false, nil } - klog.V(4).Infof("Instance %v in state RUNNING and available by SSH", i.name) + klog.V(4).Infof("Instance %v in state RUNNING and available by SSH", i.cfg.Name) return true, nil }) @@ -233,24 +260,24 @@ func (i *InstanceInfo) CreateOrGetInstance() error { } // Instance reached running state in time, make sure that cloud-init is complete - klog.V(2).Infof("Instance %v has been created successfully", i.name) + klog.V(2).Infof("Instance %v has been created successfully", i.cfg.Name) return nil } func (i *InstanceInfo) DeleteInstance() { - klog.V(4).Infof("Deleting instance %q", i.name) - _, err := i.computeService.Instances.Delete(i.project, i.zone, i.name).Do() + klog.V(4).Infof("Deleting instance %q", i.cfg.Name) + _, err := i.cfg.ComputeService.Instances.Delete(i.cfg.Project, i.cfg.Zone, i.cfg.Name).Do() if err != nil { if isGCEError(err, "notFound") { return } - klog.Errorf("Error deleting instance %q: %v", i.name, err) + klog.Errorf("Error deleting instance %q: %v", i.cfg.Name, err) } } func (i *InstanceInfo) DetachDisk(diskName string) error { klog.V(4).Infof("Detaching disk %q", diskName) - op, err := i.computeService.Instances.DetachDisk(i.project, i.zone, i.name, diskName).Do() + op, err := i.cfg.ComputeService.Instances.DetachDisk(i.cfg.Project, i.cfg.Zone, i.cfg.Name, diskName).Do() if err != nil { if isGCEError(err, "notFound") { return nil @@ -260,9 +287,9 @@ func (i *InstanceInfo) DetachDisk(diskName string) error { start := time.Now() if err := wait.Poll(5*time.Second, 1*time.Minute, func() (bool, error) { - klog.V(2).Infof("Waiting for disk %q to be detached from instance %q. %v elapsed", diskName, i.name, time.Since(start)) + klog.V(2).Infof("Waiting for disk %q to be detached from instance %q. %v elapsed", diskName, i.cfg.Name, time.Since(start)) - op, err = i.computeService.ZoneOperations.Get(i.project, i.zone, op.Name).Do() + op, err = i.cfg.ComputeService.ZoneOperations.Get(i.cfg.Project, i.cfg.Zone, op.Name).Do() if err != nil { return true, fmt.Errorf("Failed to get operation %q, err: %v", op.Name, err) } @@ -271,7 +298,7 @@ func (i *InstanceInfo) DetachDisk(diskName string) error { return err } - klog.V(4).Infof("Disk %q has been successfully detached from instance %q\n%v", diskName, i.name, op.Error) + klog.V(4).Infof("Disk %q has been successfully detached from instance %q\n%v", diskName, i.cfg.Name, op.Error) return nil } @@ -289,7 +316,7 @@ func getexternalIP(instance *compute.Instance) string { } func getTimestamp() string { - return fmt.Sprintf(time.Now().Format(timestampFormat)) + return fmt.Sprintf("%s", time.Now().Format(timestampFormat)) } // Create default SSH filewall rule if it does not exist @@ -297,7 +324,7 @@ func (i *InstanceInfo) createDefaultFirewallRule() error { var err error klog.V(4).Infof("Creating default firewall rule %s...", defaultFirewallRule) - if _, err = i.computeService.Firewalls.Get(i.project, defaultFirewallRule).Do(); err != nil { + if _, err = i.cfg.ComputeService.Firewalls.Get(i.cfg.Project, defaultFirewallRule).Do(); err != nil { klog.V(4).Infof("Default firewall rule %v does not exist, creating", defaultFirewallRule) f := &compute.Firewall{ Name: defaultFirewallRule, @@ -308,7 +335,7 @@ func (i *InstanceInfo) createDefaultFirewallRule() error { }, }, } - _, err = i.computeService.Firewalls.Insert(i.project, f).Do() + _, err = i.cfg.ComputeService.Firewalls.Insert(i.cfg.Project, f).Do() if err != nil { if gce.IsGCEError(err, "alreadyExists") { klog.V(4).Infof("Default firewall rule %v already exists, skipping creation", defaultFirewallRule) diff --git a/test/remote/runner.go b/test/remote/runner.go index 2fb8d8c93..1b2120810 100644 --- a/test/remote/runner.go +++ b/test/remote/runner.go @@ -29,12 +29,12 @@ import ( func (i *InstanceInfo) UploadAndRun(archivePath, remoteWorkspace, driverRunCmd string) (int, error) { // Create the temp staging directory - klog.V(4).Infof("Staging test binaries on %q", i.name) + klog.V(4).Infof("Staging test binaries on %q", i.cfg.Name) // Do not sudo here, so that we can use scp to copy test archive to the directdory. if output, err := i.SSHNoSudo("mkdir", remoteWorkspace); err != nil { // Exit failure with the error - return -1, fmt.Errorf("failed to create remoteWorkspace directory %q on i.name %q: %v output: %q", remoteWorkspace, i.name, err.Error(), output) + return -1, fmt.Errorf("failed to create remoteWorkspace directory %q on i.name %q: %v output: %q", remoteWorkspace, i.cfg.Name, err.Error(), output) } // Copy the archive to the staging directory @@ -49,7 +49,7 @@ func (i *InstanceInfo) UploadAndRun(archivePath, remoteWorkspace, driverRunCmd s fmt.Sprintf("cd %s", remoteWorkspace), fmt.Sprintf("tar -xzvf ./%s", archiveName), ) - klog.V(4).Infof("Extracting tar on %q", i.name) + klog.V(4).Infof("Extracting tar on %q", i.cfg.Name) // Do not use sudo here, because `sudo tar -x` will recover the file ownership inside the tar ball, but // we want the extracted files to be owned by the current user. if output, err := i.SSHNoSudo("sh", "-c", cmd); err != nil { @@ -57,7 +57,7 @@ func (i *InstanceInfo) UploadAndRun(archivePath, remoteWorkspace, driverRunCmd s return -1, fmt.Errorf("failed to extract test archive: %v, output: %q", err.Error(), output) } - klog.V(4).Infof("Starting driver on %q", i.name) + klog.V(4).Infof("Starting driver on %q", i.cfg.Name) // When the process is killed the driver should close the TCP endpoint, then we want to download the logs output, err := i.SSH(driverRunCmd) if err != nil { diff --git a/test/remote/setup-teardown.go b/test/remote/setup-teardown.go index 3db9fcf1e..3026b28b0 100644 --- a/test/remote/setup-teardown.go +++ b/test/remote/setup-teardown.go @@ -20,7 +20,6 @@ import ( "fmt" "os" - compute "google.golang.org/api/compute/v1" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/klog/v2" ) @@ -54,14 +53,13 @@ type processes struct { } // SetupInstance sets up the specified GCE Instance for E2E testing and returns a handle to the instance object for future use. -func SetupInstance(config *InstanceConfig, instanceZone, instanceName string, cs *compute.Service) (*InstanceInfo, error) { +func SetupInstance(cfg InstanceConfig) (*InstanceInfo, error) { // Create the instance in the requisite zone - instance, err := CreateInstanceInfo(config, instanceZone, instanceName, cs) - if err != nil { - return nil, err + instance := &InstanceInfo{ + cfg: cfg, } - err = instance.CreateOrGetInstance() + err := instance.CreateOrGetInstance(int(cfg.LocalSSDCount)) if err != nil { return nil, err } @@ -73,7 +71,7 @@ func SetupInstance(config *InstanceConfig, instanceZone, instanceName string, cs // that the driver is on and the CSI Client object to make CSI calls to the remote driver. func SetupNewDriverAndClient(instance *InstanceInfo, config *ClientConfig) (*TestContext, error) { archiveName := fmt.Sprintf("e2e_driver_binaries_%s.tar.gz", uuid.NewUUID()) - archivePath, err := CreateDriverArchive(archiveName, instance.architecture, config.PkgPath, config.BinPath) + archivePath, err := CreateDriverArchive(archiveName, instance.cfg.Architecture, config.PkgPath, config.BinPath) if err != nil { return nil, err } @@ -84,6 +82,13 @@ func SetupNewDriverAndClient(instance *InstanceInfo, config *ClientConfig) (*Tes } }() + // Copy dependencies + _, _ = instance.SSH("apt-get", "update") + output, err := instance.SSH("apt-get", "install", "-y", "mdadm", "lvm2") + if err != nil { + return nil, fmt.Errorf("failed to install dependencies. Output: %v, errror: %v", output, err.Error()) + } + // Upload archive to instance and run binaries driverPID, err := instance.UploadAndRun(archivePath, config.WorkspaceDir, config.RunDriverCmd) if err != nil { diff --git a/test/run-e2e.sh b/test/run-e2e.sh index 79144ea31..e86931c0d 100755 --- a/test/run-e2e.sh +++ b/test/run-e2e.sh @@ -5,4 +5,9 @@ set -x readonly PKGDIR=sigs.k8s.io/gcp-compute-persistent-disk-csi-driver -go test --timeout 30m --v "${PKGDIR}/test/e2e/tests" --run-in-prow=true --delete-instances=true --logtostderr +TIMEOUT=50m +if [ "$RUN_CONTROLLER_MODIFY_VOLUME_TESTS" = true ]; then + TIMEOUT=45m +fi + +go test --timeout "${TIMEOUT}" --v "${PKGDIR}/test/e2e/tests" --run-in-prow=true --delete-instances=true --logtostderr $@ diff --git a/test/sanity/sanity_test.go b/test/sanity/sanity_test.go index f09c64ef1..5b6251f2d 100644 --- a/test/sanity/sanity_test.go +++ b/test/sanity/sanity_test.go @@ -63,17 +63,21 @@ func TestSanity(t *testing.T) { fallbackRequisiteZones := []string{} enableStoragePools := false + enableDataCache := true multiZoneVolumeHandleConfig := driver.MultiZoneVolumeHandleConfig{} listVolumesConfig := driver.ListVolumesConfig{} mounter := mountmanager.NewFakeSafeMounter() deviceUtils := deviceutils.NewFakeDeviceUtils(true) + args := driver.NodeServerArgs{ + EnableDataCache: true, + } //Initialize GCE Driver identityServer := driver.NewIdentityServer(gceDriver) - controllerServer := driver.NewControllerServer(gceDriver, cloudProvider, 0, 5*time.Minute, fallbackRequisiteZones, enableStoragePools, multiZoneVolumeHandleConfig, listVolumesConfig) + controllerServer := driver.NewControllerServer(gceDriver, cloudProvider, 0, 5*time.Minute, fallbackRequisiteZones, enableStoragePools, enableDataCache, multiZoneVolumeHandleConfig, listVolumesConfig) fakeStatter := mountmanager.NewFakeStatterWithOptions(mounter, mountmanager.FakeStatterOptions{IsBlock: false}) - nodeServer := driver.NewNodeServer(gceDriver, mounter, deviceUtils, metadataservice.NewFakeService(), fakeStatter) + nodeServer := driver.NewNodeServer(gceDriver, mounter, deviceUtils, metadataservice.NewFakeService(), fakeStatter, args) err = gceDriver.SetupGCEDriver(driverName, vendorVersion, extraLabels, nil, identityServer, controllerServer, nodeServer) if err != nil { t.Fatalf("Failed to initialize GCE CSI Driver: %v", err.Error()) From 27b866abf1907497fe63f15b3249eb09f3500388 Mon Sep 17 00:00:00 2001 From: Sneha Aradhey Date: Wed, 19 Feb 2025 20:18:05 +0000 Subject: [PATCH 02/25] fix test suite issue --- cmd/gce-pd-csi-driver/main.go | 2 +- test/e2e/tests/setup_e2e_test.go | 32 ++++++++++++++++++++------------ test/remote/instance.go | 1 - 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/cmd/gce-pd-csi-driver/main.go b/cmd/gce-pd-csi-driver/main.go index d638d0bd2..933fd2aca 100644 --- a/cmd/gce-pd-csi-driver/main.go +++ b/cmd/gce-pd-csi-driver/main.go @@ -333,7 +333,7 @@ func urlFlag(target **url.URL, name string, usage string) { } func setupDataCache(ctx context.Context, nodeName string) error { - klog.V(2).Infof("Seting up data cache for node %s", nodeName) + klog.V(2).Infof("Setting up data cache for node %s", nodeName) if nodeName != common.TestNode { cfg, err := rest.InClusterConfig() if err != nil { diff --git a/test/e2e/tests/setup_e2e_test.go b/test/e2e/tests/setup_e2e_test.go index 1e3dc1beb..7a3bef149 100644 --- a/test/e2e/tests/setup_e2e_test.go +++ b/test/e2e/tests/setup_e2e_test.go @@ -21,6 +21,7 @@ import ( "math/rand" "strconv" "strings" + "sync" "testing" "time" @@ -72,11 +73,10 @@ func TestE2E(t *testing.T) { var _ = BeforeSuite(func() { var err error - tcc := make(chan *remote.TestContext) - defer close(tcc) - + numberOfInstancesPerZone := 2 zones := strings.Split(*zones, ",") - // Create 2 instances for each zone as we need 2 instances each zone for certain test cases + tcc := make(chan *remote.TestContext, len(zones)*numberOfInstancesPerZone) + defer close(tcc) rand.Seed(time.Now().UnixNano()) @@ -102,18 +102,21 @@ var _ = BeforeSuite(func() { klog.Infof("Running in project %v with service account %v", *project, *serviceAccount) - numberOfInstancesPerZone := 2 - - setupContext := func(zones []string, randInt int) { - for _, zone := range zones { - go func(curZone string) { + setupContext := func(zone string) { + var wg sync.WaitGroup + // Create 2 instances for each zone as we need 2 instances each zone for certain test cases + for j := 0; j < numberOfInstancesPerZone; j++ { + wg.Add(1) + go func(curZone string, randInt int) { defer GinkgoRecover() + defer wg.Done() tcc <- NewDefaultTestContext(curZone, strconv.Itoa(randInt)) - }(zone) + }(zone, j) } } - for j := 0; j < numberOfInstancesPerZone; j++ { - setupContext(zones, j) + + for _, zone := range zones { + setupContext(zone) } for i := 0; i < len(zones)*numberOfInstancesPerZone; i++ { @@ -166,6 +169,11 @@ func NewTestContext(zone, minCpuPlatform, machineType string, instanceNumber str ComputeService: computeService, LocalSSDCount: localSSDCount, } + + if machineType == *hdMachineType { + // Machine type is defaulted to c3-standard-2 which doesn't support LSSD and we don't need LSSD for HdHA test context + instanceConfig.LocalSSDCount = 0 + } i, err := remote.SetupInstance(instanceConfig) if err != nil { klog.Fatalf("Failed to setup instance %v: %v", nodeID, err) diff --git a/test/remote/instance.go b/test/remote/instance.go index 0ba1b989d..c9c3dd6d5 100644 --- a/test/remote/instance.go +++ b/test/remote/instance.go @@ -148,7 +148,6 @@ func (i *InstanceInfo) CreateOrGetInstance(localSSDCount int) error { EnableConfidentialCompute: true, } } - klog.Infof("=======Adding LocalSSD %v=============", localSSDCount) localSSDConfig := &compute.AttachedDisk{ Type: "SCRATCH", From 02c853489ace6f8637c1bad0acd62d1572dbe2cc Mon Sep 17 00:00:00 2001 From: Sneha Aradhey Date: Thu, 20 Feb 2025 16:03:59 +0000 Subject: [PATCH 03/25] Update flags for data cache --- cmd/gce-pd-csi-driver/main.go | 19 +++++++++---------- deploy/kubernetes/base/node_linux/node.yaml | 2 +- pkg/gce-pd-csi-driver/cache.go | 1 - test/e2e/utils/utils.go | 3 +-- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/cmd/gce-pd-csi-driver/main.go b/cmd/gce-pd-csi-driver/main.go index 933fd2aca..296df1d0a 100644 --- a/cmd/gce-pd-csi-driver/main.go +++ b/cmd/gce-pd-csi-driver/main.go @@ -71,13 +71,12 @@ var ( maxConcurrentFormat = flag.Int("max-concurrent-format", 1, "The maximum number of concurrent format exec calls") concurrentFormatTimeout = flag.Duration("concurrent-format-timeout", 1*time.Minute, "The maximum duration of a format operation before its concurrency token is released") - maxConcurrentFormatAndMount = flag.Int("max-concurrent-format-and-mount", 1, "If set then format and mount operations are serialized on each node. This is stronger than max-concurrent-format as it includes fsck and other mount operations") - formatAndMountTimeout = flag.Duration("format-and-mount-timeout", 1*time.Minute, "The maximum duration of a format and mount operation before another such operation will be started. Used only if --serialize-format-and-mount") - fallbackRequisiteZonesFlag = flag.String("fallback-requisite-zones", "", "Comma separated list of requisite zones that will be used if there are not sufficient zones present in requisite topologies when provisioning a disk") - enableStoragePoolsFlag = flag.Bool("enable-storage-pools", false, "If set to true, the CSI Driver will allow volumes to be provisioned in Storage Pools") - enableControllerDataCacheFlag = flag.Bool("enable-controller-data-cache", false, "If set to true, the CSI Driver will allow volumes to be provisioned with data cache configuration") - enableNodeDataCacheFlag = flag.Bool("enable-node-data-cache", false, "If set to true, the CSI Driver will allow volumes to be provisioned with data cache configuration") - nodeName = flag.String("node-name", "", "The node this driver is running on") + maxConcurrentFormatAndMount = flag.Int("max-concurrent-format-and-mount", 1, "If set then format and mount operations are serialized on each node. This is stronger than max-concurrent-format as it includes fsck and other mount operations") + formatAndMountTimeout = flag.Duration("format-and-mount-timeout", 1*time.Minute, "The maximum duration of a format and mount operation before another such operation will be started. Used only if --serialize-format-and-mount") + fallbackRequisiteZonesFlag = flag.String("fallback-requisite-zones", "", "Comma separated list of requisite zones that will be used if there are not sufficient zones present in requisite topologies when provisioning a disk") + enableStoragePoolsFlag = flag.Bool("enable-storage-pools", false, "If set to true, the CSI Driver will allow volumes to be provisioned in Storage Pools") + enableDataCacheFlag = flag.Bool("enable-data-cache", false, "If set to true, the CSI Driver will allow volumes to be provisioned with data cache configuration") + nodeName = flag.String("node-name", "", "The node this driver is running on") multiZoneVolumeHandleDiskTypesFlag = flag.String("multi-zone-volume-handle-disk-types", "", "Comma separated list of allowed disk types that can use the multi-zone volumeHandle. Used only if --multi-zone-volume-handle-enable") multiZoneVolumeHandleEnableFlag = flag.Bool("multi-zone-volume-handle-enable", false, "If set to true, the multi-zone volumeHandle feature will be enabled") @@ -217,7 +216,7 @@ func handle() { } initialBackoffDuration := time.Duration(*errorBackoffInitialDurationMs) * time.Millisecond maxBackoffDuration := time.Duration(*errorBackoffMaxDurationMs) * time.Millisecond - controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration, fallbackRequisiteZones, *enableStoragePoolsFlag, *enableControllerDataCacheFlag, multiZoneVolumeHandleConfig, listVolumesConfig) + controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration, fallbackRequisiteZones, *enableStoragePoolsFlag, *enableDataCacheFlag, multiZoneVolumeHandleConfig, listVolumesConfig) } else if *cloudConfigFilePath != "" { klog.Warningf("controller service is disabled but cloud config given - it has no effect") } @@ -236,7 +235,7 @@ func handle() { klog.Fatalf("Failed to set up metadata service: %v", err.Error()) } nsArgs := driver.NodeServerArgs{ - EnableDataCache: *enableNodeDataCacheFlag, + EnableDataCache: *enableDataCacheFlag, } nodeServer = driver.NewNodeServer(gceDriver, mounter, deviceUtils, meta, statter, nsArgs) if *maxConcurrentFormatAndMount > 0 { @@ -244,7 +243,7 @@ func handle() { } } - if *enableNodeDataCacheFlag { + if *enableDataCacheFlag { if nodeName == nil || *nodeName == "" { klog.Errorf("Data cache enabled, but --node-name not passed") } diff --git a/deploy/kubernetes/base/node_linux/node.yaml b/deploy/kubernetes/base/node_linux/node.yaml index 211963118..b191b2881 100644 --- a/deploy/kubernetes/base/node_linux/node.yaml +++ b/deploy/kubernetes/base/node_linux/node.yaml @@ -46,7 +46,7 @@ spec: - "--v=5" - "--endpoint=unix:/csi/csi.sock" - "--run-controller-service=false" - - "--enable-node-data-cache" + - "--enable-data-cache" - "--node-name=$(KUBE_NODE_NAME)" securityContext: privileged: true diff --git a/pkg/gce-pd-csi-driver/cache.go b/pkg/gce-pd-csi-driver/cache.go index 7d0f65c29..d9bd5454f 100644 --- a/pkg/gce-pd-csi-driver/cache.go +++ b/pkg/gce-pd-csi-driver/cache.go @@ -242,7 +242,6 @@ func getLvName(suffix string, volumeId string) string { } func createVg(volumeGroupName string, devicePath string, raidedLocalSsds string) error { - klog.V(2).Infof(" vgcreate=") args := []string{ "--zero", "y", diff --git a/test/e2e/utils/utils.go b/test/e2e/utils/utils.go index 3ffdfdd40..774c07753 100644 --- a/test/e2e/utils/utils.go +++ b/test/e2e/utils/utils.go @@ -67,8 +67,7 @@ func GCEClientAndDriverSetup(instance *remote.InstanceInfo, driverConfig DriverC "--use-instance-api-to-poll-attachment-disk-types=pd-ssd", "--use-instance-api-to-list-volumes-published-nodes", fmt.Sprintf("--fallback-requisite-zones=%s", strings.Join(driverConfig.Zones, ",")), - "--enable-controller-data-cache", - "--enable-node-data-cache", + "--enable-data-cache", fmt.Sprintf("--node-name=%s", utilcommon.TestNode), } extra_flags = append(extra_flags, fmt.Sprintf("--compute-endpoint=%s", driverConfig.ComputeEndpoint)) From 8b4d9e17bb6ddc5fb667715e30c54a1252b27f4e Mon Sep 17 00:00:00 2001 From: Engin Akdemir Date: Wed, 4 Jun 2025 19:27:44 +0000 Subject: [PATCH 04/25] Add hdMachineType to e2e setup code --- test/e2e/tests/setup_e2e_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/e2e/tests/setup_e2e_test.go b/test/e2e/tests/setup_e2e_test.go index 7a3bef149..136b57cce 100644 --- a/test/e2e/tests/setup_e2e_test.go +++ b/test/e2e/tests/setup_e2e_test.go @@ -51,6 +51,9 @@ var ( cloudtopHost = flag.Bool("cloudtop-host", false, "The local host is cloudtop, a kind of googler machine with special requirements to access GCP") extraDriverFlags = flag.String("extra-driver-flags", "", "Extra flags to pass to the driver") enableConfidentialCompute = flag.Bool("enable-confidential-compute", false, "Create VMs with confidential compute mode. This uses NVMe devices") + // Multi-writer is only supported on M3, C3, and N4 + // https://cloud.google.com/compute/docs/disks/sharing-disks-between-vms#hd-multi-writer + hdMachineType = flag.String("hyperdisk-machine-type", "c3-standard-4", "Type of machine to provision instance on") testContexts = []*remote.TestContext{} computeService *compute.Service From 040ec584d7fd1901fbb4e4af19923c5674d95ca5 Mon Sep 17 00:00:00 2001 From: samhalim Date: Thu, 20 Feb 2025 20:14:30 +0000 Subject: [PATCH 05/25] Implementing watcher & reboot stability for data cache to master branch. --- cmd/gce-pd-csi-driver/main.go | 35 +- go.mod | 4 +- go.sum | 2 + pkg/gce-pd-csi-driver/cache.go | 118 ++- .../github.com/fsnotify/fsnotify/.cirrus.yml | 14 + .../github.com/fsnotify/fsnotify/.gitignore | 14 +- .../github.com/fsnotify/fsnotify/CHANGELOG.md | 220 +++++- .../fsnotify/fsnotify/CONTRIBUTING.md | 162 +++- vendor/github.com/fsnotify/fsnotify/LICENSE | 47 +- vendor/github.com/fsnotify/fsnotify/README.md | 228 ++++-- .../fsnotify/fsnotify/backend_fen.go | 484 ++++++++++++ .../fsnotify/fsnotify/backend_inotify.go | 658 ++++++++++++++++ .../fsnotify/fsnotify/backend_kqueue.go | 733 ++++++++++++++++++ .../fsnotify/fsnotify/backend_other.go | 23 + .../fsnotify/fsnotify/backend_windows.go | 682 ++++++++++++++++ .../github.com/fsnotify/fsnotify/fsnotify.go | 495 +++++++++++- .../fsnotify/fsnotify/internal/darwin.go | 39 + .../fsnotify/internal/debug_darwin.go | 57 ++ .../fsnotify/internal/debug_dragonfly.go | 33 + .../fsnotify/internal/debug_freebsd.go | 42 + .../fsnotify/internal/debug_kqueue.go | 32 + .../fsnotify/fsnotify/internal/debug_linux.go | 56 ++ .../fsnotify/internal/debug_netbsd.go | 25 + .../fsnotify/internal/debug_openbsd.go | 28 + .../fsnotify/internal/debug_solaris.go | 45 ++ .../fsnotify/internal/debug_windows.go | 40 + .../fsnotify/fsnotify/internal/freebsd.go | 31 + .../fsnotify/fsnotify/internal/internal.go | 2 + .../fsnotify/fsnotify/internal/unix.go | 31 + .../fsnotify/fsnotify/internal/unix2.go | 7 + .../fsnotify/fsnotify/internal/windows.go | 41 + .../fsnotify/fsnotify/system_bsd.go | 7 + .../fsnotify/fsnotify/system_darwin.go | 8 + vendor/modules.txt | 7 +- 34 files changed, 4241 insertions(+), 209 deletions(-) create mode 100644 vendor/github.com/fsnotify/fsnotify/.cirrus.yml create mode 100644 vendor/github.com/fsnotify/fsnotify/backend_fen.go create mode 100644 vendor/github.com/fsnotify/fsnotify/backend_inotify.go create mode 100644 vendor/github.com/fsnotify/fsnotify/backend_kqueue.go create mode 100644 vendor/github.com/fsnotify/fsnotify/backend_other.go create mode 100644 vendor/github.com/fsnotify/fsnotify/backend_windows.go create mode 100644 vendor/github.com/fsnotify/fsnotify/internal/darwin.go create mode 100644 vendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go create mode 100644 vendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go create mode 100644 vendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go create mode 100644 vendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go create mode 100644 vendor/github.com/fsnotify/fsnotify/internal/debug_linux.go create mode 100644 vendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go create mode 100644 vendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go create mode 100644 vendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go create mode 100644 vendor/github.com/fsnotify/fsnotify/internal/debug_windows.go create mode 100644 vendor/github.com/fsnotify/fsnotify/internal/freebsd.go create mode 100644 vendor/github.com/fsnotify/fsnotify/internal/internal.go create mode 100644 vendor/github.com/fsnotify/fsnotify/internal/unix.go create mode 100644 vendor/github.com/fsnotify/fsnotify/internal/unix2.go create mode 100644 vendor/github.com/fsnotify/fsnotify/internal/windows.go create mode 100644 vendor/github.com/fsnotify/fsnotify/system_bsd.go create mode 100644 vendor/github.com/fsnotify/fsnotify/system_darwin.go diff --git a/cmd/gce-pd-csi-driver/main.go b/cmd/gce-pd-csi-driver/main.go index 296df1d0a..ccc6a67ae 100644 --- a/cmd/gce-pd-csi-driver/main.go +++ b/cmd/gce-pd-csi-driver/main.go @@ -241,14 +241,14 @@ func handle() { if *maxConcurrentFormatAndMount > 0 { nodeServer = nodeServer.WithSerializedFormatAndMount(*formatAndMountTimeout, *maxConcurrentFormatAndMount) } - } - - if *enableDataCacheFlag { - if nodeName == nil || *nodeName == "" { - klog.Errorf("Data cache enabled, but --node-name not passed") - } - if err := setupDataCache(ctx, *nodeName); err != nil { - klog.Errorf("DataCache setup failed: %v", err) + if *enableDataCacheFlag { + if nodeName == nil || *nodeName == "" { + klog.Errorf("Data Cache enabled, but --node-name not passed") + } + if err := setupDataCache(ctx, *nodeName, nodeServer.MetadataService.GetName()); err != nil { + klog.Errorf("DataCache setup failed: %v", err) + } + go driver.StartWatcher(*nodeName) } } @@ -331,8 +331,16 @@ func urlFlag(target **url.URL, name string, usage string) { }) } -func setupDataCache(ctx context.Context, nodeName string) error { - klog.V(2).Infof("Setting up data cache for node %s", nodeName) +func setupDataCache(ctx context.Context, nodeName string, nodeId string) error { + isAlreadyRaided, err := driver.IsRaided() + if err != nil { + klog.V(4).Infof("Errored while scanning for available LocalSSDs err:%v; continuing Raiding", err) + } else if isAlreadyRaided { + klog.V(4).Infof("Local SSDs are already RAIDed. Skipping Data Cache setup.") + return nil + } + + lssdCount := common.LocalSSDCountForDataCache if nodeName != common.TestNode { cfg, err := rest.InClusterConfig() if err != nil { @@ -357,6 +365,11 @@ func setupDataCache(ctx context.Context, nodeName string) error { return fmt.Errorf("Failed to Raid local SSDs, unable to setup data caching, got error %v", err) } - klog.V(2).Infof("Datacache enabled for node %s", nodeName) + // Initializing data cache node (VG checks w/ raided lssd) + if err := driver.InitializeDataCacheNode(nodeId); err != nil { + return err + } + + klog.V(4).Infof("LSSD caching is setup for the Data Cache enabled node %s", nodeName) return nil } diff --git a/go.mod b/go.mod index ab0f5fb53..0494ba324 100644 --- a/go.mod +++ b/go.mod @@ -57,8 +57,8 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful v2.9.5+incompatible // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect diff --git a/go.sum b/go.sum index 9bfabf281..0f436e461 100644 --- a/go.sum +++ b/go.sum @@ -1031,6 +1031,8 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fsouza/fake-gcs-server v0.0.0-20180612165233-e85be23bdaa8/go.mod h1:1/HufuJ+eaDf4KTnYdS6HJMGvMRU8d4cYTuu/1QaBbI= github.com/fsouza/fake-gcs-server v1.19.4/go.mod h1:I0/88nHCASqJJ5M7zVF0zKODkYTcuXFW5J5yajsNJnE= github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= diff --git a/pkg/gce-pd-csi-driver/cache.go b/pkg/gce-pd-csi-driver/cache.go index d9bd5454f..c9764d933 100644 --- a/pkg/gce-pd-csi-driver/cache.go +++ b/pkg/gce-pd-csi-driver/cache.go @@ -7,7 +7,10 @@ import ( "strings" csi "github.com/container-storage-interface/spec/lib/go/csi" - + fsnotify "github.com/fsnotify/fsnotify" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/klog/v2" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" @@ -42,7 +45,7 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str // Clean up Volume Group before adding the PD reduceVolumeGroup(volumeGroupName, true) } else { - err := createVg(volumeGroupName, devicePath, raidedLocalSsdPath) + err := createVg(volumeGroupName, raidedLocalSsdPath) if err != nil { return mainDevicePath, err } @@ -241,7 +244,7 @@ func getLvName(suffix string, volumeId string) string { return fmt.Sprintf("%s-%s", suffix, pvcName) } -func createVg(volumeGroupName string, devicePath string, raidedLocalSsds string) error { +func createVg(volumeGroupName string, raidedLocalSsds string) error { args := []string{ "--zero", "y", @@ -366,3 +369,112 @@ func isCachingSetup(mainLvName string) (error, bool) { } return nil, false } + +func fetchChunkSizeKiB(cacheSize string) (string, error) { + var chunkSize float64 + + cacheSizeInt, err := common.ConvertGiStringToInt64(cacheSize) + if err != nil { + return "0", err + } + // Chunksize should be divisible by 32Kib so we need (chunksize/32*1024)*32*1024 + chunkSize = (float64(cacheSizeInt) * GiB) / float64(maxAllowedChunks) + chunkSize = math.Round(chunkSize/(32*KiB)) * (32 * KiB) + chunkSize = math.Min(math.Max(chunkSize, minChunkSize), maxChunkSize) / KiB + // default chunk size unit KiB + return strconv.FormatInt(int64(chunkSize), 10) + "KiB", nil +} + +func InitializeDataCacheNode(nodeId string) error { + raidedLocalSsdPath, err := fetchRAIDedLocalSsdPath() + if err != nil { + return err + } + volumeGroupName := getVolumeGroupName(nodeId) + + vgExists := checkVgExists(volumeGroupName) + // Check if the required volume group already exists + if vgExists { + // Clean up Volume Group before adding the PD + reduceVolumeGroup(volumeGroupName, true) + + // validate that raidedLSSD is part of VG + err = validateRaidedLSSDinVG(volumeGroupName, raidedLocalSsdPath) + if err != nil { + return fmt.Errorf("failed validate local ssd in vg %v: %v", volumeGroupName, err) + } + } else { + err := createVg(volumeGroupName, raidedLocalSsdPath) + if err != nil { + return err + } + } + return nil +} + +func StartWatcher(nodeName string) { + dirToWatch := "/dev/" + watcher, err := fsnotify.NewWatcher() + if err != nil { + klog.V(2).ErrorS(err, "errored while creating watcher") + } + klog.V(2).Infof("Watcher started for directory %v", dirToWatch) + defer watcher.Close() + + // out of the box fsnotify can watch a single file, or a single directory + if err := watcher.Add(dirToWatch); err != nil { + klog.V(2).ErrorS(err, "errored while adding watcher directory") + } + errorCh := make(chan error, 1) + // Handle the error received from the watcher goroutine + go watchDiskDetaches(watcher, nodeName, errorCh) + + select { + case err := <-errorCh: + klog.Errorf("watcher encountered an error: %v", err) + } +} + +func watchDiskDetaches(watcher *fsnotify.Watcher, nodeName string, errorCh chan error) error { + for { + select { + // watch for errors + case err := <-watcher.Errors: + errorCh <- fmt.Errorf("disk update event errored: %v", err) + // watch for events + case event := <-watcher.Events: + // In case of an event i.e. creation or deletion of any new PV, we update the VG metadata. + // This might include some non-LVM changes, no harm in updating metadata multiple times. + reduceVolumeGroup(getVolumeGroupName(nodeName), true) + klog.V(2).Infof("disk attach/detach event %#v\n", event) + } + } +} + +func validateRaidedLSSDinVG(vgName string, lssdPath string) error { + args := []string{ + "--noheadings", + "-o", + "pv_name", + "--select", + "vg_name=" + vgName, + } + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "pvs", args...) + if err != nil { + return fmt.Errorf("errored while checking physical volume details %v: %s", err, info) + // On error info contains the error message which we cannot use for further steps + } + + if !strings.Contains(string(info), lssdPath) { + return addRaidedLSSDToVg(vgName, lssdPath) + } + return nil +} + +func addRaidedLSSDToVg(vgName, lssdPath string) error { + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgextend", []string{vgName, lssdPath}...) + if err != nil { + return fmt.Errorf("errored while extending VGs %v: %s", err, info) + } + return nil +} diff --git a/vendor/github.com/fsnotify/fsnotify/.cirrus.yml b/vendor/github.com/fsnotify/fsnotify/.cirrus.yml new file mode 100644 index 000000000..f4e7dbf37 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/.cirrus.yml @@ -0,0 +1,14 @@ +freebsd_task: + name: 'FreeBSD' + freebsd_instance: + image_family: freebsd-14-1 + install_script: + - pkg update -f + - pkg install -y go + test_script: + # run tests as user "cirrus" instead of root + - pw useradd cirrus -m + - chown -R cirrus:cirrus . + - FSNOTIFY_BUFFER=4096 sudo --preserve-env=FSNOTIFY_BUFFER -u cirrus go test -parallel 1 -race ./... + - sudo --preserve-env=FSNOTIFY_BUFFER -u cirrus go test -parallel 1 -race ./... + - FSNOTIFY_DEBUG=1 sudo --preserve-env=FSNOTIFY_BUFFER -u cirrus go test -parallel 1 -race -v ./... diff --git a/vendor/github.com/fsnotify/fsnotify/.gitignore b/vendor/github.com/fsnotify/fsnotify/.gitignore index 4cd0cbaf4..daea9dd6d 100644 --- a/vendor/github.com/fsnotify/fsnotify/.gitignore +++ b/vendor/github.com/fsnotify/fsnotify/.gitignore @@ -1,6 +1,10 @@ -# Setup a Global .gitignore for OS and editor generated files: -# https://help.github.com/articles/ignoring-files -# git config --global core.excludesfile ~/.gitignore_global +# go test -c output +*.test +*.test.exe -.vagrant -*.sublime-project +# Output of go build ./cmd/fsnotify +/fsnotify +/fsnotify.exe + +/test/kqueue +/test/a.out diff --git a/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md b/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md index cc01c08f5..fa854785d 100644 --- a/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md +++ b/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md @@ -1,11 +1,199 @@ # Changelog -All notable changes to this project will be documented in this file. +1.8.0 2023-10-31 +---------------- -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +### Additions -## [Unreleased] +- all: add `FSNOTIFY_DEBUG` to print debug logs to stderr ([#619]) + +### Changes and fixes + +- windows: fix behaviour of `WatchList()` to be consistent with other platforms ([#610]) + +- kqueue: ignore events with Ident=0 ([#590]) + +- kqueue: set O_CLOEXEC to prevent passing file descriptors to children ([#617]) + +- kqueue: emit events as "/path/dir/file" instead of "path/link/file" when watching a symlink ([#625]) + +- inotify: don't send event for IN_DELETE_SELF when also watching the parent ([#620]) + +- inotify: fix panic when calling Remove() in a goroutine ([#650]) + +- fen: allow watching subdirectories of watched directories ([#621]) + +[#590]: https://github.com/fsnotify/fsnotify/pull/590 +[#610]: https://github.com/fsnotify/fsnotify/pull/610 +[#617]: https://github.com/fsnotify/fsnotify/pull/617 +[#619]: https://github.com/fsnotify/fsnotify/pull/619 +[#620]: https://github.com/fsnotify/fsnotify/pull/620 +[#621]: https://github.com/fsnotify/fsnotify/pull/621 +[#625]: https://github.com/fsnotify/fsnotify/pull/625 +[#650]: https://github.com/fsnotify/fsnotify/pull/650 + +1.7.0 - 2023-10-22 +------------------ +This version of fsnotify needs Go 1.17. + +### Additions + +- illumos: add FEN backend to support illumos and Solaris. ([#371]) + +- all: add `NewBufferedWatcher()` to use a buffered channel, which can be useful + in cases where you can't control the kernel buffer and receive a large number + of events in bursts. ([#550], [#572]) + +- all: add `AddWith()`, which is identical to `Add()` but allows passing + options. ([#521]) + +- windows: allow setting the ReadDirectoryChangesW() buffer size with + `fsnotify.WithBufferSize()`; the default of 64K is the highest value that + works on all platforms and is enough for most purposes, but in some cases a + highest buffer is needed. ([#521]) + +### Changes and fixes + +- inotify: remove watcher if a watched path is renamed ([#518]) + + After a rename the reported name wasn't updated, or even an empty string. + Inotify doesn't provide any good facilities to update it, so just remove the + watcher. This is already how it worked on kqueue and FEN. + + On Windows this does work, and remains working. + +- windows: don't listen for file attribute changes ([#520]) + + File attribute changes are sent as `FILE_ACTION_MODIFIED` by the Windows API, + with no way to see if they're a file write or attribute change, so would show + up as a fsnotify.Write event. This is never useful, and could result in many + spurious Write events. + +- windows: return `ErrEventOverflow` if the buffer is full ([#525]) + + Before it would merely return "short read", making it hard to detect this + error. + +- kqueue: make sure events for all files are delivered properly when removing a + watched directory ([#526]) + + Previously they would get sent with `""` (empty string) or `"."` as the path + name. + +- kqueue: don't emit spurious Create events for symbolic links ([#524]) + + The link would get resolved but kqueue would "forget" it already saw the link + itself, resulting on a Create for every Write event for the directory. + +- all: return `ErrClosed` on `Add()` when the watcher is closed ([#516]) + +- other: add `Watcher.Errors` and `Watcher.Events` to the no-op `Watcher` in + `backend_other.go`, making it easier to use on unsupported platforms such as + WASM, AIX, etc. ([#528]) + +- other: use the `backend_other.go` no-op if the `appengine` build tag is set; + Google AppEngine forbids usage of the unsafe package so the inotify backend + won't compile there. + +[#371]: https://github.com/fsnotify/fsnotify/pull/371 +[#516]: https://github.com/fsnotify/fsnotify/pull/516 +[#518]: https://github.com/fsnotify/fsnotify/pull/518 +[#520]: https://github.com/fsnotify/fsnotify/pull/520 +[#521]: https://github.com/fsnotify/fsnotify/pull/521 +[#524]: https://github.com/fsnotify/fsnotify/pull/524 +[#525]: https://github.com/fsnotify/fsnotify/pull/525 +[#526]: https://github.com/fsnotify/fsnotify/pull/526 +[#528]: https://github.com/fsnotify/fsnotify/pull/528 +[#537]: https://github.com/fsnotify/fsnotify/pull/537 +[#550]: https://github.com/fsnotify/fsnotify/pull/550 +[#572]: https://github.com/fsnotify/fsnotify/pull/572 + +1.6.0 - 2022-10-13 +------------------ +This version of fsnotify needs Go 1.16 (this was already the case since 1.5.1, +but not documented). It also increases the minimum Linux version to 2.6.32. + +### Additions + +- all: add `Event.Has()` and `Op.Has()` ([#477]) + + This makes checking events a lot easier; for example: + + if event.Op&Write == Write && !(event.Op&Remove == Remove) { + } + + Becomes: + + if event.Has(Write) && !event.Has(Remove) { + } + +- all: add cmd/fsnotify ([#463]) + + A command-line utility for testing and some examples. + +### Changes and fixes + +- inotify: don't ignore events for files that don't exist ([#260], [#470]) + + Previously the inotify watcher would call `os.Lstat()` to check if a file + still exists before emitting events. + + This was inconsistent with other platforms and resulted in inconsistent event + reporting (e.g. when a file is quickly removed and re-created), and generally + a source of confusion. It was added in 2013 to fix a memory leak that no + longer exists. + +- all: return `ErrNonExistentWatch` when `Remove()` is called on a path that's + not watched ([#460]) + +- inotify: replace epoll() with non-blocking inotify ([#434]) + + Non-blocking inotify was not generally available at the time this library was + written in 2014, but now it is. As a result, the minimum Linux version is + bumped from 2.6.27 to 2.6.32. This hugely simplifies the code and is faster. + +- kqueue: don't check for events every 100ms ([#480]) + + The watcher would wake up every 100ms, even when there was nothing to do. Now + it waits until there is something to do. + +- macos: retry opening files on EINTR ([#475]) + +- kqueue: skip unreadable files ([#479]) + + kqueue requires a file descriptor for every file in a directory; this would + fail if a file was unreadable by the current user. Now these files are simply + skipped. + +- windows: fix renaming a watched directory if the parent is also watched ([#370]) + +- windows: increase buffer size from 4K to 64K ([#485]) + +- windows: close file handle on Remove() ([#288]) + +- kqueue: put pathname in the error if watching a file fails ([#471]) + +- inotify, windows: calling Close() more than once could race ([#465]) + +- kqueue: improve Close() performance ([#233]) + +- all: various documentation additions and clarifications. + +[#233]: https://github.com/fsnotify/fsnotify/pull/233 +[#260]: https://github.com/fsnotify/fsnotify/pull/260 +[#288]: https://github.com/fsnotify/fsnotify/pull/288 +[#370]: https://github.com/fsnotify/fsnotify/pull/370 +[#434]: https://github.com/fsnotify/fsnotify/pull/434 +[#460]: https://github.com/fsnotify/fsnotify/pull/460 +[#463]: https://github.com/fsnotify/fsnotify/pull/463 +[#465]: https://github.com/fsnotify/fsnotify/pull/465 +[#470]: https://github.com/fsnotify/fsnotify/pull/470 +[#471]: https://github.com/fsnotify/fsnotify/pull/471 +[#475]: https://github.com/fsnotify/fsnotify/pull/475 +[#477]: https://github.com/fsnotify/fsnotify/pull/477 +[#479]: https://github.com/fsnotify/fsnotify/pull/479 +[#480]: https://github.com/fsnotify/fsnotify/pull/480 +[#485]: https://github.com/fsnotify/fsnotify/pull/485 ## [1.5.4] - 2022-04-25 @@ -40,6 +228,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [#385](https://github.com/fsnotify/fsnotify/pull/385) * Go 1.14+: Fix unsafe pointer conversion [#325](https://github.com/fsnotify/fsnotify/pull/325) +## [1.4.9] - 2020-03-11 + +* Move example usage to the readme #329. This may resolve #328. + +## [1.4.8] - 2020-03-10 + +* CI: test more go versions (@nathany 1d13583d846ea9d66dcabbfefbfb9d8e6fb05216) +* Tests: Queued inotify events could have been read by the test before max_queued_events was hit (@matthias-stone #265) +* Tests: t.Fatalf -> t.Errorf in go routines (@gdey #266) +* CI: Less verbosity (@nathany #267) +* Tests: Darwin: Exchangedata is deprecated on 10.13 (@nathany #267) +* Tests: Check if channels are closed in the example (@alexeykazakov #244) +* CI: Only run golint on latest version of go and fix issues (@cpuguy83 #284) +* CI: Add windows to travis matrix (@cpuguy83 #284) +* Docs: Remover appveyor badge (@nathany 11844c0959f6fff69ba325d097fce35bd85a8e93) +* Linux: create epoll and pipe fds with close-on-exec (@JohannesEbke #219) +* Linux: open files with close-on-exec (@linxiulei #273) +* Docs: Plan to support fanotify (@nathany ab058b44498e8b7566a799372a39d150d9ea0119 ) +* Project: Add go.mod (@nathany #309) +* Project: Revise editor config (@nathany #309) +* Project: Update copyright for 2019 (@nathany #309) +* CI: Drop go1.8 from CI matrix (@nathany #309) +* Docs: Updating the FAQ section for supportability with NFS & FUSE filesystems (@Pratik32 4bf2d1fec78374803a39307bfb8d340688f4f28e ) + ## [1.4.7] - 2018-01-09 * BSD/macOS: Fix possible deadlock on closing the watcher on kqueue (thanks @nhooyr and @glycerine) diff --git a/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md b/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md index 8a642563d..e4ac2a2ff 100644 --- a/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +++ b/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md @@ -1,60 +1,144 @@ -# Contributing +Thank you for your interest in contributing to fsnotify! We try to review and +merge PRs in a reasonable timeframe, but please be aware that: -## Issues +- To avoid "wasted" work, please discuss changes on the issue tracker first. You + can just send PRs, but they may end up being rejected for one reason or the + other. -* Request features and report bugs using the [GitHub Issue Tracker](https://github.com/fsnotify/fsnotify/issues). -* Please indicate the platform you are using fsnotify on. -* A code example to reproduce the problem is appreciated. +- fsnotify is a cross-platform library, and changes must work reasonably well on + all supported platforms. -## Pull Requests +- Changes will need to be compatible; old code should still compile, and the + runtime behaviour can't change in ways that are likely to lead to problems for + users. -### Contributor License Agreement +Testing +------- +Just `go test ./...` runs all the tests; the CI runs this on all supported +platforms. Testing different platforms locally can be done with something like +[goon] or [Vagrant], but this isn't super-easy to set up at the moment. -fsnotify is derived from code in the [golang.org/x/exp](https://godoc.org/golang.org/x/exp) package and it may be included [in the standard library](https://github.com/fsnotify/fsnotify/issues/1) in the future. Therefore fsnotify carries the same [LICENSE](https://github.com/fsnotify/fsnotify/blob/master/LICENSE) as Go. Contributors retain their copyright, so you need to fill out a short form before we can accept your contribution: [Google Individual Contributor License Agreement](https://developers.google.com/open-source/cla/individual). +Use the `-short` flag to make the "stress test" run faster. -Please indicate that you have signed the CLA in your pull request. +Writing new tests +----------------- +Scripts in the testdata directory allow creating test cases in a "shell-like" +syntax. The basic format is: -### How fsnotify is Developed + script -* Development is done on feature branches. -* Tests are run on BSD, Linux, macOS and Windows. -* Pull requests are reviewed and [applied to master][am] using [hub][]. - * Maintainers may modify or squash commits rather than asking contributors to. -* To issue a new release, the maintainers will: - * Update the CHANGELOG - * Tag a version, which will become available through gopkg.in. - -### How to Fork + Output: + desired output -For smooth sailing, always use the original import path. Installing with `go get` makes this easy. +For example: -1. Install from GitHub (`go get -u github.com/fsnotify/fsnotify`) -2. Create your feature branch (`git checkout -b my-new-feature`) -3. Ensure everything works and the tests pass (see below) -4. Commit your changes (`git commit -am 'Add some feature'`) + # Create a new empty file with some data. + watch / + echo data >/file -Contribute upstream: + Output: + create /file + write /file -1. Fork fsnotify on GitHub -2. Add your remote (`git remote add fork git@github.com:mycompany/repo.git`) -3. Push to the branch (`git push fork my-new-feature`) -4. Create a new Pull Request on GitHub +Just create a new file to add a new test; select which tests to run with +`-run TestScript/[path]`. -This workflow is [thoroughly explained by Katrina Owen](https://splice.com/blog/contributing-open-source-git-repositories-go/). +script +------ +The script is a "shell-like" script: -### Testing + cmd arg arg -fsnotify uses build tags to compile different code on Linux, BSD, macOS, and Windows. +Comments are supported with `#`: -Before doing a pull request, please do your best to test your changes on multiple platforms, and list which platforms you were able/unable to test on. + # Comment + cmd arg arg # Comment -### Maintainers +All operations are done in a temp directory; a path like "/foo" is rewritten to +"/tmp/TestFoo/foo". -Help maintaining fsnotify is welcome. To be a maintainer: +Arguments can be quoted with `"` or `'`; there are no escapes and they're +functionally identical right now, but this may change in the future, so best to +assume shell-like rules. -* Submit a pull request and sign the CLA as above. -* You must be able to run the test suite on Mac, Windows, Linux and BSD. + touch "/file with spaces" -All code changes should be internal pull requests. +End-of-line escapes with `\` are not supported. -Releases are tagged using [Semantic Versioning](http://semver.org/). +### Supported commands + + watch path [ops] # Watch the path, reporting events for it. Nothing is + # watched by default. Optionally a list of ops can be + # given, as with AddWith(path, WithOps(...)). + unwatch path # Stop watching the path. + watchlist n # Assert watchlist length. + + stop # Stop running the script; for debugging. + debug [yes/no] # Enable/disable FSNOTIFY_DEBUG (tests are run in + parallel by default, so -parallel=1 is probably a good + idea). + + touch path + mkdir [-p] dir + ln -s target link # Only ln -s supported. + mkfifo path + mknod dev path + mv src dst + rm [-r] path + chmod mode path # Octal only + sleep time-in-ms + + cat path # Read path (does nothing with the data; just reads it). + echo str >>path # Append "str" to "path". + echo str >path # Truncate "path" and write "str". + + require reason # Skip the test if "reason" is true; "skip" and + skip reason # "require" behave identical; it supports both for + # readability. Possible reasons are: + # + # always Always skip this test. + # symlink Symlinks are supported (requires admin + # permissions on Windows). + # mkfifo Platform doesn't support FIFO named sockets. + # mknod Platform doesn't support device nodes. + + +output +------ +After `Output:` the desired output is given; this is indented by convention, but +that's not required. + +The format of that is: + + # Comment + event path # Comment + + system: + event path + system2: + event path + +Every event is one line, and any whitespace between the event and path are +ignored. The path can optionally be surrounded in ". Anything after a "#" is +ignored. + +Platform-specific tests can be added after GOOS; for example: + + watch / + touch /file + + Output: + # Tested if nothing else matches + create /file + + # Windows-specific test. + windows: + write /file + +You can specify multiple platforms with a comma (e.g. "windows, linux:"). +"kqueue" is a shortcut for all kqueue systems (BSD, macOS). + + +[goon]: https://github.com/arp242/goon +[Vagrant]: https://www.vagrantup.com/ +[integration_test.go]: /integration_test.go diff --git a/vendor/github.com/fsnotify/fsnotify/LICENSE b/vendor/github.com/fsnotify/fsnotify/LICENSE index e180c8fb0..fb03ade75 100644 --- a/vendor/github.com/fsnotify/fsnotify/LICENSE +++ b/vendor/github.com/fsnotify/fsnotify/LICENSE @@ -1,28 +1,25 @@ -Copyright (c) 2012 The Go Authors. All rights reserved. -Copyright (c) 2012-2019 fsnotify Authors. All rights reserved. +Copyright © 2012 The Go Authors. All rights reserved. +Copyright © fsnotify Authors. All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. +* Neither the name of Google Inc. nor the names of its contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/fsnotify/fsnotify/README.md b/vendor/github.com/fsnotify/fsnotify/README.md index 0731c5ef8..e480733d1 100644 --- a/vendor/github.com/fsnotify/fsnotify/README.md +++ b/vendor/github.com/fsnotify/fsnotify/README.md @@ -1,120 +1,184 @@ -# File system notifications for Go +fsnotify is a Go library to provide cross-platform filesystem notifications on +Windows, Linux, macOS, BSD, and illumos. -[![Go Reference](https://pkg.go.dev/badge/github.com/fsnotify/fsnotify.svg)](https://pkg.go.dev/github.com/fsnotify/fsnotify) [![Go Report Card](https://goreportcard.com/badge/github.com/fsnotify/fsnotify)](https://goreportcard.com/report/github.com/fsnotify/fsnotify) [![Maintainers Wanted](https://img.shields.io/badge/maintainers-wanted-red.svg)](https://github.com/fsnotify/fsnotify/issues/413) +Go 1.17 or newer is required; the full documentation is at +https://pkg.go.dev/github.com/fsnotify/fsnotify -fsnotify utilizes [`golang.org/x/sys`](https://pkg.go.dev/golang.org/x/sys) rather than [`syscall`](https://pkg.go.dev/syscall) from the standard library. +--- -Cross platform: Windows, Linux, BSD and macOS. +Platform support: -| Adapter | OS | Status | -| --------------------- | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | -| inotify | Linux 2.6.27 or later, Android\* | Supported | -| kqueue | BSD, macOS, iOS\* | Supported | -| ReadDirectoryChangesW | Windows | Supported | -| FSEvents | macOS | [Planned](https://github.com/fsnotify/fsnotify/issues/11) | -| FEN | Solaris 11 | [In Progress](https://github.com/fsnotify/fsnotify/pull/371) | -| fanotify | Linux 2.6.37+ | [Maybe](https://github.com/fsnotify/fsnotify/issues/114) | -| USN Journals | Windows | [Maybe](https://github.com/fsnotify/fsnotify/issues/53) | -| Polling | *All* | [Maybe](https://github.com/fsnotify/fsnotify/issues/9) | +| Backend | OS | Status | +| :-------------------- | :--------- | :------------------------------------------------------------------------ | +| inotify | Linux | Supported | +| kqueue | BSD, macOS | Supported | +| ReadDirectoryChangesW | Windows | Supported | +| FEN | illumos | Supported | +| fanotify | Linux 5.9+ | [Not yet](https://github.com/fsnotify/fsnotify/issues/114) | +| AHAFS | AIX | [aix branch]; experimental due to lack of maintainer and test environment | +| FSEvents | macOS | [Needs support in x/sys/unix][fsevents] | +| USN Journals | Windows | [Needs support in x/sys/windows][usn] | +| Polling | *All* | [Not yet](https://github.com/fsnotify/fsnotify/issues/9) | -\* Android and iOS are untested. +Linux and illumos should include Android and Solaris, but these are currently +untested. -Please see [the documentation](https://pkg.go.dev/github.com/fsnotify/fsnotify) and consult the [FAQ](#faq) for usage information. +[fsevents]: https://github.com/fsnotify/fsnotify/issues/11#issuecomment-1279133120 +[usn]: https://github.com/fsnotify/fsnotify/issues/53#issuecomment-1279829847 +[aix branch]: https://github.com/fsnotify/fsnotify/issues/353#issuecomment-1284590129 -## API stability - -fsnotify is a fork of [howeyc/fsnotify](https://github.com/howeyc/fsnotify) with a new API as of v1.0. The API is based on [this design document](http://goo.gl/MrYxyA). - -All [releases](https://github.com/fsnotify/fsnotify/releases) are tagged based on [Semantic Versioning](http://semver.org/). - -## Usage +Usage +----- +A basic example: ```go package main import ( - "log" + "log" - "github.com/fsnotify/fsnotify" + "github.com/fsnotify/fsnotify" ) func main() { - watcher, err := fsnotify.NewWatcher() - if err != nil { - log.Fatal(err) - } - defer watcher.Close() - - done := make(chan bool) - go func() { - for { - select { - case event, ok := <-watcher.Events: - if !ok { - return - } - log.Println("event:", event) - if event.Op&fsnotify.Write == fsnotify.Write { - log.Println("modified file:", event.Name) - } - case err, ok := <-watcher.Errors: - if !ok { - return - } - log.Println("error:", err) - } - } - }() - - err = watcher.Add("/tmp/foo") - if err != nil { - log.Fatal(err) - } - <-done + // Create new watcher. + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal(err) + } + defer watcher.Close() + + // Start listening for events. + go func() { + for { + select { + case event, ok := <-watcher.Events: + if !ok { + return + } + log.Println("event:", event) + if event.Has(fsnotify.Write) { + log.Println("modified file:", event.Name) + } + case err, ok := <-watcher.Errors: + if !ok { + return + } + log.Println("error:", err) + } + } + }() + + // Add a path. + err = watcher.Add("/tmp") + if err != nil { + log.Fatal(err) + } + + // Block main goroutine forever. + <-make(chan struct{}) } ``` -## Contributing +Some more examples can be found in [cmd/fsnotify](cmd/fsnotify), which can be +run with: + + % go run ./cmd/fsnotify -Please refer to [CONTRIBUTING][] before opening an issue or pull request. +Further detailed documentation can be found in godoc: +https://pkg.go.dev/github.com/fsnotify/fsnotify -## FAQ +FAQ +--- +### Will a file still be watched when it's moved to another directory? +No, not unless you are watching the location it was moved to. -**When a file is moved to another directory is it still being watched?** +### Are subdirectories watched? +No, you must add watches for any directory you want to watch (a recursive +watcher is on the roadmap: [#18]). -No (it shouldn't be, unless you are watching where it was moved to). +[#18]: https://github.com/fsnotify/fsnotify/issues/18 -**When I watch a directory, are all subdirectories watched as well?** +### Do I have to watch the Error and Event channels in a goroutine? +Yes. You can read both channels in the same goroutine using `select` (you don't +need a separate goroutine for both channels; see the example). -No, you must add watches for any directory you want to watch (a recursive watcher is on the roadmap [#18][]). +### Why don't notifications work with NFS, SMB, FUSE, /proc, or /sys? +fsnotify requires support from underlying OS to work. The current NFS and SMB +protocols does not provide network level support for file notifications, and +neither do the /proc and /sys virtual filesystems. -**Do I have to watch the Error and Event channels in a separate goroutine?** +This could be fixed with a polling watcher ([#9]), but it's not yet implemented. -As of now, yes. Looking into making this single-thread friendly (see [howeyc #7][#7]) +[#9]: https://github.com/fsnotify/fsnotify/issues/9 -**Why am I receiving multiple events for the same file on OS X?** +### Why do I get many Chmod events? +Some programs may generate a lot of attribute changes; for example Spotlight on +macOS, anti-virus programs, backup applications, and some others are known to do +this. As a rule, it's typically best to ignore Chmod events. They're often not +useful, and tend to cause problems. -Spotlight indexing on OS X can result in multiple events (see [howeyc #62][#62]). A temporary workaround is to add your folder(s) to the *Spotlight Privacy settings* until we have a native FSEvents implementation (see [#11][]). +Spotlight indexing on macOS can result in multiple events (see [#15]). A +temporary workaround is to add your folder(s) to the *Spotlight Privacy +settings* until we have a native FSEvents implementation (see [#11]). -**How many files can be watched at once?** +[#11]: https://github.com/fsnotify/fsnotify/issues/11 +[#15]: https://github.com/fsnotify/fsnotify/issues/15 -There are OS-specific limits as to how many watches can be created: -* Linux: /proc/sys/fs/inotify/max_user_watches contains the limit, reaching this limit results in a "no space left on device" error. -* BSD / OSX: sysctl variables "kern.maxfiles" and "kern.maxfilesperproc", reaching these limits results in a "too many open files" error. +### Watching a file doesn't work well +Watching individual files (rather than directories) is generally not recommended +as many programs (especially editors) update files atomically: it will write to +a temporary file which is then moved to to destination, overwriting the original +(or some variant thereof). The watcher on the original file is now lost, as that +no longer exists. -**Why don't notifications work with NFS filesystems or filesystem in userspace (FUSE)?** +The upshot of this is that a power failure or crash won't leave a half-written +file. -fsnotify requires support from underlying OS to work. The current NFS protocol does not provide network level support for file notifications. +Watch the parent directory and use `Event.Name` to filter out files you're not +interested in. There is an example of this in `cmd/fsnotify/file.go`. -[#62]: https://github.com/howeyc/fsnotify/issues/62 -[#18]: https://github.com/fsnotify/fsnotify/issues/18 -[#11]: https://github.com/fsnotify/fsnotify/issues/11 -[#7]: https://github.com/howeyc/fsnotify/issues/7 +Platform-specific notes +----------------------- +### Linux +When a file is removed a REMOVE event won't be emitted until all file +descriptors are closed; it will emit a CHMOD instead: + + fp := os.Open("file") + os.Remove("file") // CHMOD + fp.Close() // REMOVE + +This is the event that inotify sends, so not much can be changed about this. + +The `fs.inotify.max_user_watches` sysctl variable specifies the upper limit for +the number of watches per user, and `fs.inotify.max_user_instances` specifies +the maximum number of inotify instances per user. Every Watcher you create is an +"instance", and every path you add is a "watch". + +These are also exposed in `/proc` as `/proc/sys/fs/inotify/max_user_watches` and +`/proc/sys/fs/inotify/max_user_instances` + +To increase them you can use `sysctl` or write the value to proc file: + + # The default values on Linux 5.18 + sysctl fs.inotify.max_user_watches=124983 + sysctl fs.inotify.max_user_instances=128 + +To make the changes persist on reboot edit `/etc/sysctl.conf` or +`/usr/lib/sysctl.d/50-default.conf` (details differ per Linux distro; check your +distro's documentation): -[contributing]: https://github.com/fsnotify/fsnotify/blob/master/CONTRIBUTING.md + fs.inotify.max_user_watches=124983 + fs.inotify.max_user_instances=128 -## Related Projects +Reaching the limit will result in a "no space left on device" or "too many open +files" error. -* [notify](https://github.com/rjeczalik/notify) -* [fsevents](https://github.com/fsnotify/fsevents) +### kqueue (macOS, all BSD systems) +kqueue requires opening a file descriptor for every file that's being watched; +so if you're watching a directory with five files then that's six file +descriptors. You will run in to your system's "max open files" limit faster on +these platforms. +The sysctl variables `kern.maxfiles` and `kern.maxfilesperproc` can be used to +control the maximum number of open files. diff --git a/vendor/github.com/fsnotify/fsnotify/backend_fen.go b/vendor/github.com/fsnotify/fsnotify/backend_fen.go new file mode 100644 index 000000000..c349c326c --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/backend_fen.go @@ -0,0 +1,484 @@ +//go:build solaris + +// FEN backend for illumos (supported) and Solaris (untested, but should work). +// +// See port_create(3c) etc. for docs. https://www.illumos.org/man/3C/port_create + +package fsnotify + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/fsnotify/fsnotify/internal" + "golang.org/x/sys/unix" +) + +type fen struct { + Events chan Event + Errors chan error + + mu sync.Mutex + port *unix.EventPort + done chan struct{} // Channel for sending a "quit message" to the reader goroutine + dirs map[string]Op // Explicitly watched directories + watches map[string]Op // Explicitly watched non-directories +} + +func newBackend(ev chan Event, errs chan error) (backend, error) { + return newBufferedBackend(0, ev, errs) +} + +func newBufferedBackend(sz uint, ev chan Event, errs chan error) (backend, error) { + w := &fen{ + Events: ev, + Errors: errs, + dirs: make(map[string]Op), + watches: make(map[string]Op), + done: make(chan struct{}), + } + + var err error + w.port, err = unix.NewEventPort() + if err != nil { + return nil, fmt.Errorf("fsnotify.NewWatcher: %w", err) + } + + go w.readEvents() + return w, nil +} + +// sendEvent attempts to send an event to the user, returning true if the event +// was put in the channel successfully and false if the watcher has been closed. +func (w *fen) sendEvent(name string, op Op) (sent bool) { + select { + case <-w.done: + return false + case w.Events <- Event{Name: name, Op: op}: + return true + } +} + +// sendError attempts to send an error to the user, returning true if the error +// was put in the channel successfully and false if the watcher has been closed. +func (w *fen) sendError(err error) (sent bool) { + if err == nil { + return true + } + select { + case <-w.done: + return false + case w.Errors <- err: + return true + } +} + +func (w *fen) isClosed() bool { + select { + case <-w.done: + return true + default: + return false + } +} + +func (w *fen) Close() error { + // Take the lock used by associateFile to prevent lingering events from + // being processed after the close + w.mu.Lock() + defer w.mu.Unlock() + if w.isClosed() { + return nil + } + close(w.done) + return w.port.Close() +} + +func (w *fen) Add(name string) error { return w.AddWith(name) } + +func (w *fen) AddWith(name string, opts ...addOpt) error { + if w.isClosed() { + return ErrClosed + } + if debug { + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s AddWith(%q)\n", + time.Now().Format("15:04:05.000000000"), name) + } + + with := getOptions(opts...) + if !w.xSupports(with.op) { + return fmt.Errorf("%w: %s", xErrUnsupported, with.op) + } + + // Currently we resolve symlinks that were explicitly requested to be + // watched. Otherwise we would use LStat here. + stat, err := os.Stat(name) + if err != nil { + return err + } + + // Associate all files in the directory. + if stat.IsDir() { + err := w.handleDirectory(name, stat, true, w.associateFile) + if err != nil { + return err + } + + w.mu.Lock() + w.dirs[name] = with.op + w.mu.Unlock() + return nil + } + + err = w.associateFile(name, stat, true) + if err != nil { + return err + } + + w.mu.Lock() + w.watches[name] = with.op + w.mu.Unlock() + return nil +} + +func (w *fen) Remove(name string) error { + if w.isClosed() { + return nil + } + if !w.port.PathIsWatched(name) { + return fmt.Errorf("%w: %s", ErrNonExistentWatch, name) + } + if debug { + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s Remove(%q)\n", + time.Now().Format("15:04:05.000000000"), name) + } + + // The user has expressed an intent. Immediately remove this name from + // whichever watch list it might be in. If it's not in there the delete + // doesn't cause harm. + w.mu.Lock() + delete(w.watches, name) + delete(w.dirs, name) + w.mu.Unlock() + + stat, err := os.Stat(name) + if err != nil { + return err + } + + // Remove associations for every file in the directory. + if stat.IsDir() { + err := w.handleDirectory(name, stat, false, w.dissociateFile) + if err != nil { + return err + } + return nil + } + + err = w.port.DissociatePath(name) + if err != nil { + return err + } + + return nil +} + +// readEvents contains the main loop that runs in a goroutine watching for events. +func (w *fen) readEvents() { + // If this function returns, the watcher has been closed and we can close + // these channels + defer func() { + close(w.Errors) + close(w.Events) + }() + + pevents := make([]unix.PortEvent, 8) + for { + count, err := w.port.Get(pevents, 1, nil) + if err != nil && err != unix.ETIME { + // Interrupted system call (count should be 0) ignore and continue + if errors.Is(err, unix.EINTR) && count == 0 { + continue + } + // Get failed because we called w.Close() + if errors.Is(err, unix.EBADF) && w.isClosed() { + return + } + // There was an error not caused by calling w.Close() + if !w.sendError(err) { + return + } + } + + p := pevents[:count] + for _, pevent := range p { + if pevent.Source != unix.PORT_SOURCE_FILE { + // Event from unexpected source received; should never happen. + if !w.sendError(errors.New("Event from unexpected source received")) { + return + } + continue + } + + if debug { + internal.Debug(pevent.Path, pevent.Events) + } + + err = w.handleEvent(&pevent) + if !w.sendError(err) { + return + } + } + } +} + +func (w *fen) handleDirectory(path string, stat os.FileInfo, follow bool, handler func(string, os.FileInfo, bool) error) error { + files, err := os.ReadDir(path) + if err != nil { + return err + } + + // Handle all children of the directory. + for _, entry := range files { + finfo, err := entry.Info() + if err != nil { + return err + } + err = handler(filepath.Join(path, finfo.Name()), finfo, false) + if err != nil { + return err + } + } + + // And finally handle the directory itself. + return handler(path, stat, follow) +} + +// handleEvent might need to emit more than one fsnotify event if the events +// bitmap matches more than one event type (e.g. the file was both modified and +// had the attributes changed between when the association was created and the +// when event was returned) +func (w *fen) handleEvent(event *unix.PortEvent) error { + var ( + events = event.Events + path = event.Path + fmode = event.Cookie.(os.FileMode) + reRegister = true + ) + + w.mu.Lock() + _, watchedDir := w.dirs[path] + _, watchedPath := w.watches[path] + w.mu.Unlock() + isWatched := watchedDir || watchedPath + + if events&unix.FILE_DELETE != 0 { + if !w.sendEvent(path, Remove) { + return nil + } + reRegister = false + } + if events&unix.FILE_RENAME_FROM != 0 { + if !w.sendEvent(path, Rename) { + return nil + } + // Don't keep watching the new file name + reRegister = false + } + if events&unix.FILE_RENAME_TO != 0 { + // We don't report a Rename event for this case, because Rename events + // are interpreted as referring to the _old_ name of the file, and in + // this case the event would refer to the new name of the file. This + // type of rename event is not supported by fsnotify. + + // inotify reports a Remove event in this case, so we simulate this + // here. + if !w.sendEvent(path, Remove) { + return nil + } + // Don't keep watching the file that was removed + reRegister = false + } + + // The file is gone, nothing left to do. + if !reRegister { + if watchedDir { + w.mu.Lock() + delete(w.dirs, path) + w.mu.Unlock() + } + if watchedPath { + w.mu.Lock() + delete(w.watches, path) + w.mu.Unlock() + } + return nil + } + + // If we didn't get a deletion the file still exists and we're going to have + // to watch it again. Let's Stat it now so that we can compare permissions + // and have what we need to continue watching the file + + stat, err := os.Lstat(path) + if err != nil { + // This is unexpected, but we should still emit an event. This happens + // most often on "rm -r" of a subdirectory inside a watched directory We + // get a modify event of something happening inside, but by the time we + // get here, the sudirectory is already gone. Clearly we were watching + // this path but now it is gone. Let's tell the user that it was + // removed. + if !w.sendEvent(path, Remove) { + return nil + } + // Suppress extra write events on removed directories; they are not + // informative and can be confusing. + return nil + } + + // resolve symlinks that were explicitly watched as we would have at Add() + // time. this helps suppress spurious Chmod events on watched symlinks + if isWatched { + stat, err = os.Stat(path) + if err != nil { + // The symlink still exists, but the target is gone. Report the + // Remove similar to above. + if !w.sendEvent(path, Remove) { + return nil + } + // Don't return the error + } + } + + if events&unix.FILE_MODIFIED != 0 { + if fmode.IsDir() && watchedDir { + if err := w.updateDirectory(path); err != nil { + return err + } + } else { + if !w.sendEvent(path, Write) { + return nil + } + } + } + if events&unix.FILE_ATTRIB != 0 && stat != nil { + // Only send Chmod if perms changed + if stat.Mode().Perm() != fmode.Perm() { + if !w.sendEvent(path, Chmod) { + return nil + } + } + } + + if stat != nil { + // If we get here, it means we've hit an event above that requires us to + // continue watching the file or directory + return w.associateFile(path, stat, isWatched) + } + return nil +} + +func (w *fen) updateDirectory(path string) error { + // The directory was modified, so we must find unwatched entities and watch + // them. If something was removed from the directory, nothing will happen, + // as everything else should still be watched. + files, err := os.ReadDir(path) + if err != nil { + return err + } + + for _, entry := range files { + path := filepath.Join(path, entry.Name()) + if w.port.PathIsWatched(path) { + continue + } + + finfo, err := entry.Info() + if err != nil { + return err + } + err = w.associateFile(path, finfo, false) + if !w.sendError(err) { + return nil + } + if !w.sendEvent(path, Create) { + return nil + } + } + return nil +} + +func (w *fen) associateFile(path string, stat os.FileInfo, follow bool) error { + if w.isClosed() { + return ErrClosed + } + // This is primarily protecting the call to AssociatePath but it is + // important and intentional that the call to PathIsWatched is also + // protected by this mutex. Without this mutex, AssociatePath has been seen + // to error out that the path is already associated. + w.mu.Lock() + defer w.mu.Unlock() + + if w.port.PathIsWatched(path) { + // Remove the old association in favor of this one If we get ENOENT, + // then while the x/sys/unix wrapper still thought that this path was + // associated, the underlying event port did not. This call will have + // cleared up that discrepancy. The most likely cause is that the event + // has fired but we haven't processed it yet. + err := w.port.DissociatePath(path) + if err != nil && !errors.Is(err, unix.ENOENT) { + return err + } + } + + var events int + if !follow { + // Watch symlinks themselves rather than their targets unless this entry + // is explicitly watched. + events |= unix.FILE_NOFOLLOW + } + if true { // TODO: implement withOps() + events |= unix.FILE_MODIFIED + } + if true { + events |= unix.FILE_ATTRIB + } + return w.port.AssociatePath(path, stat, events, stat.Mode()) +} + +func (w *fen) dissociateFile(path string, stat os.FileInfo, unused bool) error { + if !w.port.PathIsWatched(path) { + return nil + } + return w.port.DissociatePath(path) +} + +func (w *fen) WatchList() []string { + if w.isClosed() { + return nil + } + + w.mu.Lock() + defer w.mu.Unlock() + + entries := make([]string, 0, len(w.watches)+len(w.dirs)) + for pathname := range w.dirs { + entries = append(entries, pathname) + } + for pathname := range w.watches { + entries = append(entries, pathname) + } + + return entries +} + +func (w *fen) xSupports(op Op) bool { + if op.Has(xUnportableOpen) || op.Has(xUnportableRead) || + op.Has(xUnportableCloseWrite) || op.Has(xUnportableCloseRead) { + return false + } + return true +} diff --git a/vendor/github.com/fsnotify/fsnotify/backend_inotify.go b/vendor/github.com/fsnotify/fsnotify/backend_inotify.go new file mode 100644 index 000000000..36c311694 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/backend_inotify.go @@ -0,0 +1,658 @@ +//go:build linux && !appengine + +package fsnotify + +import ( + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "sync" + "time" + "unsafe" + + "github.com/fsnotify/fsnotify/internal" + "golang.org/x/sys/unix" +) + +type inotify struct { + Events chan Event + Errors chan error + + // Store fd here as os.File.Read() will no longer return on close after + // calling Fd(). See: https://github.com/golang/go/issues/26439 + fd int + inotifyFile *os.File + watches *watches + done chan struct{} // Channel for sending a "quit message" to the reader goroutine + doneMu sync.Mutex + doneResp chan struct{} // Channel to respond to Close + + // Store rename cookies in an array, with the index wrapping to 0. Almost + // all of the time what we get is a MOVED_FROM to set the cookie and the + // next event inotify sends will be MOVED_TO to read it. However, this is + // not guaranteed – as described in inotify(7) – and we may get other events + // between the two MOVED_* events (including other MOVED_* ones). + // + // A second issue is that moving a file outside the watched directory will + // trigger a MOVED_FROM to set the cookie, but we never see the MOVED_TO to + // read and delete it. So just storing it in a map would slowly leak memory. + // + // Doing it like this gives us a simple fast LRU-cache that won't allocate. + // Ten items should be more than enough for our purpose, and a loop over + // such a short array is faster than a map access anyway (not that it hugely + // matters since we're talking about hundreds of ns at the most, but still). + cookies [10]koekje + cookieIndex uint8 + cookiesMu sync.Mutex +} + +type ( + watches struct { + mu sync.RWMutex + wd map[uint32]*watch // wd → watch + path map[string]uint32 // pathname → wd + } + watch struct { + wd uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall) + flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags) + path string // Watch path. + recurse bool // Recursion with ./...? + } + koekje struct { + cookie uint32 + path string + } +) + +func newWatches() *watches { + return &watches{ + wd: make(map[uint32]*watch), + path: make(map[string]uint32), + } +} + +func (w *watches) len() int { + w.mu.RLock() + defer w.mu.RUnlock() + return len(w.wd) +} + +func (w *watches) add(ww *watch) { + w.mu.Lock() + defer w.mu.Unlock() + w.wd[ww.wd] = ww + w.path[ww.path] = ww.wd +} + +func (w *watches) remove(wd uint32) { + w.mu.Lock() + defer w.mu.Unlock() + watch := w.wd[wd] // Could have had Remove() called. See #616. + if watch == nil { + return + } + delete(w.path, watch.path) + delete(w.wd, wd) +} + +func (w *watches) removePath(path string) ([]uint32, error) { + w.mu.Lock() + defer w.mu.Unlock() + + path, recurse := recursivePath(path) + wd, ok := w.path[path] + if !ok { + return nil, fmt.Errorf("%w: %s", ErrNonExistentWatch, path) + } + + watch := w.wd[wd] + if recurse && !watch.recurse { + return nil, fmt.Errorf("can't use /... with non-recursive watch %q", path) + } + + delete(w.path, path) + delete(w.wd, wd) + if !watch.recurse { + return []uint32{wd}, nil + } + + wds := make([]uint32, 0, 8) + wds = append(wds, wd) + for p, rwd := range w.path { + if filepath.HasPrefix(p, path) { + delete(w.path, p) + delete(w.wd, rwd) + wds = append(wds, rwd) + } + } + return wds, nil +} + +func (w *watches) byPath(path string) *watch { + w.mu.RLock() + defer w.mu.RUnlock() + return w.wd[w.path[path]] +} + +func (w *watches) byWd(wd uint32) *watch { + w.mu.RLock() + defer w.mu.RUnlock() + return w.wd[wd] +} + +func (w *watches) updatePath(path string, f func(*watch) (*watch, error)) error { + w.mu.Lock() + defer w.mu.Unlock() + + var existing *watch + wd, ok := w.path[path] + if ok { + existing = w.wd[wd] + } + + upd, err := f(existing) + if err != nil { + return err + } + if upd != nil { + w.wd[upd.wd] = upd + w.path[upd.path] = upd.wd + + if upd.wd != wd { + delete(w.wd, wd) + } + } + + return nil +} + +func newBackend(ev chan Event, errs chan error) (backend, error) { + return newBufferedBackend(0, ev, errs) +} + +func newBufferedBackend(sz uint, ev chan Event, errs chan error) (backend, error) { + // Need to set nonblocking mode for SetDeadline to work, otherwise blocking + // I/O operations won't terminate on close. + fd, errno := unix.InotifyInit1(unix.IN_CLOEXEC | unix.IN_NONBLOCK) + if fd == -1 { + return nil, errno + } + + w := &inotify{ + Events: ev, + Errors: errs, + fd: fd, + inotifyFile: os.NewFile(uintptr(fd), ""), + watches: newWatches(), + done: make(chan struct{}), + doneResp: make(chan struct{}), + } + + go w.readEvents() + return w, nil +} + +// Returns true if the event was sent, or false if watcher is closed. +func (w *inotify) sendEvent(e Event) bool { + select { + case <-w.done: + return false + case w.Events <- e: + return true + } +} + +// Returns true if the error was sent, or false if watcher is closed. +func (w *inotify) sendError(err error) bool { + if err == nil { + return true + } + select { + case <-w.done: + return false + case w.Errors <- err: + return true + } +} + +func (w *inotify) isClosed() bool { + select { + case <-w.done: + return true + default: + return false + } +} + +func (w *inotify) Close() error { + w.doneMu.Lock() + if w.isClosed() { + w.doneMu.Unlock() + return nil + } + close(w.done) + w.doneMu.Unlock() + + // Causes any blocking reads to return with an error, provided the file + // still supports deadline operations. + err := w.inotifyFile.Close() + if err != nil { + return err + } + + // Wait for goroutine to close + <-w.doneResp + + return nil +} + +func (w *inotify) Add(name string) error { return w.AddWith(name) } + +func (w *inotify) AddWith(path string, opts ...addOpt) error { + if w.isClosed() { + return ErrClosed + } + if debug { + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s AddWith(%q)\n", + time.Now().Format("15:04:05.000000000"), path) + } + + with := getOptions(opts...) + if !w.xSupports(with.op) { + return fmt.Errorf("%w: %s", xErrUnsupported, with.op) + } + + path, recurse := recursivePath(path) + if recurse { + return filepath.WalkDir(path, func(root string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + if root == path { + return fmt.Errorf("fsnotify: not a directory: %q", path) + } + return nil + } + + // Send a Create event when adding new directory from a recursive + // watch; this is for "mkdir -p one/two/three". Usually all those + // directories will be created before we can set up watchers on the + // subdirectories, so only "one" would be sent as a Create event and + // not "one/two" and "one/two/three" (inotifywait -r has the same + // problem). + if with.sendCreate && root != path { + w.sendEvent(Event{Name: root, Op: Create}) + } + + return w.add(root, with, true) + }) + } + + return w.add(path, with, false) +} + +func (w *inotify) add(path string, with withOpts, recurse bool) error { + var flags uint32 + if with.noFollow { + flags |= unix.IN_DONT_FOLLOW + } + if with.op.Has(Create) { + flags |= unix.IN_CREATE + } + if with.op.Has(Write) { + flags |= unix.IN_MODIFY + } + if with.op.Has(Remove) { + flags |= unix.IN_DELETE | unix.IN_DELETE_SELF + } + if with.op.Has(Rename) { + flags |= unix.IN_MOVED_TO | unix.IN_MOVED_FROM | unix.IN_MOVE_SELF + } + if with.op.Has(Chmod) { + flags |= unix.IN_ATTRIB + } + if with.op.Has(xUnportableOpen) { + flags |= unix.IN_OPEN + } + if with.op.Has(xUnportableRead) { + flags |= unix.IN_ACCESS + } + if with.op.Has(xUnportableCloseWrite) { + flags |= unix.IN_CLOSE_WRITE + } + if with.op.Has(xUnportableCloseRead) { + flags |= unix.IN_CLOSE_NOWRITE + } + return w.register(path, flags, recurse) +} + +func (w *inotify) register(path string, flags uint32, recurse bool) error { + return w.watches.updatePath(path, func(existing *watch) (*watch, error) { + if existing != nil { + flags |= existing.flags | unix.IN_MASK_ADD + } + + wd, err := unix.InotifyAddWatch(w.fd, path, flags) + if wd == -1 { + return nil, err + } + + if existing == nil { + return &watch{ + wd: uint32(wd), + path: path, + flags: flags, + recurse: recurse, + }, nil + } + + existing.wd = uint32(wd) + existing.flags = flags + return existing, nil + }) +} + +func (w *inotify) Remove(name string) error { + if w.isClosed() { + return nil + } + if debug { + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s Remove(%q)\n", + time.Now().Format("15:04:05.000000000"), name) + } + return w.remove(filepath.Clean(name)) +} + +func (w *inotify) remove(name string) error { + wds, err := w.watches.removePath(name) + if err != nil { + return err + } + + for _, wd := range wds { + _, err := unix.InotifyRmWatch(w.fd, wd) + if err != nil { + // TODO: Perhaps it's not helpful to return an error here in every + // case; the only two possible errors are: + // + // EBADF, which happens when w.fd is not a valid file descriptor of + // any kind. + // + // EINVAL, which is when fd is not an inotify descriptor or wd is + // not a valid watch descriptor. Watch descriptors are invalidated + // when they are removed explicitly or implicitly; explicitly by + // inotify_rm_watch, implicitly when the file they are watching is + // deleted. + return err + } + } + return nil +} + +func (w *inotify) WatchList() []string { + if w.isClosed() { + return nil + } + + entries := make([]string, 0, w.watches.len()) + w.watches.mu.RLock() + for pathname := range w.watches.path { + entries = append(entries, pathname) + } + w.watches.mu.RUnlock() + + return entries +} + +// readEvents reads from the inotify file descriptor, converts the +// received events into Event objects and sends them via the Events channel +func (w *inotify) readEvents() { + defer func() { + close(w.doneResp) + close(w.Errors) + close(w.Events) + }() + + var ( + buf [unix.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events + errno error // Syscall errno + ) + for { + // See if we have been closed. + if w.isClosed() { + return + } + + n, err := w.inotifyFile.Read(buf[:]) + switch { + case errors.Unwrap(err) == os.ErrClosed: + return + case err != nil: + if !w.sendError(err) { + return + } + continue + } + + if n < unix.SizeofInotifyEvent { + var err error + if n == 0 { + err = io.EOF // If EOF is received. This should really never happen. + } else if n < 0 { + err = errno // If an error occurred while reading. + } else { + err = errors.New("notify: short read in readEvents()") // Read was too short. + } + if !w.sendError(err) { + return + } + continue + } + + // We don't know how many events we just read into the buffer + // While the offset points to at least one whole event... + var offset uint32 + for offset <= uint32(n-unix.SizeofInotifyEvent) { + var ( + // Point "raw" to the event in the buffer + raw = (*unix.InotifyEvent)(unsafe.Pointer(&buf[offset])) + mask = uint32(raw.Mask) + nameLen = uint32(raw.Len) + // Move to the next event in the buffer + next = func() { offset += unix.SizeofInotifyEvent + nameLen } + ) + + if mask&unix.IN_Q_OVERFLOW != 0 { + if !w.sendError(ErrEventOverflow) { + return + } + } + + /// If the event happened to the watched directory or the watched + /// file, the kernel doesn't append the filename to the event, but + /// we would like to always fill the the "Name" field with a valid + /// filename. We retrieve the path of the watch from the "paths" + /// map. + watch := w.watches.byWd(uint32(raw.Wd)) + /// Can be nil if Remove() was called in another goroutine for this + /// path inbetween reading the events from the kernel and reading + /// the internal state. Not much we can do about it, so just skip. + /// See #616. + if watch == nil { + next() + continue + } + + name := watch.path + if nameLen > 0 { + /// Point "bytes" at the first byte of the filename + bytes := (*[unix.PathMax]byte)(unsafe.Pointer(&buf[offset+unix.SizeofInotifyEvent]))[:nameLen:nameLen] + /// The filename is padded with NULL bytes. TrimRight() gets rid of those. + name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000") + } + + if debug { + internal.Debug(name, raw.Mask, raw.Cookie) + } + + if mask&unix.IN_IGNORED != 0 { //&& event.Op != 0 + next() + continue + } + + // inotify will automatically remove the watch on deletes; just need + // to clean our state here. + if mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF { + w.watches.remove(watch.wd) + } + + // We can't really update the state when a watched path is moved; + // only IN_MOVE_SELF is sent and not IN_MOVED_{FROM,TO}. So remove + // the watch. + if mask&unix.IN_MOVE_SELF == unix.IN_MOVE_SELF { + if watch.recurse { + next() // Do nothing + continue + } + + err := w.remove(watch.path) + if err != nil && !errors.Is(err, ErrNonExistentWatch) { + if !w.sendError(err) { + return + } + } + } + + /// Skip if we're watching both this path and the parent; the parent + /// will already send a delete so no need to do it twice. + if mask&unix.IN_DELETE_SELF != 0 { + if _, ok := w.watches.path[filepath.Dir(watch.path)]; ok { + next() + continue + } + } + + ev := w.newEvent(name, mask, raw.Cookie) + // Need to update watch path for recurse. + if watch.recurse { + isDir := mask&unix.IN_ISDIR == unix.IN_ISDIR + /// New directory created: set up watch on it. + if isDir && ev.Has(Create) { + err := w.register(ev.Name, watch.flags, true) + if !w.sendError(err) { + return + } + + // This was a directory rename, so we need to update all + // the children. + // + // TODO: this is of course pretty slow; we should use a + // better data structure for storing all of this, e.g. store + // children in the watch. I have some code for this in my + // kqueue refactor we can use in the future. For now I'm + // okay with this as it's not publicly available. + // Correctness first, performance second. + if ev.renamedFrom != "" { + w.watches.mu.Lock() + for k, ww := range w.watches.wd { + if k == watch.wd || ww.path == ev.Name { + continue + } + if strings.HasPrefix(ww.path, ev.renamedFrom) { + ww.path = strings.Replace(ww.path, ev.renamedFrom, ev.Name, 1) + w.watches.wd[k] = ww + } + } + w.watches.mu.Unlock() + } + } + } + + /// Send the events that are not ignored on the events channel + if !w.sendEvent(ev) { + return + } + next() + } + } +} + +func (w *inotify) isRecursive(path string) bool { + ww := w.watches.byPath(path) + if ww == nil { // path could be a file, so also check the Dir. + ww = w.watches.byPath(filepath.Dir(path)) + } + return ww != nil && ww.recurse +} + +func (w *inotify) newEvent(name string, mask, cookie uint32) Event { + e := Event{Name: name} + if mask&unix.IN_CREATE == unix.IN_CREATE || mask&unix.IN_MOVED_TO == unix.IN_MOVED_TO { + e.Op |= Create + } + if mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF || mask&unix.IN_DELETE == unix.IN_DELETE { + e.Op |= Remove + } + if mask&unix.IN_MODIFY == unix.IN_MODIFY { + e.Op |= Write + } + if mask&unix.IN_OPEN == unix.IN_OPEN { + e.Op |= xUnportableOpen + } + if mask&unix.IN_ACCESS == unix.IN_ACCESS { + e.Op |= xUnportableRead + } + if mask&unix.IN_CLOSE_WRITE == unix.IN_CLOSE_WRITE { + e.Op |= xUnportableCloseWrite + } + if mask&unix.IN_CLOSE_NOWRITE == unix.IN_CLOSE_NOWRITE { + e.Op |= xUnportableCloseRead + } + if mask&unix.IN_MOVE_SELF == unix.IN_MOVE_SELF || mask&unix.IN_MOVED_FROM == unix.IN_MOVED_FROM { + e.Op |= Rename + } + if mask&unix.IN_ATTRIB == unix.IN_ATTRIB { + e.Op |= Chmod + } + + if cookie != 0 { + if mask&unix.IN_MOVED_FROM == unix.IN_MOVED_FROM { + w.cookiesMu.Lock() + w.cookies[w.cookieIndex] = koekje{cookie: cookie, path: e.Name} + w.cookieIndex++ + if w.cookieIndex > 9 { + w.cookieIndex = 0 + } + w.cookiesMu.Unlock() + } else if mask&unix.IN_MOVED_TO == unix.IN_MOVED_TO { + w.cookiesMu.Lock() + var prev string + for _, c := range w.cookies { + if c.cookie == cookie { + prev = c.path + break + } + } + w.cookiesMu.Unlock() + e.renamedFrom = prev + } + } + return e +} + +func (w *inotify) xSupports(op Op) bool { + return true // Supports everything. +} + +func (w *inotify) state() { + w.watches.mu.Lock() + defer w.watches.mu.Unlock() + for wd, ww := range w.watches.wd { + fmt.Fprintf(os.Stderr, "%4d: recurse=%t %q\n", wd, ww.recurse, ww.path) + } +} diff --git a/vendor/github.com/fsnotify/fsnotify/backend_kqueue.go b/vendor/github.com/fsnotify/fsnotify/backend_kqueue.go new file mode 100644 index 000000000..d8de5ab76 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/backend_kqueue.go @@ -0,0 +1,733 @@ +//go:build freebsd || openbsd || netbsd || dragonfly || darwin + +package fsnotify + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "sync" + "time" + + "github.com/fsnotify/fsnotify/internal" + "golang.org/x/sys/unix" +) + +type kqueue struct { + Events chan Event + Errors chan error + + kq int // File descriptor (as returned by the kqueue() syscall). + closepipe [2]int // Pipe used for closing kq. + watches *watches + done chan struct{} + doneMu sync.Mutex +} + +type ( + watches struct { + mu sync.RWMutex + wd map[int]watch // wd → watch + path map[string]int // pathname → wd + byDir map[string]map[int]struct{} // dirname(path) → wd + seen map[string]struct{} // Keep track of if we know this file exists. + byUser map[string]struct{} // Watches added with Watcher.Add() + } + watch struct { + wd int + name string + linkName string // In case of links; name is the target, and this is the link. + isDir bool + dirFlags uint32 + } +) + +func newWatches() *watches { + return &watches{ + wd: make(map[int]watch), + path: make(map[string]int), + byDir: make(map[string]map[int]struct{}), + seen: make(map[string]struct{}), + byUser: make(map[string]struct{}), + } +} + +func (w *watches) listPaths(userOnly bool) []string { + w.mu.RLock() + defer w.mu.RUnlock() + + if userOnly { + l := make([]string, 0, len(w.byUser)) + for p := range w.byUser { + l = append(l, p) + } + return l + } + + l := make([]string, 0, len(w.path)) + for p := range w.path { + l = append(l, p) + } + return l +} + +func (w *watches) watchesInDir(path string) []string { + w.mu.RLock() + defer w.mu.RUnlock() + + l := make([]string, 0, 4) + for fd := range w.byDir[path] { + info := w.wd[fd] + if _, ok := w.byUser[info.name]; !ok { + l = append(l, info.name) + } + } + return l +} + +// Mark path as added by the user. +func (w *watches) addUserWatch(path string) { + w.mu.Lock() + defer w.mu.Unlock() + w.byUser[path] = struct{}{} +} + +func (w *watches) addLink(path string, fd int) { + w.mu.Lock() + defer w.mu.Unlock() + + w.path[path] = fd + w.seen[path] = struct{}{} +} + +func (w *watches) add(path, linkPath string, fd int, isDir bool) { + w.mu.Lock() + defer w.mu.Unlock() + + w.path[path] = fd + w.wd[fd] = watch{wd: fd, name: path, linkName: linkPath, isDir: isDir} + + parent := filepath.Dir(path) + byDir, ok := w.byDir[parent] + if !ok { + byDir = make(map[int]struct{}, 1) + w.byDir[parent] = byDir + } + byDir[fd] = struct{}{} +} + +func (w *watches) byWd(fd int) (watch, bool) { + w.mu.RLock() + defer w.mu.RUnlock() + info, ok := w.wd[fd] + return info, ok +} + +func (w *watches) byPath(path string) (watch, bool) { + w.mu.RLock() + defer w.mu.RUnlock() + info, ok := w.wd[w.path[path]] + return info, ok +} + +func (w *watches) updateDirFlags(path string, flags uint32) { + w.mu.Lock() + defer w.mu.Unlock() + + fd := w.path[path] + info := w.wd[fd] + info.dirFlags = flags + w.wd[fd] = info +} + +func (w *watches) remove(fd int, path string) bool { + w.mu.Lock() + defer w.mu.Unlock() + + isDir := w.wd[fd].isDir + delete(w.path, path) + delete(w.byUser, path) + + parent := filepath.Dir(path) + delete(w.byDir[parent], fd) + + if len(w.byDir[parent]) == 0 { + delete(w.byDir, parent) + } + + delete(w.wd, fd) + delete(w.seen, path) + return isDir +} + +func (w *watches) markSeen(path string, exists bool) { + w.mu.Lock() + defer w.mu.Unlock() + if exists { + w.seen[path] = struct{}{} + } else { + delete(w.seen, path) + } +} + +func (w *watches) seenBefore(path string) bool { + w.mu.RLock() + defer w.mu.RUnlock() + _, ok := w.seen[path] + return ok +} + +func newBackend(ev chan Event, errs chan error) (backend, error) { + return newBufferedBackend(0, ev, errs) +} + +func newBufferedBackend(sz uint, ev chan Event, errs chan error) (backend, error) { + kq, closepipe, err := newKqueue() + if err != nil { + return nil, err + } + + w := &kqueue{ + Events: ev, + Errors: errs, + kq: kq, + closepipe: closepipe, + done: make(chan struct{}), + watches: newWatches(), + } + + go w.readEvents() + return w, nil +} + +// newKqueue creates a new kernel event queue and returns a descriptor. +// +// This registers a new event on closepipe, which will trigger an event when +// it's closed. This way we can use kevent() without timeout/polling; without +// the closepipe, it would block forever and we wouldn't be able to stop it at +// all. +func newKqueue() (kq int, closepipe [2]int, err error) { + kq, err = unix.Kqueue() + if kq == -1 { + return kq, closepipe, err + } + + // Register the close pipe. + err = unix.Pipe(closepipe[:]) + if err != nil { + unix.Close(kq) + return kq, closepipe, err + } + unix.CloseOnExec(closepipe[0]) + unix.CloseOnExec(closepipe[1]) + + // Register changes to listen on the closepipe. + changes := make([]unix.Kevent_t, 1) + // SetKevent converts int to the platform-specific types. + unix.SetKevent(&changes[0], closepipe[0], unix.EVFILT_READ, + unix.EV_ADD|unix.EV_ENABLE|unix.EV_ONESHOT) + + ok, err := unix.Kevent(kq, changes, nil, nil) + if ok == -1 { + unix.Close(kq) + unix.Close(closepipe[0]) + unix.Close(closepipe[1]) + return kq, closepipe, err + } + return kq, closepipe, nil +} + +// Returns true if the event was sent, or false if watcher is closed. +func (w *kqueue) sendEvent(e Event) bool { + select { + case <-w.done: + return false + case w.Events <- e: + return true + } +} + +// Returns true if the error was sent, or false if watcher is closed. +func (w *kqueue) sendError(err error) bool { + if err == nil { + return true + } + select { + case <-w.done: + return false + case w.Errors <- err: + return true + } +} + +func (w *kqueue) isClosed() bool { + select { + case <-w.done: + return true + default: + return false + } +} + +func (w *kqueue) Close() error { + w.doneMu.Lock() + if w.isClosed() { + w.doneMu.Unlock() + return nil + } + close(w.done) + w.doneMu.Unlock() + + pathsToRemove := w.watches.listPaths(false) + for _, name := range pathsToRemove { + w.Remove(name) + } + + // Send "quit" message to the reader goroutine. + unix.Close(w.closepipe[1]) + return nil +} + +func (w *kqueue) Add(name string) error { return w.AddWith(name) } + +func (w *kqueue) AddWith(name string, opts ...addOpt) error { + if debug { + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s AddWith(%q)\n", + time.Now().Format("15:04:05.000000000"), name) + } + + with := getOptions(opts...) + if !w.xSupports(with.op) { + return fmt.Errorf("%w: %s", xErrUnsupported, with.op) + } + + _, err := w.addWatch(name, noteAllEvents) + if err != nil { + return err + } + w.watches.addUserWatch(name) + return nil +} + +func (w *kqueue) Remove(name string) error { + if debug { + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s Remove(%q)\n", + time.Now().Format("15:04:05.000000000"), name) + } + return w.remove(name, true) +} + +func (w *kqueue) remove(name string, unwatchFiles bool) error { + if w.isClosed() { + return nil + } + + name = filepath.Clean(name) + info, ok := w.watches.byPath(name) + if !ok { + return fmt.Errorf("%w: %s", ErrNonExistentWatch, name) + } + + err := w.register([]int{info.wd}, unix.EV_DELETE, 0) + if err != nil { + return err + } + + unix.Close(info.wd) + + isDir := w.watches.remove(info.wd, name) + + // Find all watched paths that are in this directory that are not external. + if unwatchFiles && isDir { + pathsToRemove := w.watches.watchesInDir(name) + for _, name := range pathsToRemove { + // Since these are internal, not much sense in propagating error to + // the user, as that will just confuse them with an error about a + // path they did not explicitly watch themselves. + w.Remove(name) + } + } + return nil +} + +func (w *kqueue) WatchList() []string { + if w.isClosed() { + return nil + } + return w.watches.listPaths(true) +} + +// Watch all events (except NOTE_EXTEND, NOTE_LINK, NOTE_REVOKE) +const noteAllEvents = unix.NOTE_DELETE | unix.NOTE_WRITE | unix.NOTE_ATTRIB | unix.NOTE_RENAME + +// addWatch adds name to the watched file set; the flags are interpreted as +// described in kevent(2). +// +// Returns the real path to the file which was added, with symlinks resolved. +func (w *kqueue) addWatch(name string, flags uint32) (string, error) { + if w.isClosed() { + return "", ErrClosed + } + + name = filepath.Clean(name) + + info, alreadyWatching := w.watches.byPath(name) + if !alreadyWatching { + fi, err := os.Lstat(name) + if err != nil { + return "", err + } + + // Don't watch sockets or named pipes. + if (fi.Mode()&os.ModeSocket == os.ModeSocket) || (fi.Mode()&os.ModeNamedPipe == os.ModeNamedPipe) { + return "", nil + } + + // Follow symlinks. + if fi.Mode()&os.ModeSymlink == os.ModeSymlink { + link, err := os.Readlink(name) + if err != nil { + // Return nil because Linux can add unresolvable symlinks to the + // watch list without problems, so maintain consistency with + // that. There will be no file events for broken symlinks. + // TODO: more specific check; returns os.PathError; ENOENT? + return "", nil + } + + _, alreadyWatching = w.watches.byPath(link) + if alreadyWatching { + // Add to watches so we don't get spurious Create events later + // on when we diff the directories. + w.watches.addLink(name, 0) + return link, nil + } + + info.linkName = name + name = link + fi, err = os.Lstat(name) + if err != nil { + return "", nil + } + } + + // Retry on EINTR; open() can return EINTR in practice on macOS. + // See #354, and Go issues 11180 and 39237. + for { + info.wd, err = unix.Open(name, openMode, 0) + if err == nil { + break + } + if errors.Is(err, unix.EINTR) { + continue + } + + return "", err + } + + info.isDir = fi.IsDir() + } + + err := w.register([]int{info.wd}, unix.EV_ADD|unix.EV_CLEAR|unix.EV_ENABLE, flags) + if err != nil { + unix.Close(info.wd) + return "", err + } + + if !alreadyWatching { + w.watches.add(name, info.linkName, info.wd, info.isDir) + } + + // Watch the directory if it has not been watched before, or if it was + // watched before, but perhaps only a NOTE_DELETE (watchDirectoryFiles) + if info.isDir { + watchDir := (flags&unix.NOTE_WRITE) == unix.NOTE_WRITE && + (!alreadyWatching || (info.dirFlags&unix.NOTE_WRITE) != unix.NOTE_WRITE) + w.watches.updateDirFlags(name, flags) + + if watchDir { + if err := w.watchDirectoryFiles(name); err != nil { + return "", err + } + } + } + return name, nil +} + +// readEvents reads from kqueue and converts the received kevents into +// Event values that it sends down the Events channel. +func (w *kqueue) readEvents() { + defer func() { + close(w.Events) + close(w.Errors) + _ = unix.Close(w.kq) + unix.Close(w.closepipe[0]) + }() + + eventBuffer := make([]unix.Kevent_t, 10) + for { + kevents, err := w.read(eventBuffer) + // EINTR is okay, the syscall was interrupted before timeout expired. + if err != nil && err != unix.EINTR { + if !w.sendError(fmt.Errorf("fsnotify.readEvents: %w", err)) { + return + } + } + + for _, kevent := range kevents { + var ( + wd = int(kevent.Ident) + mask = uint32(kevent.Fflags) + ) + + // Shut down the loop when the pipe is closed, but only after all + // other events have been processed. + if wd == w.closepipe[0] { + return + } + + path, ok := w.watches.byWd(wd) + if debug { + internal.Debug(path.name, &kevent) + } + + // On macOS it seems that sometimes an event with Ident=0 is + // delivered, and no other flags/information beyond that, even + // though we never saw such a file descriptor. For example in + // TestWatchSymlink/277 (usually at the end, but sometimes sooner): + // + // fmt.Printf("READ: %2d %#v\n", kevent.Ident, kevent) + // unix.Kevent_t{Ident:0x2a, Filter:-4, Flags:0x25, Fflags:0x2, Data:0, Udata:(*uint8)(nil)} + // unix.Kevent_t{Ident:0x0, Filter:-4, Flags:0x25, Fflags:0x2, Data:0, Udata:(*uint8)(nil)} + // + // The first is a normal event, the second with Ident 0. No error + // flag, no data, no ... nothing. + // + // I read a bit through bsd/kern_event.c from the xnu source, but I + // don't really see an obvious location where this is triggered – + // this doesn't seem intentional, but idk... + // + // Technically fd 0 is a valid descriptor, so only skip it if + // there's no path, and if we're on macOS. + if !ok && kevent.Ident == 0 && runtime.GOOS == "darwin" { + continue + } + + event := w.newEvent(path.name, path.linkName, mask) + + if event.Has(Rename) || event.Has(Remove) { + w.remove(event.Name, false) + w.watches.markSeen(event.Name, false) + } + + if path.isDir && event.Has(Write) && !event.Has(Remove) { + w.dirChange(event.Name) + } else if !w.sendEvent(event) { + return + } + + if event.Has(Remove) { + // Look for a file that may have overwritten this; for example, + // mv f1 f2 will delete f2, then create f2. + if path.isDir { + fileDir := filepath.Clean(event.Name) + _, found := w.watches.byPath(fileDir) + if found { + // TODO: this branch is never triggered in any test. + // Added in d6220df (2012). + // isDir check added in 8611c35 (2016): https://github.com/fsnotify/fsnotify/pull/111 + // + // I don't really get how this can be triggered either. + // And it wasn't triggered in the patch that added it, + // either. + // + // Original also had a comment: + // make sure the directory exists before we watch for + // changes. When we do a recursive watch and perform + // rm -rf, the parent directory might have gone + // missing, ignore the missing directory and let the + // upcoming delete event remove the watch from the + // parent directory. + err := w.dirChange(fileDir) + if !w.sendError(err) { + return + } + } + } else { + path := filepath.Clean(event.Name) + if fi, err := os.Lstat(path); err == nil { + err := w.sendCreateIfNew(path, fi) + if !w.sendError(err) { + return + } + } + } + } + } + } +} + +// newEvent returns an platform-independent Event based on kqueue Fflags. +func (w *kqueue) newEvent(name, linkName string, mask uint32) Event { + e := Event{Name: name} + if linkName != "" { + // If the user watched "/path/link" then emit events as "/path/link" + // rather than "/path/target". + e.Name = linkName + } + + if mask&unix.NOTE_DELETE == unix.NOTE_DELETE { + e.Op |= Remove + } + if mask&unix.NOTE_WRITE == unix.NOTE_WRITE { + e.Op |= Write + } + if mask&unix.NOTE_RENAME == unix.NOTE_RENAME { + e.Op |= Rename + } + if mask&unix.NOTE_ATTRIB == unix.NOTE_ATTRIB { + e.Op |= Chmod + } + // No point sending a write and delete event at the same time: if it's gone, + // then it's gone. + if e.Op.Has(Write) && e.Op.Has(Remove) { + e.Op &^= Write + } + return e +} + +// watchDirectoryFiles to mimic inotify when adding a watch on a directory +func (w *kqueue) watchDirectoryFiles(dirPath string) error { + files, err := os.ReadDir(dirPath) + if err != nil { + return err + } + + for _, f := range files { + path := filepath.Join(dirPath, f.Name()) + + fi, err := f.Info() + if err != nil { + return fmt.Errorf("%q: %w", path, err) + } + + cleanPath, err := w.internalWatch(path, fi) + if err != nil { + // No permission to read the file; that's not a problem: just skip. + // But do add it to w.fileExists to prevent it from being picked up + // as a "new" file later (it still shows up in the directory + // listing). + switch { + case errors.Is(err, unix.EACCES) || errors.Is(err, unix.EPERM): + cleanPath = filepath.Clean(path) + default: + return fmt.Errorf("%q: %w", path, err) + } + } + + w.watches.markSeen(cleanPath, true) + } + + return nil +} + +// Search the directory for new files and send an event for them. +// +// This functionality is to have the BSD watcher match the inotify, which sends +// a create event for files created in a watched directory. +func (w *kqueue) dirChange(dir string) error { + files, err := os.ReadDir(dir) + if err != nil { + // Directory no longer exists: we can ignore this safely. kqueue will + // still give us the correct events. + if errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("fsnotify.dirChange: %w", err) + } + + for _, f := range files { + fi, err := f.Info() + if err != nil { + return fmt.Errorf("fsnotify.dirChange: %w", err) + } + + err = w.sendCreateIfNew(filepath.Join(dir, fi.Name()), fi) + if err != nil { + // Don't need to send an error if this file isn't readable. + if errors.Is(err, unix.EACCES) || errors.Is(err, unix.EPERM) { + return nil + } + return fmt.Errorf("fsnotify.dirChange: %w", err) + } + } + return nil +} + +// Send a create event if the file isn't already being tracked, and start +// watching this file. +func (w *kqueue) sendCreateIfNew(path string, fi os.FileInfo) error { + if !w.watches.seenBefore(path) { + if !w.sendEvent(Event{Name: path, Op: Create}) { + return nil + } + } + + // Like watchDirectoryFiles, but without doing another ReadDir. + path, err := w.internalWatch(path, fi) + if err != nil { + return err + } + w.watches.markSeen(path, true) + return nil +} + +func (w *kqueue) internalWatch(name string, fi os.FileInfo) (string, error) { + if fi.IsDir() { + // mimic Linux providing delete events for subdirectories, but preserve + // the flags used if currently watching subdirectory + info, _ := w.watches.byPath(name) + return w.addWatch(name, info.dirFlags|unix.NOTE_DELETE|unix.NOTE_RENAME) + } + + // watch file to mimic Linux inotify + return w.addWatch(name, noteAllEvents) +} + +// Register events with the queue. +func (w *kqueue) register(fds []int, flags int, fflags uint32) error { + changes := make([]unix.Kevent_t, len(fds)) + for i, fd := range fds { + // SetKevent converts int to the platform-specific types. + unix.SetKevent(&changes[i], fd, unix.EVFILT_VNODE, flags) + changes[i].Fflags = fflags + } + + // Register the events. + success, err := unix.Kevent(w.kq, changes, nil, nil) + if success == -1 { + return err + } + return nil +} + +// read retrieves pending events, or waits until an event occurs. +func (w *kqueue) read(events []unix.Kevent_t) ([]unix.Kevent_t, error) { + n, err := unix.Kevent(w.kq, nil, events, nil) + if err != nil { + return nil, err + } + return events[0:n], nil +} + +func (w *kqueue) xSupports(op Op) bool { + if runtime.GOOS == "freebsd" { + //return true // Supports everything. + } + if op.Has(xUnportableOpen) || op.Has(xUnportableRead) || + op.Has(xUnportableCloseWrite) || op.Has(xUnportableCloseRead) { + return false + } + return true +} diff --git a/vendor/github.com/fsnotify/fsnotify/backend_other.go b/vendor/github.com/fsnotify/fsnotify/backend_other.go new file mode 100644 index 000000000..5eb5dbc66 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/backend_other.go @@ -0,0 +1,23 @@ +//go:build appengine || (!darwin && !dragonfly && !freebsd && !openbsd && !linux && !netbsd && !solaris && !windows) + +package fsnotify + +import "errors" + +type other struct { + Events chan Event + Errors chan error +} + +func newBackend(ev chan Event, errs chan error) (backend, error) { + return nil, errors.New("fsnotify not supported on the current platform") +} +func newBufferedBackend(sz uint, ev chan Event, errs chan error) (backend, error) { + return newBackend(ev, errs) +} +func (w *other) Close() error { return nil } +func (w *other) WatchList() []string { return nil } +func (w *other) Add(name string) error { return nil } +func (w *other) AddWith(name string, opts ...addOpt) error { return nil } +func (w *other) Remove(name string) error { return nil } +func (w *other) xSupports(op Op) bool { return false } diff --git a/vendor/github.com/fsnotify/fsnotify/backend_windows.go b/vendor/github.com/fsnotify/fsnotify/backend_windows.go new file mode 100644 index 000000000..c54a63083 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/backend_windows.go @@ -0,0 +1,682 @@ +//go:build windows + +// Windows backend based on ReadDirectoryChangesW() +// +// https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-readdirectorychangesw + +package fsnotify + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "sync" + "time" + "unsafe" + + "github.com/fsnotify/fsnotify/internal" + "golang.org/x/sys/windows" +) + +type readDirChangesW struct { + Events chan Event + Errors chan error + + port windows.Handle // Handle to completion port + input chan *input // Inputs to the reader are sent on this channel + quit chan chan<- error + + mu sync.Mutex // Protects access to watches, closed + watches watchMap // Map of watches (key: i-number) + closed bool // Set to true when Close() is first called +} + +func newBackend(ev chan Event, errs chan error) (backend, error) { + return newBufferedBackend(50, ev, errs) +} + +func newBufferedBackend(sz uint, ev chan Event, errs chan error) (backend, error) { + port, err := windows.CreateIoCompletionPort(windows.InvalidHandle, 0, 0, 0) + if err != nil { + return nil, os.NewSyscallError("CreateIoCompletionPort", err) + } + w := &readDirChangesW{ + Events: ev, + Errors: errs, + port: port, + watches: make(watchMap), + input: make(chan *input, 1), + quit: make(chan chan<- error, 1), + } + go w.readEvents() + return w, nil +} + +func (w *readDirChangesW) isClosed() bool { + w.mu.Lock() + defer w.mu.Unlock() + return w.closed +} + +func (w *readDirChangesW) sendEvent(name, renamedFrom string, mask uint64) bool { + if mask == 0 { + return false + } + + event := w.newEvent(name, uint32(mask)) + event.renamedFrom = renamedFrom + select { + case ch := <-w.quit: + w.quit <- ch + case w.Events <- event: + } + return true +} + +// Returns true if the error was sent, or false if watcher is closed. +func (w *readDirChangesW) sendError(err error) bool { + if err == nil { + return true + } + select { + case w.Errors <- err: + return true + case <-w.quit: + return false + } +} + +func (w *readDirChangesW) Close() error { + if w.isClosed() { + return nil + } + + w.mu.Lock() + w.closed = true + w.mu.Unlock() + + // Send "quit" message to the reader goroutine + ch := make(chan error) + w.quit <- ch + if err := w.wakeupReader(); err != nil { + return err + } + return <-ch +} + +func (w *readDirChangesW) Add(name string) error { return w.AddWith(name) } + +func (w *readDirChangesW) AddWith(name string, opts ...addOpt) error { + if w.isClosed() { + return ErrClosed + } + if debug { + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s AddWith(%q)\n", + time.Now().Format("15:04:05.000000000"), filepath.ToSlash(name)) + } + + with := getOptions(opts...) + if !w.xSupports(with.op) { + return fmt.Errorf("%w: %s", xErrUnsupported, with.op) + } + if with.bufsize < 4096 { + return fmt.Errorf("fsnotify.WithBufferSize: buffer size cannot be smaller than 4096 bytes") + } + + in := &input{ + op: opAddWatch, + path: filepath.Clean(name), + flags: sysFSALLEVENTS, + reply: make(chan error), + bufsize: with.bufsize, + } + w.input <- in + if err := w.wakeupReader(); err != nil { + return err + } + return <-in.reply +} + +func (w *readDirChangesW) Remove(name string) error { + if w.isClosed() { + return nil + } + if debug { + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s Remove(%q)\n", + time.Now().Format("15:04:05.000000000"), filepath.ToSlash(name)) + } + + in := &input{ + op: opRemoveWatch, + path: filepath.Clean(name), + reply: make(chan error), + } + w.input <- in + if err := w.wakeupReader(); err != nil { + return err + } + return <-in.reply +} + +func (w *readDirChangesW) WatchList() []string { + if w.isClosed() { + return nil + } + + w.mu.Lock() + defer w.mu.Unlock() + + entries := make([]string, 0, len(w.watches)) + for _, entry := range w.watches { + for _, watchEntry := range entry { + for name := range watchEntry.names { + entries = append(entries, filepath.Join(watchEntry.path, name)) + } + // the directory itself is being watched + if watchEntry.mask != 0 { + entries = append(entries, watchEntry.path) + } + } + } + + return entries +} + +// These options are from the old golang.org/x/exp/winfsnotify, where you could +// add various options to the watch. This has long since been removed. +// +// The "sys" in the name is misleading as they're not part of any "system". +// +// This should all be removed at some point, and just use windows.FILE_NOTIFY_* +const ( + sysFSALLEVENTS = 0xfff + sysFSCREATE = 0x100 + sysFSDELETE = 0x200 + sysFSDELETESELF = 0x400 + sysFSMODIFY = 0x2 + sysFSMOVE = 0xc0 + sysFSMOVEDFROM = 0x40 + sysFSMOVEDTO = 0x80 + sysFSMOVESELF = 0x800 + sysFSIGNORED = 0x8000 +) + +func (w *readDirChangesW) newEvent(name string, mask uint32) Event { + e := Event{Name: name} + if mask&sysFSCREATE == sysFSCREATE || mask&sysFSMOVEDTO == sysFSMOVEDTO { + e.Op |= Create + } + if mask&sysFSDELETE == sysFSDELETE || mask&sysFSDELETESELF == sysFSDELETESELF { + e.Op |= Remove + } + if mask&sysFSMODIFY == sysFSMODIFY { + e.Op |= Write + } + if mask&sysFSMOVE == sysFSMOVE || mask&sysFSMOVESELF == sysFSMOVESELF || mask&sysFSMOVEDFROM == sysFSMOVEDFROM { + e.Op |= Rename + } + return e +} + +const ( + opAddWatch = iota + opRemoveWatch +) + +const ( + provisional uint64 = 1 << (32 + iota) +) + +type input struct { + op int + path string + flags uint32 + bufsize int + reply chan error +} + +type inode struct { + handle windows.Handle + volume uint32 + index uint64 +} + +type watch struct { + ov windows.Overlapped + ino *inode // i-number + recurse bool // Recursive watch? + path string // Directory path + mask uint64 // Directory itself is being watched with these notify flags + names map[string]uint64 // Map of names being watched and their notify flags + rename string // Remembers the old name while renaming a file + buf []byte // buffer, allocated later +} + +type ( + indexMap map[uint64]*watch + watchMap map[uint32]indexMap +) + +func (w *readDirChangesW) wakeupReader() error { + err := windows.PostQueuedCompletionStatus(w.port, 0, 0, nil) + if err != nil { + return os.NewSyscallError("PostQueuedCompletionStatus", err) + } + return nil +} + +func (w *readDirChangesW) getDir(pathname string) (dir string, err error) { + attr, err := windows.GetFileAttributes(windows.StringToUTF16Ptr(pathname)) + if err != nil { + return "", os.NewSyscallError("GetFileAttributes", err) + } + if attr&windows.FILE_ATTRIBUTE_DIRECTORY != 0 { + dir = pathname + } else { + dir, _ = filepath.Split(pathname) + dir = filepath.Clean(dir) + } + return +} + +func (w *readDirChangesW) getIno(path string) (ino *inode, err error) { + h, err := windows.CreateFile(windows.StringToUTF16Ptr(path), + windows.FILE_LIST_DIRECTORY, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, + nil, windows.OPEN_EXISTING, + windows.FILE_FLAG_BACKUP_SEMANTICS|windows.FILE_FLAG_OVERLAPPED, 0) + if err != nil { + return nil, os.NewSyscallError("CreateFile", err) + } + + var fi windows.ByHandleFileInformation + err = windows.GetFileInformationByHandle(h, &fi) + if err != nil { + windows.CloseHandle(h) + return nil, os.NewSyscallError("GetFileInformationByHandle", err) + } + ino = &inode{ + handle: h, + volume: fi.VolumeSerialNumber, + index: uint64(fi.FileIndexHigh)<<32 | uint64(fi.FileIndexLow), + } + return ino, nil +} + +// Must run within the I/O thread. +func (m watchMap) get(ino *inode) *watch { + if i := m[ino.volume]; i != nil { + return i[ino.index] + } + return nil +} + +// Must run within the I/O thread. +func (m watchMap) set(ino *inode, watch *watch) { + i := m[ino.volume] + if i == nil { + i = make(indexMap) + m[ino.volume] = i + } + i[ino.index] = watch +} + +// Must run within the I/O thread. +func (w *readDirChangesW) addWatch(pathname string, flags uint64, bufsize int) error { + pathname, recurse := recursivePath(pathname) + + dir, err := w.getDir(pathname) + if err != nil { + return err + } + + ino, err := w.getIno(dir) + if err != nil { + return err + } + w.mu.Lock() + watchEntry := w.watches.get(ino) + w.mu.Unlock() + if watchEntry == nil { + _, err := windows.CreateIoCompletionPort(ino.handle, w.port, 0, 0) + if err != nil { + windows.CloseHandle(ino.handle) + return os.NewSyscallError("CreateIoCompletionPort", err) + } + watchEntry = &watch{ + ino: ino, + path: dir, + names: make(map[string]uint64), + recurse: recurse, + buf: make([]byte, bufsize), + } + w.mu.Lock() + w.watches.set(ino, watchEntry) + w.mu.Unlock() + flags |= provisional + } else { + windows.CloseHandle(ino.handle) + } + if pathname == dir { + watchEntry.mask |= flags + } else { + watchEntry.names[filepath.Base(pathname)] |= flags + } + + err = w.startRead(watchEntry) + if err != nil { + return err + } + + if pathname == dir { + watchEntry.mask &= ^provisional + } else { + watchEntry.names[filepath.Base(pathname)] &= ^provisional + } + return nil +} + +// Must run within the I/O thread. +func (w *readDirChangesW) remWatch(pathname string) error { + pathname, recurse := recursivePath(pathname) + + dir, err := w.getDir(pathname) + if err != nil { + return err + } + ino, err := w.getIno(dir) + if err != nil { + return err + } + + w.mu.Lock() + watch := w.watches.get(ino) + w.mu.Unlock() + + if recurse && !watch.recurse { + return fmt.Errorf("can't use \\... with non-recursive watch %q", pathname) + } + + err = windows.CloseHandle(ino.handle) + if err != nil { + w.sendError(os.NewSyscallError("CloseHandle", err)) + } + if watch == nil { + return fmt.Errorf("%w: %s", ErrNonExistentWatch, pathname) + } + if pathname == dir { + w.sendEvent(watch.path, "", watch.mask&sysFSIGNORED) + watch.mask = 0 + } else { + name := filepath.Base(pathname) + w.sendEvent(filepath.Join(watch.path, name), "", watch.names[name]&sysFSIGNORED) + delete(watch.names, name) + } + + return w.startRead(watch) +} + +// Must run within the I/O thread. +func (w *readDirChangesW) deleteWatch(watch *watch) { + for name, mask := range watch.names { + if mask&provisional == 0 { + w.sendEvent(filepath.Join(watch.path, name), "", mask&sysFSIGNORED) + } + delete(watch.names, name) + } + if watch.mask != 0 { + if watch.mask&provisional == 0 { + w.sendEvent(watch.path, "", watch.mask&sysFSIGNORED) + } + watch.mask = 0 + } +} + +// Must run within the I/O thread. +func (w *readDirChangesW) startRead(watch *watch) error { + err := windows.CancelIo(watch.ino.handle) + if err != nil { + w.sendError(os.NewSyscallError("CancelIo", err)) + w.deleteWatch(watch) + } + mask := w.toWindowsFlags(watch.mask) + for _, m := range watch.names { + mask |= w.toWindowsFlags(m) + } + if mask == 0 { + err := windows.CloseHandle(watch.ino.handle) + if err != nil { + w.sendError(os.NewSyscallError("CloseHandle", err)) + } + w.mu.Lock() + delete(w.watches[watch.ino.volume], watch.ino.index) + w.mu.Unlock() + return nil + } + + // We need to pass the array, rather than the slice. + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&watch.buf)) + rdErr := windows.ReadDirectoryChanges(watch.ino.handle, + (*byte)(unsafe.Pointer(hdr.Data)), uint32(hdr.Len), + watch.recurse, mask, nil, &watch.ov, 0) + if rdErr != nil { + err := os.NewSyscallError("ReadDirectoryChanges", rdErr) + if rdErr == windows.ERROR_ACCESS_DENIED && watch.mask&provisional == 0 { + // Watched directory was probably removed + w.sendEvent(watch.path, "", watch.mask&sysFSDELETESELF) + err = nil + } + w.deleteWatch(watch) + w.startRead(watch) + return err + } + return nil +} + +// readEvents reads from the I/O completion port, converts the +// received events into Event objects and sends them via the Events channel. +// Entry point to the I/O thread. +func (w *readDirChangesW) readEvents() { + var ( + n uint32 + key uintptr + ov *windows.Overlapped + ) + runtime.LockOSThread() + + for { + // This error is handled after the watch == nil check below. + qErr := windows.GetQueuedCompletionStatus(w.port, &n, &key, &ov, windows.INFINITE) + + watch := (*watch)(unsafe.Pointer(ov)) + if watch == nil { + select { + case ch := <-w.quit: + w.mu.Lock() + var indexes []indexMap + for _, index := range w.watches { + indexes = append(indexes, index) + } + w.mu.Unlock() + for _, index := range indexes { + for _, watch := range index { + w.deleteWatch(watch) + w.startRead(watch) + } + } + + err := windows.CloseHandle(w.port) + if err != nil { + err = os.NewSyscallError("CloseHandle", err) + } + close(w.Events) + close(w.Errors) + ch <- err + return + case in := <-w.input: + switch in.op { + case opAddWatch: + in.reply <- w.addWatch(in.path, uint64(in.flags), in.bufsize) + case opRemoveWatch: + in.reply <- w.remWatch(in.path) + } + default: + } + continue + } + + switch qErr { + case nil: + // No error + case windows.ERROR_MORE_DATA: + if watch == nil { + w.sendError(errors.New("ERROR_MORE_DATA has unexpectedly null lpOverlapped buffer")) + } else { + // The i/o succeeded but the buffer is full. + // In theory we should be building up a full packet. + // In practice we can get away with just carrying on. + n = uint32(unsafe.Sizeof(watch.buf)) + } + case windows.ERROR_ACCESS_DENIED: + // Watched directory was probably removed + w.sendEvent(watch.path, "", watch.mask&sysFSDELETESELF) + w.deleteWatch(watch) + w.startRead(watch) + continue + case windows.ERROR_OPERATION_ABORTED: + // CancelIo was called on this handle + continue + default: + w.sendError(os.NewSyscallError("GetQueuedCompletionPort", qErr)) + continue + } + + var offset uint32 + for { + if n == 0 { + w.sendError(ErrEventOverflow) + break + } + + // Point "raw" to the event in the buffer + raw := (*windows.FileNotifyInformation)(unsafe.Pointer(&watch.buf[offset])) + + // Create a buf that is the size of the path name + size := int(raw.FileNameLength / 2) + var buf []uint16 + // TODO: Use unsafe.Slice in Go 1.17; https://stackoverflow.com/questions/51187973 + sh := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + sh.Data = uintptr(unsafe.Pointer(&raw.FileName)) + sh.Len = size + sh.Cap = size + name := windows.UTF16ToString(buf) + fullname := filepath.Join(watch.path, name) + + if debug { + internal.Debug(fullname, raw.Action) + } + + var mask uint64 + switch raw.Action { + case windows.FILE_ACTION_REMOVED: + mask = sysFSDELETESELF + case windows.FILE_ACTION_MODIFIED: + mask = sysFSMODIFY + case windows.FILE_ACTION_RENAMED_OLD_NAME: + watch.rename = name + case windows.FILE_ACTION_RENAMED_NEW_NAME: + // Update saved path of all sub-watches. + old := filepath.Join(watch.path, watch.rename) + w.mu.Lock() + for _, watchMap := range w.watches { + for _, ww := range watchMap { + if strings.HasPrefix(ww.path, old) { + ww.path = filepath.Join(fullname, strings.TrimPrefix(ww.path, old)) + } + } + } + w.mu.Unlock() + + if watch.names[watch.rename] != 0 { + watch.names[name] |= watch.names[watch.rename] + delete(watch.names, watch.rename) + mask = sysFSMOVESELF + } + } + + if raw.Action != windows.FILE_ACTION_RENAMED_NEW_NAME { + w.sendEvent(fullname, "", watch.names[name]&mask) + } + if raw.Action == windows.FILE_ACTION_REMOVED { + w.sendEvent(fullname, "", watch.names[name]&sysFSIGNORED) + delete(watch.names, name) + } + + if watch.rename != "" && raw.Action == windows.FILE_ACTION_RENAMED_NEW_NAME { + w.sendEvent(fullname, filepath.Join(watch.path, watch.rename), watch.mask&w.toFSnotifyFlags(raw.Action)) + } else { + w.sendEvent(fullname, "", watch.mask&w.toFSnotifyFlags(raw.Action)) + } + + if raw.Action == windows.FILE_ACTION_RENAMED_NEW_NAME { + w.sendEvent(filepath.Join(watch.path, watch.rename), "", watch.names[name]&mask) + } + + // Move to the next event in the buffer + if raw.NextEntryOffset == 0 { + break + } + offset += raw.NextEntryOffset + + // Error! + if offset >= n { + //lint:ignore ST1005 Windows should be capitalized + w.sendError(errors.New("Windows system assumed buffer larger than it is, events have likely been missed")) + break + } + } + + if err := w.startRead(watch); err != nil { + w.sendError(err) + } + } +} + +func (w *readDirChangesW) toWindowsFlags(mask uint64) uint32 { + var m uint32 + if mask&sysFSMODIFY != 0 { + m |= windows.FILE_NOTIFY_CHANGE_LAST_WRITE + } + if mask&(sysFSMOVE|sysFSCREATE|sysFSDELETE) != 0 { + m |= windows.FILE_NOTIFY_CHANGE_FILE_NAME | windows.FILE_NOTIFY_CHANGE_DIR_NAME + } + return m +} + +func (w *readDirChangesW) toFSnotifyFlags(action uint32) uint64 { + switch action { + case windows.FILE_ACTION_ADDED: + return sysFSCREATE + case windows.FILE_ACTION_REMOVED: + return sysFSDELETE + case windows.FILE_ACTION_MODIFIED: + return sysFSMODIFY + case windows.FILE_ACTION_RENAMED_OLD_NAME: + return sysFSMOVEDFROM + case windows.FILE_ACTION_RENAMED_NEW_NAME: + return sysFSMOVEDTO + } + return 0 +} + +func (w *readDirChangesW) xSupports(op Op) bool { + if op.Has(xUnportableOpen) || op.Has(xUnportableRead) || + op.Has(xUnportableCloseWrite) || op.Has(xUnportableCloseRead) { + return false + } + return true +} diff --git a/vendor/github.com/fsnotify/fsnotify/fsnotify.go b/vendor/github.com/fsnotify/fsnotify/fsnotify.go index 0f4ee52e8..0760efe91 100644 --- a/vendor/github.com/fsnotify/fsnotify/fsnotify.go +++ b/vendor/github.com/fsnotify/fsnotify/fsnotify.go @@ -1,69 +1,494 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !plan9 -// +build !plan9 - -// Package fsnotify provides a platform-independent interface for file system notifications. +// Package fsnotify provides a cross-platform interface for file system +// notifications. +// +// Currently supported systems: +// +// - Linux via inotify +// - BSD, macOS via kqueue +// - Windows via ReadDirectoryChangesW +// - illumos via FEN +// +// # FSNOTIFY_DEBUG +// +// Set the FSNOTIFY_DEBUG environment variable to "1" to print debug messages to +// stderr. This can be useful to track down some problems, especially in cases +// where fsnotify is used as an indirect dependency. +// +// Every event will be printed as soon as there's something useful to print, +// with as little processing from fsnotify. +// +// Example output: +// +// FSNOTIFY_DEBUG: 11:34:23.633087586 256:IN_CREATE → "/tmp/file-1" +// FSNOTIFY_DEBUG: 11:34:23.633202319 4:IN_ATTRIB → "/tmp/file-1" +// FSNOTIFY_DEBUG: 11:34:28.989728764 512:IN_DELETE → "/tmp/file-1" package fsnotify import ( - "bytes" "errors" "fmt" + "os" + "path/filepath" + "strings" ) -// Event represents a single file system notification. +// Watcher watches a set of paths, delivering events on a channel. +// +// A watcher should not be copied (e.g. pass it by pointer, rather than by +// value). +// +// # Linux notes +// +// When a file is removed a Remove event won't be emitted until all file +// descriptors are closed, and deletes will always emit a Chmod. For example: +// +// fp := os.Open("file") +// os.Remove("file") // Triggers Chmod +// fp.Close() // Triggers Remove +// +// This is the event that inotify sends, so not much can be changed about this. +// +// The fs.inotify.max_user_watches sysctl variable specifies the upper limit +// for the number of watches per user, and fs.inotify.max_user_instances +// specifies the maximum number of inotify instances per user. Every Watcher you +// create is an "instance", and every path you add is a "watch". +// +// These are also exposed in /proc as /proc/sys/fs/inotify/max_user_watches and +// /proc/sys/fs/inotify/max_user_instances +// +// To increase them you can use sysctl or write the value to the /proc file: +// +// # Default values on Linux 5.18 +// sysctl fs.inotify.max_user_watches=124983 +// sysctl fs.inotify.max_user_instances=128 +// +// To make the changes persist on reboot edit /etc/sysctl.conf or +// /usr/lib/sysctl.d/50-default.conf (details differ per Linux distro; check +// your distro's documentation): +// +// fs.inotify.max_user_watches=124983 +// fs.inotify.max_user_instances=128 +// +// Reaching the limit will result in a "no space left on device" or "too many open +// files" error. +// +// # kqueue notes (macOS, BSD) +// +// kqueue requires opening a file descriptor for every file that's being watched; +// so if you're watching a directory with five files then that's six file +// descriptors. You will run in to your system's "max open files" limit faster on +// these platforms. +// +// The sysctl variables kern.maxfiles and kern.maxfilesperproc can be used to +// control the maximum number of open files, as well as /etc/login.conf on BSD +// systems. +// +// # Windows notes +// +// Paths can be added as "C:\\path\\to\\dir", but forward slashes +// ("C:/path/to/dir") will also work. +// +// When a watched directory is removed it will always send an event for the +// directory itself, but may not send events for all files in that directory. +// Sometimes it will send events for all files, sometimes it will send no +// events, and often only for some files. +// +// The default ReadDirectoryChangesW() buffer size is 64K, which is the largest +// value that is guaranteed to work with SMB filesystems. If you have many +// events in quick succession this may not be enough, and you will have to use +// [WithBufferSize] to increase the value. +type Watcher struct { + b backend + + // Events sends the filesystem change events. + // + // fsnotify can send the following events; a "path" here can refer to a + // file, directory, symbolic link, or special file like a FIFO. + // + // fsnotify.Create A new path was created; this may be followed by one + // or more Write events if data also gets written to a + // file. + // + // fsnotify.Remove A path was removed. + // + // fsnotify.Rename A path was renamed. A rename is always sent with the + // old path as Event.Name, and a Create event will be + // sent with the new name. Renames are only sent for + // paths that are currently watched; e.g. moving an + // unmonitored file into a monitored directory will + // show up as just a Create. Similarly, renaming a file + // to outside a monitored directory will show up as + // only a Rename. + // + // fsnotify.Write A file or named pipe was written to. A Truncate will + // also trigger a Write. A single "write action" + // initiated by the user may show up as one or multiple + // writes, depending on when the system syncs things to + // disk. For example when compiling a large Go program + // you may get hundreds of Write events, and you may + // want to wait until you've stopped receiving them + // (see the dedup example in cmd/fsnotify). + // + // Some systems may send Write event for directories + // when the directory content changes. + // + // fsnotify.Chmod Attributes were changed. On Linux this is also sent + // when a file is removed (or more accurately, when a + // link to an inode is removed). On kqueue it's sent + // when a file is truncated. On Windows it's never + // sent. + Events chan Event + + // Errors sends any errors. + Errors chan error +} + +// Event represents a file system notification. type Event struct { - Name string // Relative path to the file or directory. - Op Op // File operation that triggered the event. + // Path to the file or directory. + // + // Paths are relative to the input; for example with Add("dir") the Name + // will be set to "dir/file" if you create that file, but if you use + // Add("/path/to/dir") it will be "/path/to/dir/file". + Name string + + // File operation that triggered the event. + // + // This is a bitmask and some systems may send multiple operations at once. + // Use the Event.Has() method instead of comparing with ==. + Op Op + + // Create events will have this set to the old path if it's a rename. This + // only works when both the source and destination are watched. It's not + // reliable when watching individual files, only directories. + // + // For example "mv /tmp/file /tmp/rename" will emit: + // + // Event{Op: Rename, Name: "/tmp/file"} + // Event{Op: Create, Name: "/tmp/rename", RenamedFrom: "/tmp/file"} + renamedFrom string } // Op describes a set of file operations. type Op uint32 -// These are the generalized file operations that can trigger a notification. +// The operations fsnotify can trigger; see the documentation on [Watcher] for a +// full description, and check them with [Event.Has]. const ( + // A new pathname was created. Create Op = 1 << iota + + // The pathname was written to; this does *not* mean the write has finished, + // and a write can be followed by more writes. Write + + // The path was removed; any watches on it will be removed. Some "remove" + // operations may trigger a Rename if the file is actually moved (for + // example "remove to trash" is often a rename). Remove + + // The path was renamed to something else; any watches on it will be + // removed. Rename + + // File attributes were changed. + // + // It's generally not recommended to take action on this event, as it may + // get triggered very frequently by some software. For example, Spotlight + // indexing on macOS, anti-virus software, backup software, etc. Chmod + + // File descriptor was opened. + // + // Only works on Linux and FreeBSD. + xUnportableOpen + + // File was read from. + // + // Only works on Linux and FreeBSD. + xUnportableRead + + // File opened for writing was closed. + // + // Only works on Linux and FreeBSD. + // + // The advantage of using this over Write is that it's more reliable than + // waiting for Write events to stop. It's also faster (if you're not + // listening to Write events): copying a file of a few GB can easily + // generate tens of thousands of Write events in a short span of time. + xUnportableCloseWrite + + // File opened for reading was closed. + // + // Only works on Linux and FreeBSD. + xUnportableCloseRead ) -func (op Op) String() string { - // Use a buffer for efficient string concatenation - var buffer bytes.Buffer +var ( + // ErrNonExistentWatch is used when Remove() is called on a path that's not + // added. + ErrNonExistentWatch = errors.New("fsnotify: can't remove non-existent watch") + + // ErrClosed is used when trying to operate on a closed Watcher. + ErrClosed = errors.New("fsnotify: watcher already closed") + + // ErrEventOverflow is reported from the Errors channel when there are too + // many events: + // + // - inotify: inotify returns IN_Q_OVERFLOW – because there are too + // many queued events (the fs.inotify.max_queued_events + // sysctl can be used to increase this). + // - windows: The buffer size is too small; WithBufferSize() can be used to increase it. + // - kqueue, fen: Not used. + ErrEventOverflow = errors.New("fsnotify: queue or buffer overflow") + + // ErrUnsupported is returned by AddWith() when WithOps() specified an + // Unportable event that's not supported on this platform. + xErrUnsupported = errors.New("fsnotify: not supported with this backend") +) + +// NewWatcher creates a new Watcher. +func NewWatcher() (*Watcher, error) { + ev, errs := make(chan Event), make(chan error) + b, err := newBackend(ev, errs) + if err != nil { + return nil, err + } + return &Watcher{b: b, Events: ev, Errors: errs}, nil +} + +// NewBufferedWatcher creates a new Watcher with a buffered Watcher.Events +// channel. +// +// The main use case for this is situations with a very large number of events +// where the kernel buffer size can't be increased (e.g. due to lack of +// permissions). An unbuffered Watcher will perform better for almost all use +// cases, and whenever possible you will be better off increasing the kernel +// buffers instead of adding a large userspace buffer. +func NewBufferedWatcher(sz uint) (*Watcher, error) { + ev, errs := make(chan Event), make(chan error) + b, err := newBufferedBackend(sz, ev, errs) + if err != nil { + return nil, err + } + return &Watcher{b: b, Events: ev, Errors: errs}, nil +} + +// Add starts monitoring the path for changes. +// +// A path can only be watched once; watching it more than once is a no-op and will +// not return an error. Paths that do not yet exist on the filesystem cannot be +// watched. +// +// A watch will be automatically removed if the watched path is deleted or +// renamed. The exception is the Windows backend, which doesn't remove the +// watcher on renames. +// +// Notifications on network filesystems (NFS, SMB, FUSE, etc.) or special +// filesystems (/proc, /sys, etc.) generally don't work. +// +// Returns [ErrClosed] if [Watcher.Close] was called. +// +// See [Watcher.AddWith] for a version that allows adding options. +// +// # Watching directories +// +// All files in a directory are monitored, including new files that are created +// after the watcher is started. Subdirectories are not watched (i.e. it's +// non-recursive). +// +// # Watching files +// +// Watching individual files (rather than directories) is generally not +// recommended as many programs (especially editors) update files atomically: it +// will write to a temporary file which is then moved to destination, +// overwriting the original (or some variant thereof). The watcher on the +// original file is now lost, as that no longer exists. +// +// The upshot of this is that a power failure or crash won't leave a +// half-written file. +// +// Watch the parent directory and use Event.Name to filter out files you're not +// interested in. There is an example of this in cmd/fsnotify/file.go. +func (w *Watcher) Add(path string) error { return w.b.Add(path) } + +// AddWith is like [Watcher.Add], but allows adding options. When using Add() +// the defaults described below are used. +// +// Possible options are: +// +// - [WithBufferSize] sets the buffer size for the Windows backend; no-op on +// other platforms. The default is 64K (65536 bytes). +func (w *Watcher) AddWith(path string, opts ...addOpt) error { return w.b.AddWith(path, opts...) } + +// Remove stops monitoring the path for changes. +// +// Directories are always removed non-recursively. For example, if you added +// /tmp/dir and /tmp/dir/subdir then you will need to remove both. +// +// Removing a path that has not yet been added returns [ErrNonExistentWatch]. +// +// Returns nil if [Watcher.Close] was called. +func (w *Watcher) Remove(path string) error { return w.b.Remove(path) } + +// Close removes all watches and closes the Events channel. +func (w *Watcher) Close() error { return w.b.Close() } - if op&Create == Create { - buffer.WriteString("|CREATE") +// WatchList returns all paths explicitly added with [Watcher.Add] (and are not +// yet removed). +// +// Returns nil if [Watcher.Close] was called. +func (w *Watcher) WatchList() []string { return w.b.WatchList() } + +// Supports reports if all the listed operations are supported by this platform. +// +// Create, Write, Remove, Rename, and Chmod are always supported. It can only +// return false for an Op starting with Unportable. +func (w *Watcher) xSupports(op Op) bool { return w.b.xSupports(op) } + +func (o Op) String() string { + var b strings.Builder + if o.Has(Create) { + b.WriteString("|CREATE") + } + if o.Has(Remove) { + b.WriteString("|REMOVE") + } + if o.Has(Write) { + b.WriteString("|WRITE") } - if op&Remove == Remove { - buffer.WriteString("|REMOVE") + if o.Has(xUnportableOpen) { + b.WriteString("|OPEN") } - if op&Write == Write { - buffer.WriteString("|WRITE") + if o.Has(xUnportableRead) { + b.WriteString("|READ") } - if op&Rename == Rename { - buffer.WriteString("|RENAME") + if o.Has(xUnportableCloseWrite) { + b.WriteString("|CLOSE_WRITE") } - if op&Chmod == Chmod { - buffer.WriteString("|CHMOD") + if o.Has(xUnportableCloseRead) { + b.WriteString("|CLOSE_READ") } - if buffer.Len() == 0 { - return "" + if o.Has(Rename) { + b.WriteString("|RENAME") } - return buffer.String()[1:] // Strip leading pipe + if o.Has(Chmod) { + b.WriteString("|CHMOD") + } + if b.Len() == 0 { + return "[no events]" + } + return b.String()[1:] } -// String returns a string representation of the event in the form -// "file: REMOVE|WRITE|..." +// Has reports if this operation has the given operation. +func (o Op) Has(h Op) bool { return o&h != 0 } + +// Has reports if this event has the given operation. +func (e Event) Has(op Op) bool { return e.Op.Has(op) } + +// String returns a string representation of the event with their path. func (e Event) String() string { - return fmt.Sprintf("%q: %s", e.Name, e.Op.String()) + if e.renamedFrom != "" { + return fmt.Sprintf("%-13s %q ← %q", e.Op.String(), e.Name, e.renamedFrom) + } + return fmt.Sprintf("%-13s %q", e.Op.String(), e.Name) } -// Common errors that can be reported by a watcher -var ( - ErrEventOverflow = errors.New("fsnotify queue overflow") +type ( + backend interface { + Add(string) error + AddWith(string, ...addOpt) error + Remove(string) error + WatchList() []string + Close() error + xSupports(Op) bool + } + addOpt func(opt *withOpts) + withOpts struct { + bufsize int + op Op + noFollow bool + sendCreate bool + } ) + +var debug = func() bool { + // Check for exactly "1" (rather than mere existence) so we can add + // options/flags in the future. I don't know if we ever want that, but it's + // nice to leave the option open. + return os.Getenv("FSNOTIFY_DEBUG") == "1" +}() + +var defaultOpts = withOpts{ + bufsize: 65536, // 64K + op: Create | Write | Remove | Rename | Chmod, +} + +func getOptions(opts ...addOpt) withOpts { + with := defaultOpts + for _, o := range opts { + if o != nil { + o(&with) + } + } + return with +} + +// WithBufferSize sets the [ReadDirectoryChangesW] buffer size. +// +// This only has effect on Windows systems, and is a no-op for other backends. +// +// The default value is 64K (65536 bytes) which is the highest value that works +// on all filesystems and should be enough for most applications, but if you +// have a large burst of events it may not be enough. You can increase it if +// you're hitting "queue or buffer overflow" errors ([ErrEventOverflow]). +// +// [ReadDirectoryChangesW]: https://learn.microsoft.com/en-gb/windows/win32/api/winbase/nf-winbase-readdirectorychangesw +func WithBufferSize(bytes int) addOpt { + return func(opt *withOpts) { opt.bufsize = bytes } +} + +// WithOps sets which operations to listen for. The default is [Create], +// [Write], [Remove], [Rename], and [Chmod]. +// +// Excluding operations you're not interested in can save quite a bit of CPU +// time; in some use cases there may be hundreds of thousands of useless Write +// or Chmod operations per second. +// +// This can also be used to add unportable operations not supported by all +// platforms; unportable operations all start with "Unportable": +// [UnportableOpen], [UnportableRead], [UnportableCloseWrite], and +// [UnportableCloseRead]. +// +// AddWith returns an error when using an unportable operation that's not +// supported. Use [Watcher.Support] to check for support. +func withOps(op Op) addOpt { + return func(opt *withOpts) { opt.op = op } +} + +// WithNoFollow disables following symlinks, so the symlinks themselves are +// watched. +func withNoFollow() addOpt { + return func(opt *withOpts) { opt.noFollow = true } +} + +// "Internal" option for recursive watches on inotify. +func withCreate() addOpt { + return func(opt *withOpts) { opt.sendCreate = true } +} + +var enableRecurse = false + +// Check if this path is recursive (ends with "/..." or "\..."), and return the +// path with the /... stripped. +func recursivePath(path string) (string, bool) { + path = filepath.Clean(path) + if !enableRecurse { // Only enabled in tests for now. + return path, false + } + if filepath.Base(path) == "..." { + return filepath.Dir(path), true + } + return path, false +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/darwin.go b/vendor/github.com/fsnotify/fsnotify/internal/darwin.go new file mode 100644 index 000000000..b0eab1009 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/darwin.go @@ -0,0 +1,39 @@ +//go:build darwin + +package internal + +import ( + "syscall" + + "golang.org/x/sys/unix" +) + +var ( + SyscallEACCES = syscall.EACCES + UnixEACCES = unix.EACCES +) + +var maxfiles uint64 + +// Go 1.19 will do this automatically: https://go-review.googlesource.com/c/go/+/393354/ +func SetRlimit() { + var l syscall.Rlimit + err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &l) + if err == nil && l.Cur != l.Max { + l.Cur = l.Max + syscall.Setrlimit(syscall.RLIMIT_NOFILE, &l) + } + maxfiles = l.Cur + + if n, err := syscall.SysctlUint32("kern.maxfiles"); err == nil && uint64(n) < maxfiles { + maxfiles = uint64(n) + } + + if n, err := syscall.SysctlUint32("kern.maxfilesperproc"); err == nil && uint64(n) < maxfiles { + maxfiles = uint64(n) + } +} + +func Maxfiles() uint64 { return maxfiles } +func Mkfifo(path string, mode uint32) error { return unix.Mkfifo(path, mode) } +func Mknod(path string, mode uint32, dev int) error { return unix.Mknod(path, mode, dev) } diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go new file mode 100644 index 000000000..928319fb0 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go @@ -0,0 +1,57 @@ +package internal + +import "golang.org/x/sys/unix" + +var names = []struct { + n string + m uint32 +}{ + {"NOTE_ABSOLUTE", unix.NOTE_ABSOLUTE}, + {"NOTE_ATTRIB", unix.NOTE_ATTRIB}, + {"NOTE_BACKGROUND", unix.NOTE_BACKGROUND}, + {"NOTE_CHILD", unix.NOTE_CHILD}, + {"NOTE_CRITICAL", unix.NOTE_CRITICAL}, + {"NOTE_DELETE", unix.NOTE_DELETE}, + {"NOTE_EXEC", unix.NOTE_EXEC}, + {"NOTE_EXIT", unix.NOTE_EXIT}, + {"NOTE_EXITSTATUS", unix.NOTE_EXITSTATUS}, + {"NOTE_EXIT_CSERROR", unix.NOTE_EXIT_CSERROR}, + {"NOTE_EXIT_DECRYPTFAIL", unix.NOTE_EXIT_DECRYPTFAIL}, + {"NOTE_EXIT_DETAIL", unix.NOTE_EXIT_DETAIL}, + {"NOTE_EXIT_DETAIL_MASK", unix.NOTE_EXIT_DETAIL_MASK}, + {"NOTE_EXIT_MEMORY", unix.NOTE_EXIT_MEMORY}, + {"NOTE_EXIT_REPARENTED", unix.NOTE_EXIT_REPARENTED}, + {"NOTE_EXTEND", unix.NOTE_EXTEND}, + {"NOTE_FFAND", unix.NOTE_FFAND}, + {"NOTE_FFCOPY", unix.NOTE_FFCOPY}, + {"NOTE_FFCTRLMASK", unix.NOTE_FFCTRLMASK}, + {"NOTE_FFLAGSMASK", unix.NOTE_FFLAGSMASK}, + {"NOTE_FFNOP", unix.NOTE_FFNOP}, + {"NOTE_FFOR", unix.NOTE_FFOR}, + {"NOTE_FORK", unix.NOTE_FORK}, + {"NOTE_FUNLOCK", unix.NOTE_FUNLOCK}, + {"NOTE_LEEWAY", unix.NOTE_LEEWAY}, + {"NOTE_LINK", unix.NOTE_LINK}, + {"NOTE_LOWAT", unix.NOTE_LOWAT}, + {"NOTE_MACHTIME", unix.NOTE_MACHTIME}, + {"NOTE_MACH_CONTINUOUS_TIME", unix.NOTE_MACH_CONTINUOUS_TIME}, + {"NOTE_NONE", unix.NOTE_NONE}, + {"NOTE_NSECONDS", unix.NOTE_NSECONDS}, + {"NOTE_OOB", unix.NOTE_OOB}, + //{"NOTE_PCTRLMASK", unix.NOTE_PCTRLMASK}, -0x100000 (?!) + {"NOTE_PDATAMASK", unix.NOTE_PDATAMASK}, + {"NOTE_REAP", unix.NOTE_REAP}, + {"NOTE_RENAME", unix.NOTE_RENAME}, + {"NOTE_REVOKE", unix.NOTE_REVOKE}, + {"NOTE_SECONDS", unix.NOTE_SECONDS}, + {"NOTE_SIGNAL", unix.NOTE_SIGNAL}, + {"NOTE_TRACK", unix.NOTE_TRACK}, + {"NOTE_TRACKERR", unix.NOTE_TRACKERR}, + {"NOTE_TRIGGER", unix.NOTE_TRIGGER}, + {"NOTE_USECONDS", unix.NOTE_USECONDS}, + {"NOTE_VM_ERROR", unix.NOTE_VM_ERROR}, + {"NOTE_VM_PRESSURE", unix.NOTE_VM_PRESSURE}, + {"NOTE_VM_PRESSURE_SUDDEN_TERMINATE", unix.NOTE_VM_PRESSURE_SUDDEN_TERMINATE}, + {"NOTE_VM_PRESSURE_TERMINATE", unix.NOTE_VM_PRESSURE_TERMINATE}, + {"NOTE_WRITE", unix.NOTE_WRITE}, +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go new file mode 100644 index 000000000..3186b0c34 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go @@ -0,0 +1,33 @@ +package internal + +import "golang.org/x/sys/unix" + +var names = []struct { + n string + m uint32 +}{ + {"NOTE_ATTRIB", unix.NOTE_ATTRIB}, + {"NOTE_CHILD", unix.NOTE_CHILD}, + {"NOTE_DELETE", unix.NOTE_DELETE}, + {"NOTE_EXEC", unix.NOTE_EXEC}, + {"NOTE_EXIT", unix.NOTE_EXIT}, + {"NOTE_EXTEND", unix.NOTE_EXTEND}, + {"NOTE_FFAND", unix.NOTE_FFAND}, + {"NOTE_FFCOPY", unix.NOTE_FFCOPY}, + {"NOTE_FFCTRLMASK", unix.NOTE_FFCTRLMASK}, + {"NOTE_FFLAGSMASK", unix.NOTE_FFLAGSMASK}, + {"NOTE_FFNOP", unix.NOTE_FFNOP}, + {"NOTE_FFOR", unix.NOTE_FFOR}, + {"NOTE_FORK", unix.NOTE_FORK}, + {"NOTE_LINK", unix.NOTE_LINK}, + {"NOTE_LOWAT", unix.NOTE_LOWAT}, + {"NOTE_OOB", unix.NOTE_OOB}, + {"NOTE_PCTRLMASK", unix.NOTE_PCTRLMASK}, + {"NOTE_PDATAMASK", unix.NOTE_PDATAMASK}, + {"NOTE_RENAME", unix.NOTE_RENAME}, + {"NOTE_REVOKE", unix.NOTE_REVOKE}, + {"NOTE_TRACK", unix.NOTE_TRACK}, + {"NOTE_TRACKERR", unix.NOTE_TRACKERR}, + {"NOTE_TRIGGER", unix.NOTE_TRIGGER}, + {"NOTE_WRITE", unix.NOTE_WRITE}, +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go new file mode 100644 index 000000000..f69fdb930 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go @@ -0,0 +1,42 @@ +package internal + +import "golang.org/x/sys/unix" + +var names = []struct { + n string + m uint32 +}{ + {"NOTE_ABSTIME", unix.NOTE_ABSTIME}, + {"NOTE_ATTRIB", unix.NOTE_ATTRIB}, + {"NOTE_CHILD", unix.NOTE_CHILD}, + {"NOTE_CLOSE", unix.NOTE_CLOSE}, + {"NOTE_CLOSE_WRITE", unix.NOTE_CLOSE_WRITE}, + {"NOTE_DELETE", unix.NOTE_DELETE}, + {"NOTE_EXEC", unix.NOTE_EXEC}, + {"NOTE_EXIT", unix.NOTE_EXIT}, + {"NOTE_EXTEND", unix.NOTE_EXTEND}, + {"NOTE_FFAND", unix.NOTE_FFAND}, + {"NOTE_FFCOPY", unix.NOTE_FFCOPY}, + {"NOTE_FFCTRLMASK", unix.NOTE_FFCTRLMASK}, + {"NOTE_FFLAGSMASK", unix.NOTE_FFLAGSMASK}, + {"NOTE_FFNOP", unix.NOTE_FFNOP}, + {"NOTE_FFOR", unix.NOTE_FFOR}, + {"NOTE_FILE_POLL", unix.NOTE_FILE_POLL}, + {"NOTE_FORK", unix.NOTE_FORK}, + {"NOTE_LINK", unix.NOTE_LINK}, + {"NOTE_LOWAT", unix.NOTE_LOWAT}, + {"NOTE_MSECONDS", unix.NOTE_MSECONDS}, + {"NOTE_NSECONDS", unix.NOTE_NSECONDS}, + {"NOTE_OPEN", unix.NOTE_OPEN}, + {"NOTE_PCTRLMASK", unix.NOTE_PCTRLMASK}, + {"NOTE_PDATAMASK", unix.NOTE_PDATAMASK}, + {"NOTE_READ", unix.NOTE_READ}, + {"NOTE_RENAME", unix.NOTE_RENAME}, + {"NOTE_REVOKE", unix.NOTE_REVOKE}, + {"NOTE_SECONDS", unix.NOTE_SECONDS}, + {"NOTE_TRACK", unix.NOTE_TRACK}, + {"NOTE_TRACKERR", unix.NOTE_TRACKERR}, + {"NOTE_TRIGGER", unix.NOTE_TRIGGER}, + {"NOTE_USECONDS", unix.NOTE_USECONDS}, + {"NOTE_WRITE", unix.NOTE_WRITE}, +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go new file mode 100644 index 000000000..607e683bd --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go @@ -0,0 +1,32 @@ +//go:build freebsd || openbsd || netbsd || dragonfly || darwin + +package internal + +import ( + "fmt" + "os" + "strings" + "time" + + "golang.org/x/sys/unix" +) + +func Debug(name string, kevent *unix.Kevent_t) { + mask := uint32(kevent.Fflags) + + var ( + l []string + unknown = mask + ) + for _, n := range names { + if mask&n.m == n.m { + l = append(l, n.n) + unknown ^= n.m + } + } + if unknown > 0 { + l = append(l, fmt.Sprintf("0x%x", unknown)) + } + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s %10d:%-60s → %q\n", + time.Now().Format("15:04:05.000000000"), mask, strings.Join(l, " | "), name) +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_linux.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_linux.go new file mode 100644 index 000000000..35c734be4 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_linux.go @@ -0,0 +1,56 @@ +package internal + +import ( + "fmt" + "os" + "strings" + "time" + + "golang.org/x/sys/unix" +) + +func Debug(name string, mask, cookie uint32) { + names := []struct { + n string + m uint32 + }{ + {"IN_ACCESS", unix.IN_ACCESS}, + {"IN_ATTRIB", unix.IN_ATTRIB}, + {"IN_CLOSE", unix.IN_CLOSE}, + {"IN_CLOSE_NOWRITE", unix.IN_CLOSE_NOWRITE}, + {"IN_CLOSE_WRITE", unix.IN_CLOSE_WRITE}, + {"IN_CREATE", unix.IN_CREATE}, + {"IN_DELETE", unix.IN_DELETE}, + {"IN_DELETE_SELF", unix.IN_DELETE_SELF}, + {"IN_IGNORED", unix.IN_IGNORED}, + {"IN_ISDIR", unix.IN_ISDIR}, + {"IN_MODIFY", unix.IN_MODIFY}, + {"IN_MOVE", unix.IN_MOVE}, + {"IN_MOVED_FROM", unix.IN_MOVED_FROM}, + {"IN_MOVED_TO", unix.IN_MOVED_TO}, + {"IN_MOVE_SELF", unix.IN_MOVE_SELF}, + {"IN_OPEN", unix.IN_OPEN}, + {"IN_Q_OVERFLOW", unix.IN_Q_OVERFLOW}, + {"IN_UNMOUNT", unix.IN_UNMOUNT}, + } + + var ( + l []string + unknown = mask + ) + for _, n := range names { + if mask&n.m == n.m { + l = append(l, n.n) + unknown ^= n.m + } + } + if unknown > 0 { + l = append(l, fmt.Sprintf("0x%x", unknown)) + } + var c string + if cookie > 0 { + c = fmt.Sprintf("(cookie: %d) ", cookie) + } + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s %-30s → %s%q\n", + time.Now().Format("15:04:05.000000000"), strings.Join(l, "|"), c, name) +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go new file mode 100644 index 000000000..e5b3b6f69 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go @@ -0,0 +1,25 @@ +package internal + +import "golang.org/x/sys/unix" + +var names = []struct { + n string + m uint32 +}{ + {"NOTE_ATTRIB", unix.NOTE_ATTRIB}, + {"NOTE_CHILD", unix.NOTE_CHILD}, + {"NOTE_DELETE", unix.NOTE_DELETE}, + {"NOTE_EXEC", unix.NOTE_EXEC}, + {"NOTE_EXIT", unix.NOTE_EXIT}, + {"NOTE_EXTEND", unix.NOTE_EXTEND}, + {"NOTE_FORK", unix.NOTE_FORK}, + {"NOTE_LINK", unix.NOTE_LINK}, + {"NOTE_LOWAT", unix.NOTE_LOWAT}, + {"NOTE_PCTRLMASK", unix.NOTE_PCTRLMASK}, + {"NOTE_PDATAMASK", unix.NOTE_PDATAMASK}, + {"NOTE_RENAME", unix.NOTE_RENAME}, + {"NOTE_REVOKE", unix.NOTE_REVOKE}, + {"NOTE_TRACK", unix.NOTE_TRACK}, + {"NOTE_TRACKERR", unix.NOTE_TRACKERR}, + {"NOTE_WRITE", unix.NOTE_WRITE}, +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go new file mode 100644 index 000000000..1dd455bc5 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go @@ -0,0 +1,28 @@ +package internal + +import "golang.org/x/sys/unix" + +var names = []struct { + n string + m uint32 +}{ + {"NOTE_ATTRIB", unix.NOTE_ATTRIB}, + // {"NOTE_CHANGE", unix.NOTE_CHANGE}, // Not on 386? + {"NOTE_CHILD", unix.NOTE_CHILD}, + {"NOTE_DELETE", unix.NOTE_DELETE}, + {"NOTE_EOF", unix.NOTE_EOF}, + {"NOTE_EXEC", unix.NOTE_EXEC}, + {"NOTE_EXIT", unix.NOTE_EXIT}, + {"NOTE_EXTEND", unix.NOTE_EXTEND}, + {"NOTE_FORK", unix.NOTE_FORK}, + {"NOTE_LINK", unix.NOTE_LINK}, + {"NOTE_LOWAT", unix.NOTE_LOWAT}, + {"NOTE_PCTRLMASK", unix.NOTE_PCTRLMASK}, + {"NOTE_PDATAMASK", unix.NOTE_PDATAMASK}, + {"NOTE_RENAME", unix.NOTE_RENAME}, + {"NOTE_REVOKE", unix.NOTE_REVOKE}, + {"NOTE_TRACK", unix.NOTE_TRACK}, + {"NOTE_TRACKERR", unix.NOTE_TRACKERR}, + {"NOTE_TRUNCATE", unix.NOTE_TRUNCATE}, + {"NOTE_WRITE", unix.NOTE_WRITE}, +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go new file mode 100644 index 000000000..f1b2e73bd --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go @@ -0,0 +1,45 @@ +package internal + +import ( + "fmt" + "os" + "strings" + "time" + + "golang.org/x/sys/unix" +) + +func Debug(name string, mask int32) { + names := []struct { + n string + m int32 + }{ + {"FILE_ACCESS", unix.FILE_ACCESS}, + {"FILE_MODIFIED", unix.FILE_MODIFIED}, + {"FILE_ATTRIB", unix.FILE_ATTRIB}, + {"FILE_TRUNC", unix.FILE_TRUNC}, + {"FILE_NOFOLLOW", unix.FILE_NOFOLLOW}, + {"FILE_DELETE", unix.FILE_DELETE}, + {"FILE_RENAME_TO", unix.FILE_RENAME_TO}, + {"FILE_RENAME_FROM", unix.FILE_RENAME_FROM}, + {"UNMOUNTED", unix.UNMOUNTED}, + {"MOUNTEDOVER", unix.MOUNTEDOVER}, + {"FILE_EXCEPTION", unix.FILE_EXCEPTION}, + } + + var ( + l []string + unknown = mask + ) + for _, n := range names { + if mask&n.m == n.m { + l = append(l, n.n) + unknown ^= n.m + } + } + if unknown > 0 { + l = append(l, fmt.Sprintf("0x%x", unknown)) + } + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s %10d:%-30s → %q\n", + time.Now().Format("15:04:05.000000000"), mask, strings.Join(l, " | "), name) +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_windows.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_windows.go new file mode 100644 index 000000000..52bf4ce53 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_windows.go @@ -0,0 +1,40 @@ +package internal + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "golang.org/x/sys/windows" +) + +func Debug(name string, mask uint32) { + names := []struct { + n string + m uint32 + }{ + {"FILE_ACTION_ADDED", windows.FILE_ACTION_ADDED}, + {"FILE_ACTION_REMOVED", windows.FILE_ACTION_REMOVED}, + {"FILE_ACTION_MODIFIED", windows.FILE_ACTION_MODIFIED}, + {"FILE_ACTION_RENAMED_OLD_NAME", windows.FILE_ACTION_RENAMED_OLD_NAME}, + {"FILE_ACTION_RENAMED_NEW_NAME", windows.FILE_ACTION_RENAMED_NEW_NAME}, + } + + var ( + l []string + unknown = mask + ) + for _, n := range names { + if mask&n.m == n.m { + l = append(l, n.n) + unknown ^= n.m + } + } + if unknown > 0 { + l = append(l, fmt.Sprintf("0x%x", unknown)) + } + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s %-65s → %q\n", + time.Now().Format("15:04:05.000000000"), strings.Join(l, " | "), filepath.ToSlash(name)) +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/freebsd.go b/vendor/github.com/fsnotify/fsnotify/internal/freebsd.go new file mode 100644 index 000000000..547df1df8 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/freebsd.go @@ -0,0 +1,31 @@ +//go:build freebsd + +package internal + +import ( + "syscall" + + "golang.org/x/sys/unix" +) + +var ( + SyscallEACCES = syscall.EACCES + UnixEACCES = unix.EACCES +) + +var maxfiles uint64 + +func SetRlimit() { + // Go 1.19 will do this automatically: https://go-review.googlesource.com/c/go/+/393354/ + var l syscall.Rlimit + err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &l) + if err == nil && l.Cur != l.Max { + l.Cur = l.Max + syscall.Setrlimit(syscall.RLIMIT_NOFILE, &l) + } + maxfiles = uint64(l.Cur) +} + +func Maxfiles() uint64 { return maxfiles } +func Mkfifo(path string, mode uint32) error { return unix.Mkfifo(path, mode) } +func Mknod(path string, mode uint32, dev int) error { return unix.Mknod(path, mode, uint64(dev)) } diff --git a/vendor/github.com/fsnotify/fsnotify/internal/internal.go b/vendor/github.com/fsnotify/fsnotify/internal/internal.go new file mode 100644 index 000000000..7daa45e19 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/internal.go @@ -0,0 +1,2 @@ +// Package internal contains some helpers. +package internal diff --git a/vendor/github.com/fsnotify/fsnotify/internal/unix.go b/vendor/github.com/fsnotify/fsnotify/internal/unix.go new file mode 100644 index 000000000..30976ce97 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/unix.go @@ -0,0 +1,31 @@ +//go:build !windows && !darwin && !freebsd + +package internal + +import ( + "syscall" + + "golang.org/x/sys/unix" +) + +var ( + SyscallEACCES = syscall.EACCES + UnixEACCES = unix.EACCES +) + +var maxfiles uint64 + +func SetRlimit() { + // Go 1.19 will do this automatically: https://go-review.googlesource.com/c/go/+/393354/ + var l syscall.Rlimit + err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &l) + if err == nil && l.Cur != l.Max { + l.Cur = l.Max + syscall.Setrlimit(syscall.RLIMIT_NOFILE, &l) + } + maxfiles = uint64(l.Cur) +} + +func Maxfiles() uint64 { return maxfiles } +func Mkfifo(path string, mode uint32) error { return unix.Mkfifo(path, mode) } +func Mknod(path string, mode uint32, dev int) error { return unix.Mknod(path, mode, dev) } diff --git a/vendor/github.com/fsnotify/fsnotify/internal/unix2.go b/vendor/github.com/fsnotify/fsnotify/internal/unix2.go new file mode 100644 index 000000000..37dfeddc2 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/unix2.go @@ -0,0 +1,7 @@ +//go:build !windows + +package internal + +func HasPrivilegesForSymlink() bool { + return true +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/windows.go b/vendor/github.com/fsnotify/fsnotify/internal/windows.go new file mode 100644 index 000000000..a72c64954 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/windows.go @@ -0,0 +1,41 @@ +//go:build windows + +package internal + +import ( + "errors" + + "golang.org/x/sys/windows" +) + +// Just a dummy. +var ( + SyscallEACCES = errors.New("dummy") + UnixEACCES = errors.New("dummy") +) + +func SetRlimit() {} +func Maxfiles() uint64 { return 1<<64 - 1 } +func Mkfifo(path string, mode uint32) error { return errors.New("no FIFOs on Windows") } +func Mknod(path string, mode uint32, dev int) error { return errors.New("no device nodes on Windows") } + +func HasPrivilegesForSymlink() bool { + var sid *windows.SID + err := windows.AllocateAndInitializeSid( + &windows.SECURITY_NT_AUTHORITY, + 2, + windows.SECURITY_BUILTIN_DOMAIN_RID, + windows.DOMAIN_ALIAS_RID_ADMINS, + 0, 0, 0, 0, 0, 0, + &sid) + if err != nil { + return false + } + defer windows.FreeSid(sid) + token := windows.Token(0) + member, err := token.IsMember(sid) + if err != nil { + return false + } + return member || token.IsElevated() +} diff --git a/vendor/github.com/fsnotify/fsnotify/system_bsd.go b/vendor/github.com/fsnotify/fsnotify/system_bsd.go new file mode 100644 index 000000000..f65e8fe3e --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/system_bsd.go @@ -0,0 +1,7 @@ +//go:build freebsd || openbsd || netbsd || dragonfly + +package fsnotify + +import "golang.org/x/sys/unix" + +const openMode = unix.O_NONBLOCK | unix.O_RDONLY | unix.O_CLOEXEC diff --git a/vendor/github.com/fsnotify/fsnotify/system_darwin.go b/vendor/github.com/fsnotify/fsnotify/system_darwin.go new file mode 100644 index 000000000..a29fc7aab --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/system_darwin.go @@ -0,0 +1,8 @@ +//go:build darwin + +package fsnotify + +import "golang.org/x/sys/unix" + +// note: this constant is not defined on BSD +const openMode = unix.O_EVTONLY | unix.O_CLOEXEC diff --git a/vendor/modules.txt b/vendor/modules.txt index 4bb8c6176..f5baecac6 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -83,10 +83,11 @@ github.com/emicklei/go-restful/log # github.com/felixge/httpsnoop v1.0.4 ## explicit; go 1.13 github.com/felixge/httpsnoop -# github.com/fsnotify/fsnotify v1.5.4 -## explicit; go 1.16 +# github.com/fsnotify/fsnotify v1.8.0 +## explicit; go 1.17 github.com/fsnotify/fsnotify -# github.com/go-logr/logr v1.4.1 +github.com/fsnotify/fsnotify/internal +# github.com/go-logr/logr v1.4.2 ## explicit; go 1.18 github.com/go-logr/logr github.com/go-logr/logr/funcr From 8f2a701befb5d68bfc16da0d3b47e9bcf597960f Mon Sep 17 00:00:00 2001 From: samhalim Date: Sat, 22 Feb 2025 00:41:24 +0000 Subject: [PATCH 06/25] Return error for any Data Cache related code in NodeStageVolume & NodeUnstageVolume. --- pkg/gce-pd-csi-driver/node.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/gce-pd-csi-driver/node.go b/pkg/gce-pd-csi-driver/node.go index c8170b27e..23f9c150c 100644 --- a/pkg/gce-pd-csi-driver/node.go +++ b/pkg/gce-pd-csi-driver/node.go @@ -338,7 +338,7 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage } devicePath, err = setupCaching(devFsPath, req, nodeId) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("Error setting up cache: %v", err.Error())) + return nil, status.Error(codes.DataLoss, fmt.Sprintf("Error setting up cache: %v", err.Error())) } } @@ -494,7 +494,7 @@ func (ns *GCENodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUns nodeId := ns.MetadataService.GetName() err := cleanupCache(volumeID, nodeId) if err != nil { - klog.Errorf("Failed to cleanup cache for volume %s: %v", volumeID, err) + return nil, status.Errorf(codes.DataLoss, "Failed to cleanup cache for volume %s: %v", volumeID, err) } } klog.V(4).Infof("NodeUnstageVolume succeeded on %v from %s", volumeID, stagingTargetPath) From 4212f84e586b8916965790aade068e90caa1bbc3 Mon Sep 17 00:00:00 2001 From: Hung Nguyen Date: Mon, 24 Feb 2025 23:25:43 +0000 Subject: [PATCH 07/25] update RAIDing and validation steps --- cmd/gce-pd-csi-driver/main.go | 81 +++++--- pkg/common/constants.go | 7 + pkg/common/runcmd.go | 12 +- pkg/gce-pd-csi-driver/cache.go | 277 ++++++++++++++++++++-------- pkg/gce-pd-csi-driver/controller.go | 2 +- pkg/gce-pd-csi-driver/node.go | 6 +- test/e2e/tests/setup_e2e_test.go | 5 +- test/remote/client-wrappers.go | 11 +- 8 files changed, 284 insertions(+), 117 deletions(-) diff --git a/cmd/gce-pd-csi-driver/main.go b/cmd/gce-pd-csi-driver/main.go index ccc6a67ae..b16ba2473 100644 --- a/cmd/gce-pd-csi-driver/main.go +++ b/cmd/gce-pd-csi-driver/main.go @@ -27,12 +27,8 @@ import ( "strings" "time" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "k8s.io/klog/v2" "k8s.io/utils/strings/slices" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/deviceutils" gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute" @@ -95,9 +91,7 @@ var ( ) const ( - driverName = "pd.csi.storage.gke.io" - dataCacheLabel = "datacache-storage-gke-io" - dataCacheLabelValue = "enabled" + driverName = "pd.csi.storage.gke.io" ) func init() { @@ -331,37 +325,74 @@ func urlFlag(target **url.URL, name string, usage string) { }) } -func setupDataCache(ctx context.Context, nodeName string, nodeId string) error { +func fetchLssdsForRaiding(lssdCount int) ([]string, error) { + allLssds, err := driver.FetchAllLssds() + if err != nil { + return nil, fmt.Errorf("Error listing all LSSDs %v", err) + } + + raidedLssds, err := driver.FetchRaidedLssds() + if err != nil { + return nil, fmt.Errorf("Error listing RAIDed LSSDs %v", err) + } + + unRaidedLssds := []string{} + for _, l := range allLssds { + if !slices.Contains(raidedLssds, l) { + unRaidedLssds = append(unRaidedLssds, l) + } + if len(unRaidedLssds) == lssdCount { + break + } + } + + LSSDsWithEmptyMountPoint, err := driver.FetchLSSDsWihtEmptyMountPoint() + if err != nil { + return nil, fmt.Errorf("Error listing LSSDs with empty mountpoint: %v", err) + } + + // We need to ensure the disks to be used for Datacache are both unRAIDed & not containing mountpoints for ephemeral storage already + availableLssds := slices.Filter(nil, unRaidedLssds, func(e string) bool { + return slices.Contains(LSSDsWithEmptyMountPoint, e) + }) + + if len(availableLssds) == 0 { + return nil, fmt.Errorf("No LSSDs available to set up caching") + } + + if len(availableLssds) < lssdCount { + return nil, fmt.Errorf("Not enough LSSDs available to set up caching. Available LSSDs: %v, wanted LSSDs: %v", len(availableLssds), lssdCount) + } + return availableLssds, nil +} + +func setupDataCache(ctx context.Context, nodeName string) error { isAlreadyRaided, err := driver.IsRaided() if err != nil { - klog.V(4).Infof("Errored while scanning for available LocalSSDs err:%v; continuing Raiding", err) + klog.V(2).Infof("Errored while scanning for available LocalSSDs err:%v; continuing Raiding", err) } else if isAlreadyRaided { - klog.V(4).Infof("Local SSDs are already RAIDed. Skipping Data Cache setup.") + klog.V(2).Infof("Local SSDs are already RAIDed. Skipping Datacache setup.") return nil } lssdCount := common.LocalSSDCountForDataCache if nodeName != common.TestNode { - cfg, err := rest.InClusterConfig() - if err != nil { - return err - } - kubeClient, err := kubernetes.NewForConfig(cfg) - if err != nil { - return err + var err error + lssdCount, err = driver.GetDataCacheCountFromNodeLabel(ctx, nodeName) + if lssdCount == 0 { + klog.Infof("Datacache is not enabled on node %v", nodeName) + return nil } - node, err := kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) if err != nil { - // We could retry, but this error will also crashloop the driver which may be as good a way to retry as any. return err } - if val, found := node.GetLabels()[dataCacheLabel]; !found || val != dataCacheLabelValue { - klog.V(2).Infof("Datacache not enabled for node %s; node label %s=%s and not %s", nodeName, dataCacheLabel, val, dataCacheLabelValue) - return nil - } } - klog.V(2).Info("Raiding local ssds to setup data cache") - if err := driver.RaidLocalSsds(); err != nil { + lssdNames, err := fetchLssdsForRaiding(lssdCount) + if err != nil { + klog.Fatalf("Failed to get sufficient SSDs for Datacache's caching setup: %v", err) + } + klog.V(2).Infof("Raiding local ssds to setup data cache: %v", lssdNames) + if err := driver.RaidLocalSsds(lssdNames); err != nil { return fmt.Errorf("Failed to Raid local SSDs, unable to setup data caching, got error %v", err) } diff --git a/pkg/common/constants.go b/pkg/common/constants.go index 0279dd3cc..5e7ec51af 100644 --- a/pkg/common/constants.go +++ b/pkg/common/constants.go @@ -44,6 +44,13 @@ const ( ContexLocalSsdCacheSize = "local-ssd-cache-size" // Node name for E2E tests TestNode = "test-node-csi-e2e" + + // Default LSSD count for datacache E2E tests + LocalSSDCountForDataCache = 2 + + // Node label for datacache + NodeLabelPrefix = "cloud.google.com/%s" + DataCacheLssdCountLabel = "gke-data-cache-disk" ) // doc https://cloud.google.com/compute/docs/disks/hyperdisks#max-total-disks-per-vm diff --git a/pkg/common/runcmd.go b/pkg/common/runcmd.go index 71240d2a9..39457dfb1 100644 --- a/pkg/common/runcmd.go +++ b/pkg/common/runcmd.go @@ -16,7 +16,7 @@ const ( // RunCommand wraps a k8s exec to deal with the no child process error. Same as exec.CombinedOutput. // On error, the output is included so callers don't need to echo it again. -func RunCommand(pipeCmd string, pipeCmdArg string, cmd1 string, execCmdArgs ...string) ([]byte, error) { +func RunCommand(pipeCmd string, pipeCmdArg []string, cmd1 string, execCmdArgs ...string) ([]byte, error) { execCmd1 := exec.Command(cmd1, execCmdArgs...) if pipeCmd != "" { @@ -47,9 +47,9 @@ func checkError(err error, execCmd exec.Cmd) error { } return err } -func execPipeCommand(pipeCmd string, pipeCmdArg string, execCmd1 *exec.Cmd) ([]byte, error) { +func execPipeCommand(pipeCmd string, pipeCmdArg []string, execCmd1 *exec.Cmd) ([]byte, error) { - execPipeCmd := exec.Command(pipeCmd, pipeCmdArg) + execPipeCmd := exec.Command(pipeCmd, pipeCmdArg...) stdoutPipe, err := execCmd1.StdoutPipe() if err != nil { klog.Errorf("failed command %v: got error:%v", execCmd1, err) @@ -63,8 +63,12 @@ func execPipeCommand(pipeCmd string, pipeCmdArg string, execCmd1 *exec.Cmd) ([]b execPipeCmd.Stdin = stdoutPipe output, err := execPipeCmd.CombinedOutput() if err != nil { + // Some commands (such as grep) will return an error with exit status of 1 + if len(output) == 0 && err.(*exec.ExitError).ExitCode() == 1 { + return output, nil + } err = checkError(err, *execPipeCmd) - return nil, fmt.Errorf("%s failed: %w; output: %s", pipeCmd, err, string(output)) + return nil, fmt.Errorf("%s failed: %w; output: %s", execPipeCmd, err, string(output)) } return output, nil diff --git a/pkg/gce-pd-csi-driver/cache.go b/pkg/gce-pd-csi-driver/cache.go index c9764d933..c079a3360 100644 --- a/pkg/gce-pd-csi-driver/cache.go +++ b/pkg/gce-pd-csi-driver/cache.go @@ -1,45 +1,57 @@ package gceGCEDriver import ( + "context" "fmt" "regexp" "strconv" "strings" csi "github.com/container-storage-interface/spec/lib/go/csi" - fsnotify "github.com/fsnotify/fsnotify" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/klog/v2" - "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" ) const ( - cacheSuffix = "csi-fast" - mainLvSuffix = "csi-main" - raidedLocalSsdName = "csi-driver-data-cache" - raidMode = "0" - raidedLssdPrefix = "/dev/md/" + cacheSuffix = "csi-fast" + mainLvSuffix = "csi-main" + raidedLocalSsdName = "csi-driver-data-cache" + raidMode = "0" + initialRaidedLocalSsdPath = "/dev/md0" ) -var raidedLocalSsdPath = raidedLssdPrefix + raidedLocalSsdName +func fetchRAIDedLocalSsdPath() (string, error) { + args := []string{ + "--detail", + "--scan", + } + info, err := common.RunCommand("grep", []string{raidedLocalSsdName}, "mdadm", args...) + if err != nil || len(info) == 0 { + return "", fmt.Errorf("Error getting RAIDed device path for Datacache %v, output:%v ===============", err, string(info)) + } + infoString := strings.TrimSpace(string(info)) + infoSlice := strings.Split(infoString, " ") + + // We want to get the second element in the array, which is the path to the RAIDed device + return infoSlice[1], nil +} func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId string) (string, error) { + + // The device path may have changed after rebooting, so we need to fetch the path again + raidedLocalSsdPath, err := fetchRAIDedLocalSsdPath() + if err != nil { + return "", err + } + volumeId := req.GetVolumeId() volumeGroupName := getVolumeGroupName(nodeId) mainDevicePath := "/dev/" + volumeGroupName + "/" + getLvName(mainLvSuffix, volumeId) mainLvName := getLvName(mainLvSuffix, volumeId) klog.V(2).Infof("Volume group available on node %v ", volumeGroupName) - - info, err := common.RunCommand("grep", raidedLocalSsdName, "ls", raidedLssdPrefix) - if err != nil { - klog.Errorf("failed while listing raided devices, err: %v, output:%v", err, info) - } - infoString := strings.TrimSpace(string(info)) - raidedLocalSsdPath = raidedLssdPrefix + infoString - vgExists := checkVgExists(volumeGroupName) if vgExists { // Clean up Volume Group before adding the PD @@ -58,22 +70,24 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str "-o", "vg_name", } - info, err = common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "pvs", args...) + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "pvs", args...) if err != nil { klog.Errorf("errored while checking physical volume details %v: %s", err, info) // On error info contains the error message which we cannot use for further steps info = nil } - infoString = strings.TrimSpace(strings.ReplaceAll(string(info), "\n", " ")) + infoString := strings.TrimSpace(strings.ReplaceAll(string(info), "\n", " ")) infoString = strings.ReplaceAll(infoString, ".", "") infoString = strings.ReplaceAll(infoString, "\"", "") infoSlice := strings.Split(strings.TrimSpace(infoString), " ") vgNameForPv := strings.TrimSpace(infoSlice[(len(infoSlice) - 1)]) + klog.V(2).Infof("============================== Physical volume is part of Volume group: %v ==============================", vgNameForPv) if vgNameForPv == volumeGroupName { - klog.V(2).Infof("Physical Volume(PV) already exists in the Volume Group %v", volumeGroupName) + klog.V(2).Infof("============================== Physical Volume(PV) already exists in the Volume Group ==============================") } else if vgNameForPv != "VG" && vgNameForPv != "" { - info, err = common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "vgchange", []string{"-an", vgNameForPv}...) + + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgchange", []string{"-an", vgNameForPv}...) if err != nil { klog.Errorf("Errored while deactivating VG %v: err: %v: %s", vgNameForPv, err, info) } @@ -81,6 +95,7 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str reduceVolumeGroup(vgNameForPv, false) _, isCached := isCachingSetup(mainLvName) // We will continue to uncache even if it errors to check caching as it is not a terminal issue. + if isCached { // Uncache LV args = []string{ @@ -89,20 +104,20 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str "--force", "-y", // force remove cache without flushing data } - info, err = common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "lvconvert", args...) + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvconvert", args...) if err != nil { return "", fmt.Errorf("errored while uncaching main LV. %v: %s", err, info) } // CLean up volume group to remove any dangling PV refrences reduceVolumeGroup(vgNameForPv, false) } - info, err = common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "vgmerge", []string{volumeGroupName, vgNameForPv}...) + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgmerge", []string{volumeGroupName, vgNameForPv}...) if err != nil { return "", fmt.Errorf("Errored while merging the PV Volume group %s into %s %v: %s", vgNameForPv, volumeGroupName, err, info) } } else { - info, err := common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "vgextend", []string{volumeGroupName, devicePath}...) + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgextend", []string{volumeGroupName, devicePath}...) if err != nil { return "", fmt.Errorf("Errored while extending Volume group to add PV %v, error: %v: %s", devicePath, err, info) } @@ -115,7 +130,7 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str "-o", "lv_name", } - lvList, err := common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "lvs", args...) + lvList, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvs", args...) if err != nil { return mainDevicePath, fmt.Errorf("Errored while checking logical volume for the device %s %w: %s", devicePath, err, info) } @@ -129,7 +144,7 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str volumeGroupName, devicePath, } - info, err = common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "lvcreate", args...) + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvcreate", args...) if err != nil { return mainDevicePath, fmt.Errorf("Errored setting up logical volume for the volume %s %w: %s", devicePath, err, info) } @@ -144,7 +159,7 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str // Validate that cache is setup for required size klog.V(2).Infof("Assuming valid data cache size and mode, resizing cache is not supported") } else { - fastCacheSize := req.GetPublishContext()[common.ContexLocalSsdCacheSize] + fastCacheSize := req.GetPublishContext()[common.ContextDataCacheSize] chunkSize := "960" // Cannot use default chunk size(64KiB) as it errors on maxChunksAllowed. Unit - KiB args = []string{ "--yes", @@ -156,7 +171,7 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str volumeGroupName, raidedLocalSsdPath, } - info, err = common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "lvcreate", args...) + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvcreate", args...) if err != nil { return mainDevicePath, fmt.Errorf("Errored while creating cache %w: %s", err, info) } @@ -177,14 +192,14 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str "--force", "-y", } - info, err = common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "lvconvert", args...) + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvconvert", args...) if err != nil { return mainDevicePath, fmt.Errorf("Errored while setting up caching for volume %s %w: %s", devicePath, err, info) } } // activate all the LVs in the Volume group - info, err = common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "vgchange", []string{"-ay", volumeGroupName}...) + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgchange", []string{"-ay", volumeGroupName}...) if err != nil { // The logical volumes would not be accessible if the group is not activated return mainDevicePath, fmt.Errorf("Failed to activate volume group %v %v:%s", volumeGroupName, err, info) @@ -192,9 +207,142 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str return mainDevicePath, nil } +func ValidateDataCacheConfig(dataCacheMode string, datacacheSize string, ctx context.Context, nodeName string) error { + if dataCacheMode != "" && datacacheSize != "" { + isAlreadyRaided, err := IsRaided() + if err != nil { + return fmt.Errorf("Local SSDs are not setup for caching; got error: %v", err) + } + if !isAlreadyRaided { + return fmt.Errorf("Local SSDs are not setup for caching") + } + return nil + } + klog.Infof("Data cache is not enabled for PVC") + return nil +} + +func GetDataCacheCountFromNodeLabel(ctx context.Context, nodeName string) (int, error) { + if nodeName == common.TestNode { + return common.LocalSSDCountForDataCache, nil + } + cfg, err := rest.InClusterConfig() + // We want to capture API errors with node label fetching, so return -1 + // in those cases instead of 0. + if err != nil { + return -1, err + } + kubeClient, err := kubernetes.NewForConfig(cfg) + if err != nil { + return -1, err + } + node, err := kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + // We could retry, but this error will also crashloop the driver which may be as good a way to retry as any. + return -1, err + } + if val, found := node.GetLabels()[fmt.Sprintf(common.NodeLabelPrefix, common.DataCacheLssdCountLabel)]; found { + dataCacheCount, err := strconv.Atoi(val) + if err != nil { + return -1, fmt.Errorf("Error getting Datacache's LSSD count from node label: %v", err) + } + klog.Infof("Number of local SSDs requested for Datacache: %v", dataCacheCount) + return dataCacheCount, nil + } + return 0, fmt.Errorf("Cannot get Datacache's LSSD count from node label") +} + +func FetchRaidedLssdCountForDatacache() (int, error) { + args := []string{ + "--detail", + initialRaidedLocalSsdPath, + } + info, err := common.RunCommand("grep", []string{"Raid Devices"}, "mdadm", args...) + if err != nil { + return 0, fmt.Errorf("Error getting RAIDed devices for Datacache") + } + if len(info) != 0 { + raidedDeviceInfo := strings.Split(strings.TrimSpace(string(info)), ":") + // raidedDeviceInfo should be in "Raid Devices : X" format + raidedDeviceCount, _ := strconv.Atoi(strings.TrimSpace(raidedDeviceInfo[1])) + return raidedDeviceCount, nil + } + return 0, nil +} + +func FetchRaidedLssds() ([]string, error) { + raidedLssdList := []string{} + + args := []string{ + "--detail", + "--scan", + "--export", + } + + info, err := common.RunCommand("grep", []string{"/dev"}, "mdadm", args...) + if err != nil { + return nil, fmt.Errorf("error fetching RAIDed LSSDs: %v; err:%v", info, err) + } + + if len(info) != 0 { + infoList := strings.Split(strings.TrimSpace(string(info)), "\n") + for _, ssd := range infoList { + ssdInfo := strings.TrimSpace(ssd) + // SSD name comes after "=" on each output line (e.g. MD_DEVICE_dev_nvme3n1_DEV=/dev/nvme3n1) + ssdName := strings.Split(ssdInfo, "=")[1] + raidedLssdList = append(raidedLssdList, ssdName) + } + } + + klog.V(2).Infof("Raided NVME list %v", raidedLssdList) + + return raidedLssdList, nil +} + +func FetchAllLssds() ([]string, error) { + diskList := []string{} + + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipeCmdArg */, "lsblk", []string{"-o", "NAME,MODEL", "-p", "-d", "-n"}...) + if err != nil { + return nil, fmt.Errorf("errored while fetching NVME disks info: %v; err:%v", info, err) + } + infoList := strings.Split(strings.TrimSpace(string(info)), "\n") + re, err := regexp.Compile("nvme_card([0-9]+)?$") + if err != nil { + klog.V(2).ErrorS(err, "Errored while compiling to check PD or LSSD") + } + for _, ssd := range infoList { + ssd = strings.TrimSpace(ssd) + if strings.HasPrefix(ssd, "/dev/nvme") { + ssdDetails := strings.Split(ssd, " ") + lssd := re.MatchString(ssdDetails[1]) + if lssd { + diskList = append(diskList, strings.TrimSpace(ssdDetails[0])) + } + } + } + + klog.V(2).Infof("NVME list %v", diskList) + + return diskList, nil +} + +func FetchLSSDsWihtEmptyMountPoint() ([]string, error) { + info, err := common.RunCommand("grep", []string{"-E", `^\S+\s*$`} /* pipeCmdArg */, "lsblk", []string{"-o", "NAME,MOUNTPOINT", "-pdn"}...) + if err != nil { + return nil, fmt.Errorf("Error while fetching disks with no mount point: %v; err:%v", info, err) + } + infoList := strings.Split(string(info), "\n") + diskList := []string{} + for _, ssd := range infoList { + diskList = append(diskList, strings.TrimSpace(ssd)) + } + return diskList, nil +} + func checkVgExists(volumeGroupName string) bool { args := []string{} - info, err := common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "vgscan", args...) + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgscan", args...) if err != nil { klog.Errorf("Errored while checking if volume group exists %v: %s", err, info) return false @@ -215,7 +363,7 @@ func cleanupCache(volumeId string, nodeId string) error { "-an", "/dev/" + volumeGroupName + "/" + mainLvName, } - info, err := common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "lvchange", args...) + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvchange", args...) if err != nil { return fmt.Errorf("Failed to deactivate volume for uncaching %s %v: %s", volumeId, err, info) } @@ -224,7 +372,7 @@ func cleanupCache(volumeId string, nodeId string) error { volumeGroupName + "/" + mainLvName, "-y", } - info, err = common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "lvconvert", args...) + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvconvert", args...) if err != nil { return fmt.Errorf("Failed to uncache volume %s %w: %s", volumeId, err, info) } @@ -252,14 +400,14 @@ func createVg(volumeGroupName string, raidedLocalSsds string) error { raidedLocalSsds, "-v", } - info, err := common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "vgcreate", args...) + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgcreate", args...) if err != nil { return fmt.Errorf("Volume group creation failed %w: %s", err, info) } klog.Infof("Volume group creation succeeded for %v", volumeGroupName) args = []string{} - info, err = common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "vgscan", args...) + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgscan", args...) if err != nil { klog.Errorf("Failed to scan for volume group post creation, continuing: %v: %s", err, info) } @@ -274,75 +422,54 @@ func reduceVolumeGroup(volumeGroupName string, force bool) { if force { args = append(args, "--force") } - info, err := common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "vgreduce", args...) + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgreduce", args...) if err != nil { klog.Errorf("Errored while cleaning up volume group %v: %s", err, info) } } -func RaidLocalSsds() error { - isAlreadyRaided, err := isRaided() - if err != nil { - klog.V(2).Infof("Errored while scanning for available LocalSSDs err:%v; continuing Raiding", err) - } else if isAlreadyRaided { - klog.V(2).Infof("Local SSDs are already RAIDed, no further action needed here") - return nil - } - diskList := []string{} - info, err := common.RunCommand("" /* pipedCmd */, "" /* pipeCmdArg */, "lsblk", []string{"-o", "NAME,MODEL", "-p", "-d", "-n"}...) - if err != nil { - return fmt.Errorf("Failed to fetch LSSD info: %v; err:%v", info, err) - } - infoList := strings.Split(strings.TrimSpace(string(info)), "\n") - re, err := regexp.Compile("nvme_card([0-9]+)?$") - if err != nil { - return fmt.Errorf("Errored while compiling to check PD or LSSD %s", err) - } - for _, ssd := range infoList { - ssd = strings.TrimSpace(ssd) - if strings.HasPrefix(ssd, "/dev/nvme") { - ssdDetails := strings.Split(ssd, " ") - lssd := re.MatchString(ssdDetails[1]) - if lssd { - diskList = append(diskList, strings.TrimSpace(ssdDetails[0])) - } - } - } - nvmeDiskCount := len(diskList) - if nvmeDiskCount == 0 { - return fmt.Errorf("No local SSDs found for raiding") - } +func RaidLocalSsds(availableLssds []string) error { args := []string{ "--create", - raidedLssdPrefix + raidedLocalSsdName, + initialRaidedLocalSsdPath, + "--name", + raidedLocalSsdName, "-l" + raidMode, // Force RAIDing as sometime it might fail for caution if there is just 1 LSSD present as 1 LSSD need not be RAIDed "--force", "-n", - strconv.Itoa(nvmeDiskCount), + strconv.Itoa(len(availableLssds)), } - args = append(args, diskList...) - info, err = common.RunCommand("" /* pipedCmd */, "" /* pipeCmdArg */, "mdadm", args...) + args = append(args, availableLssds...) + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipeCmdArg */, "mdadm", args...) if err != nil { return fmt.Errorf("errored while RAIDing LSSDs info: %v; err:%v", info, err) } // Validate if Raided successfully - isAlreadyRaided, err = isRaided() + isAlreadyRaided, err := IsRaided() if err != nil { klog.V(2).Infof("Errored while scanning for available raided LocalSSDs err:%v=", err) } if !isAlreadyRaided { return fmt.Errorf("failed raiding, raided device not found on scanning") } + + raidedDataCacheCount, err := FetchRaidedLssdCountForDatacache() + if err != nil { + return err + } + if raidedDataCacheCount != len(availableLssds) { + return fmt.Errorf("Local SSDs reserved do not match the requested count") + } return nil } -func isRaided() (bool, error) { +func IsRaided() (bool, error) { args := []string{ "--detail", "--scan", } - info, err := common.RunCommand("" /* pipedCmd */, "" /* pipeCmdArg */, "mdadm", args...) + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipeCmdArg */, "mdadm", args...) if err != nil { return false, fmt.Errorf("errored while scanning for raided LSSD %v: %s", err, info) } @@ -360,7 +487,7 @@ func isCachingSetup(mainLvName string) (error, bool) { "-o", "pool_lv", } - poolName, err := common.RunCommand("" /* pipedCmd */, "" /* pipeCmdArg */, "lvs", args...) + poolName, err := common.RunCommand("" /* pipedCmd */, nil /* pipeCmdArg */, "lvs", args...) if err != nil { return fmt.Errorf("Failed to check if caching is setup %w", err), false } diff --git a/pkg/gce-pd-csi-driver/controller.go b/pkg/gce-pd-csi-driver/controller.go index a2172bef7..7a5f6cbc0 100644 --- a/pkg/gce-pd-csi-driver/controller.go +++ b/pkg/gce-pd-csi-driver/controller.go @@ -976,7 +976,7 @@ func (gceCS *GCEControllerServer) executeControllerPublishVolume(ctx context.Con if gceCS.enableDataCache && req.GetVolumeContext() != nil { if req.GetVolumeContext()[common.ContextDataCacheSize] != "" { pubVolResp.PublishContext = map[string]string{} - pubVolResp.PublishContext[common.ContexLocalSsdCacheSize] = req.GetVolumeContext()[common.ContextDataCacheSize] + pubVolResp.PublishContext[common.ContextDataCacheSize] = req.GetVolumeContext()[common.ContextDataCacheSize] pubVolResp.PublishContext[common.ContextDataCacheMode] = req.GetVolumeContext()[common.ContextDataCacheMode] } } diff --git a/pkg/gce-pd-csi-driver/node.go b/pkg/gce-pd-csi-driver/node.go index 23f9c150c..97442c0c8 100644 --- a/pkg/gce-pd-csi-driver/node.go +++ b/pkg/gce-pd-csi-driver/node.go @@ -328,7 +328,7 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage klog.Infof("Successfully found attached GCE PD %q at device path %s.", volumeKey.Name, devicePath) - if ns.EnableDataCache && req.GetPublishContext()[common.ContexLocalSsdCacheSize] != "" { + if ns.EnableDataCache && req.GetPublishContext()[common.ContextDataCacheSize] != "" { if len(nodeId) == 0 { return nil, status.Error(codes.InvalidArgument, "NodeStageVolume Node ID must be provided") } @@ -336,6 +336,10 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage if err != nil { klog.Errorf("filepath.EvalSymlinks(%q) failed when trying to create volume group: %v", devicePath, err) } + configError := ValidateDataCacheConfig(req.GetPublishContext()[common.ContextDataCacheMode], req.GetPublishContext()[common.ContextDataCacheSize], ctx, nodeId) + if configError != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("Error validate configuration for Datacache: %v", err.Error())) + } devicePath, err = setupCaching(devFsPath, req, nodeId) if err != nil { return nil, status.Error(codes.DataLoss, fmt.Sprintf("Error setting up cache: %v", err.Error())) diff --git a/test/e2e/tests/setup_e2e_test.go b/test/e2e/tests/setup_e2e_test.go index 136b57cce..882a32b6e 100644 --- a/test/e2e/tests/setup_e2e_test.go +++ b/test/e2e/tests/setup_e2e_test.go @@ -33,6 +33,7 @@ import ( compute "google.golang.org/api/compute/v1" "k8s.io/klog/v2" "k8s.io/utils/strings/slices" + "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils" remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote" ) @@ -62,8 +63,6 @@ var ( kmsClient *cloudkms.KeyManagementClient ) -const localSSDCount int64 = 2 - func init() { klog.InitFlags(flag.CommandLine) } @@ -170,7 +169,7 @@ func NewTestContext(zone, minCpuPlatform, machineType string, instanceNumber str CloudtopHost: *cloudtopHost, EnableConfidentialCompute: *enableConfidentialCompute, ComputeService: computeService, - LocalSSDCount: localSSDCount, + LocalSSDCount: common.LocalSSDCountForDataCache, } if machineType == *hdMachineType { diff --git a/test/remote/client-wrappers.go b/test/remote/client-wrappers.go index efb0035f3..a7445c0a2 100644 --- a/test/remote/client-wrappers.go +++ b/test/remote/client-wrappers.go @@ -54,12 +54,7 @@ var ( const ( // Keys in the volume context. - contextForceAttach = "force-attach" - contextDataCacheSize = "data-cache-size" - contextDataCacheMode = "data-cache-mode" - - // Keys in the publish context - contexLocalSsdCacheSize = "local-ssd-cache-size" + contextForceAttach = "force-attach" defaultLocalSsdCacheSize = "200Gi" defaultDataCacheMode = common.DataCacheModeWriteThrough @@ -203,8 +198,8 @@ func (c *CsiClient) NodeStageBlockVolume(volId, stageDir string, setupDataCache func (c *CsiClient) NodeStageVolume(volId string, stageDir string, volumeCap *csipb.VolumeCapability, setupDataCache bool) error { publishContext := map[string]string{} if setupDataCache { - publishContext[contexLocalSsdCacheSize] = defaultLocalSsdCacheSize - publishContext[contextDataCacheMode] = defaultDataCacheMode + publishContext[common.ContextDataCacheSize] = defaultLocalSsdCacheSize + publishContext[common.ContextDataCacheMode] = defaultDataCacheMode } nodeStageReq := &csipb.NodeStageVolumeRequest{ VolumeId: volId, From 723acf504421b7e589baa087c9d6edc5442d734f Mon Sep 17 00:00:00 2001 From: Hung Nguyen Date: Tue, 25 Feb 2025 00:13:51 +0000 Subject: [PATCH 08/25] update test setup to avoid running Datacache setup on machines not supporting LSSDs --- cmd/gce-pd-csi-driver/main.go | 45 +++++++++++------------ pkg/common/constants.go | 2 +- pkg/gce-pd-csi-driver/cache.go | 65 +++++++++++++++++----------------- pkg/gce-pd-csi-driver/node.go | 2 +- test/e2e/utils/utils.go | 7 ++-- test/remote/instance.go | 4 +++ 6 files changed, 64 insertions(+), 61 deletions(-) diff --git a/cmd/gce-pd-csi-driver/main.go b/cmd/gce-pd-csi-driver/main.go index b16ba2473..895ed5fa1 100644 --- a/cmd/gce-pd-csi-driver/main.go +++ b/cmd/gce-pd-csi-driver/main.go @@ -71,7 +71,7 @@ var ( formatAndMountTimeout = flag.Duration("format-and-mount-timeout", 1*time.Minute, "The maximum duration of a format and mount operation before another such operation will be started. Used only if --serialize-format-and-mount") fallbackRequisiteZonesFlag = flag.String("fallback-requisite-zones", "", "Comma separated list of requisite zones that will be used if there are not sufficient zones present in requisite topologies when provisioning a disk") enableStoragePoolsFlag = flag.Bool("enable-storage-pools", false, "If set to true, the CSI Driver will allow volumes to be provisioned in Storage Pools") - enableDataCacheFlag = flag.Bool("enable-data-cache", false, "If set to true, the CSI Driver will allow volumes to be provisioned with data cache configuration") + enableDataCacheFlag = flag.Bool("enable-data-cache", false, "If set to true, the CSI Driver will allow volumes to be provisioned with Data Cache configuration") nodeName = flag.String("node-name", "", "The node this driver is running on") multiZoneVolumeHandleDiskTypesFlag = flag.String("multi-zone-volume-handle-disk-types", "", "Comma separated list of allowed disk types that can use the multi-zone volumeHandle. Used only if --multi-zone-volume-handle-enable") @@ -123,7 +123,7 @@ func handle() { if version == "" { klog.Fatalf("version must be set at compile time") } - klog.V(2).Infof("Driver vendor version %v", version) + klog.V(4).Infof("Driver vendor version %v", version) // Start tracing as soon as possible if *enableOtelTracing { @@ -235,14 +235,14 @@ func handle() { if *maxConcurrentFormatAndMount > 0 { nodeServer = nodeServer.WithSerializedFormatAndMount(*formatAndMountTimeout, *maxConcurrentFormatAndMount) } - if *enableDataCacheFlag { - if nodeName == nil || *nodeName == "" { - klog.Errorf("Data Cache enabled, but --node-name not passed") - } - if err := setupDataCache(ctx, *nodeName, nodeServer.MetadataService.GetName()); err != nil { - klog.Errorf("DataCache setup failed: %v", err) - } - go driver.StartWatcher(*nodeName) + } + + if *enableDataCacheFlag { + if nodeName == nil || *nodeName == "" { + klog.Errorf("Data Cache enabled, but --node-name not passed") + } + if err := setupDataCache(ctx, *nodeName); err != nil { + klog.Errorf("Data Cache setup failed: %v", err) } } @@ -351,7 +351,7 @@ func fetchLssdsForRaiding(lssdCount int) ([]string, error) { return nil, fmt.Errorf("Error listing LSSDs with empty mountpoint: %v", err) } - // We need to ensure the disks to be used for Datacache are both unRAIDed & not containing mountpoints for ephemeral storage already + // We need to ensure the disks to be used for Data Cache are both unRAIDed & not containing mountpoints for ephemeral storage already availableLssds := slices.Filter(nil, unRaidedLssds, func(e string) bool { return slices.Contains(LSSDsWithEmptyMountPoint, e) }) @@ -369,9 +369,9 @@ func fetchLssdsForRaiding(lssdCount int) ([]string, error) { func setupDataCache(ctx context.Context, nodeName string) error { isAlreadyRaided, err := driver.IsRaided() if err != nil { - klog.V(2).Infof("Errored while scanning for available LocalSSDs err:%v; continuing Raiding", err) + klog.V(4).Infof("Errored while scanning for available LocalSSDs err:%v; continuing Raiding", err) } else if isAlreadyRaided { - klog.V(2).Infof("Local SSDs are already RAIDed. Skipping Datacache setup.") + klog.V(4).Infof("Local SSDs are already RAIDed. Skipping Data Cache setup.") return nil } @@ -379,26 +379,21 @@ func setupDataCache(ctx context.Context, nodeName string) error { if nodeName != common.TestNode { var err error lssdCount, err = driver.GetDataCacheCountFromNodeLabel(ctx, nodeName) - if lssdCount == 0 { - klog.Infof("Datacache is not enabled on node %v", nodeName) - return nil - } if err != nil { return err } + if lssdCount == 0 { + klog.V(4).Infof("Data Cache is not enabled on node %v, so skipping caching setup", nodeName) + return nil + } } lssdNames, err := fetchLssdsForRaiding(lssdCount) if err != nil { - klog.Fatalf("Failed to get sufficient SSDs for Datacache's caching setup: %v", err) + klog.Fatalf("Failed to get sufficient SSDs for Data Cache's caching setup: %v", err) } - klog.V(2).Infof("Raiding local ssds to setup data cache: %v", lssdNames) + klog.V(4).Infof("Raiding local ssds to setup Data Cache: %v", lssdNames) if err := driver.RaidLocalSsds(lssdNames); err != nil { - return fmt.Errorf("Failed to Raid local SSDs, unable to setup data caching, got error %v", err) - } - - // Initializing data cache node (VG checks w/ raided lssd) - if err := driver.InitializeDataCacheNode(nodeId); err != nil { - return err + return fmt.Errorf("Failed to Raid local SSDs, unable to setup Data Cache, got error %v", err) } klog.V(4).Infof("LSSD caching is setup for the Data Cache enabled node %s", nodeName) diff --git a/pkg/common/constants.go b/pkg/common/constants.go index 5e7ec51af..104a50b47 100644 --- a/pkg/common/constants.go +++ b/pkg/common/constants.go @@ -48,7 +48,7 @@ const ( // Default LSSD count for datacache E2E tests LocalSSDCountForDataCache = 2 - // Node label for datacache + // Node label for Data Cache (only applicable to GKE nodes) NodeLabelPrefix = "cloud.google.com/%s" DataCacheLssdCountLabel = "gke-data-cache-disk" ) diff --git a/pkg/gce-pd-csi-driver/cache.go b/pkg/gce-pd-csi-driver/cache.go index c079a3360..3d3bc021d 100644 --- a/pkg/gce-pd-csi-driver/cache.go +++ b/pkg/gce-pd-csi-driver/cache.go @@ -16,11 +16,10 @@ import ( ) const ( - cacheSuffix = "csi-fast" - mainLvSuffix = "csi-main" - raidedLocalSsdName = "csi-driver-data-cache" - raidMode = "0" - initialRaidedLocalSsdPath = "/dev/md0" + cacheSuffix = "csi-fast" + mainLvSuffix = "csi-main" + raidedLocalSsdName = "csi-driver-data-cache" + raidMode = "0" ) func fetchRAIDedLocalSsdPath() (string, error) { @@ -30,12 +29,13 @@ func fetchRAIDedLocalSsdPath() (string, error) { } info, err := common.RunCommand("grep", []string{raidedLocalSsdName}, "mdadm", args...) if err != nil || len(info) == 0 { - return "", fmt.Errorf("Error getting RAIDed device path for Datacache %v, output:%v ===============", err, string(info)) + return "", fmt.Errorf("Error getting RAIDed device path for Data Cache %v, output:%v", err, string(info)) } infoString := strings.TrimSpace(string(info)) infoSlice := strings.Split(infoString, " ") - // We want to get the second element in the array, which is the path to the RAIDed device + // We want to get the second element in the array (sample: ARRAY /dev/md126 metadata=1.2 name=csi-driver-data-cache UUID=*), + // which is the path to the RAIDed device return infoSlice[1], nil } @@ -51,7 +51,7 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str volumeGroupName := getVolumeGroupName(nodeId) mainDevicePath := "/dev/" + volumeGroupName + "/" + getLvName(mainLvSuffix, volumeId) mainLvName := getLvName(mainLvSuffix, volumeId) - klog.V(2).Infof("Volume group available on node %v ", volumeGroupName) + klog.V(4).Infof("Volume group available on node %v ", volumeGroupName) vgExists := checkVgExists(volumeGroupName) if vgExists { // Clean up Volume Group before adding the PD @@ -82,9 +82,9 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str infoString = strings.ReplaceAll(infoString, "\"", "") infoSlice := strings.Split(strings.TrimSpace(infoString), " ") vgNameForPv := strings.TrimSpace(infoSlice[(len(infoSlice) - 1)]) - klog.V(2).Infof("============================== Physical volume is part of Volume group: %v ==============================", vgNameForPv) + klog.V(4).Infof("Physical volume is part of Volume group: %v", vgNameForPv) if vgNameForPv == volumeGroupName { - klog.V(2).Infof("============================== Physical Volume(PV) already exists in the Volume Group ==============================") + klog.V(4).Infof("Physical Volume(PV) already exists in the Volume Group") } else if vgNameForPv != "VG" && vgNameForPv != "" { info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgchange", []string{"-an", vgNameForPv}...) @@ -157,7 +157,7 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str cacheLvName := getLvName(cacheSuffix, volumeId) if isCached { // Validate that cache is setup for required size - klog.V(2).Infof("Assuming valid data cache size and mode, resizing cache is not supported") + klog.V(4).Infof("Assuming valid data cache size and mode, resizing cache is not supported") } else { fastCacheSize := req.GetPublishContext()[common.ContextDataCacheSize] chunkSize := "960" // Cannot use default chunk size(64KiB) as it errors on maxChunksAllowed. Unit - KiB @@ -207,8 +207,8 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str return mainDevicePath, nil } -func ValidateDataCacheConfig(dataCacheMode string, datacacheSize string, ctx context.Context, nodeName string) error { - if dataCacheMode != "" && datacacheSize != "" { +func ValidateDataCacheConfig(dataCacheMode string, dataCacheSize string, ctx context.Context, nodeName string) error { + if dataCacheMode != "" && dataCacheSize != "" { isAlreadyRaided, err := IsRaided() if err != nil { return fmt.Errorf("Local SSDs are not setup for caching; got error: %v", err) @@ -218,48 +218,50 @@ func ValidateDataCacheConfig(dataCacheMode string, datacacheSize string, ctx con } return nil } - klog.Infof("Data cache is not enabled for PVC") + klog.V(4).Infof("Data Cache is not enabled for PVC (data-cache-size: %v, data-cache-mode: %v). Please set both these parameters in StorageClass to enable caching", dataCacheSize, dataCacheMode) return nil } func GetDataCacheCountFromNodeLabel(ctx context.Context, nodeName string) (int, error) { - if nodeName == common.TestNode { - return common.LocalSSDCountForDataCache, nil - } cfg, err := rest.InClusterConfig() // We want to capture API errors with node label fetching, so return -1 // in those cases instead of 0. if err != nil { - return -1, err + return 0, err } kubeClient, err := kubernetes.NewForConfig(cfg) if err != nil { - return -1, err + return 0, err } node, err := kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) if err != nil { // We could retry, but this error will also crashloop the driver which may be as good a way to retry as any. - return -1, err + return 0, err } if val, found := node.GetLabels()[fmt.Sprintf(common.NodeLabelPrefix, common.DataCacheLssdCountLabel)]; found { dataCacheCount, err := strconv.Atoi(val) if err != nil { - return -1, fmt.Errorf("Error getting Datacache's LSSD count from node label: %v", err) + return 0, fmt.Errorf("Error getting Data Cache's LSSD count from node label: %v", err) } - klog.Infof("Number of local SSDs requested for Datacache: %v", dataCacheCount) + klog.V(4).Infof("Number of local SSDs requested for Data Cache: %v", dataCacheCount) return dataCacheCount, nil } - return 0, fmt.Errorf("Cannot get Datacache's LSSD count from node label") + // This will be returned for a non-Data-Cache node pool + return 0, nil } func FetchRaidedLssdCountForDatacache() (int, error) { + raidedPath, err := fetchRAIDedLocalSsdPath() + if err != nil { + return 0, err + } args := []string{ "--detail", - initialRaidedLocalSsdPath, + raidedPath, } info, err := common.RunCommand("grep", []string{"Raid Devices"}, "mdadm", args...) if err != nil { - return 0, fmt.Errorf("Error getting RAIDed devices for Datacache") + return 0, fmt.Errorf("Error getting RAIDed devices for Data Cache") } if len(info) != 0 { raidedDeviceInfo := strings.Split(strings.TrimSpace(string(info)), ":") @@ -294,7 +296,7 @@ func FetchRaidedLssds() ([]string, error) { } } - klog.V(2).Infof("Raided NVME list %v", raidedLssdList) + klog.V(4).Infof("Raided NVME list %v", raidedLssdList) return raidedLssdList, nil } @@ -309,7 +311,7 @@ func FetchAllLssds() ([]string, error) { infoList := strings.Split(strings.TrimSpace(string(info)), "\n") re, err := regexp.Compile("nvme_card([0-9]+)?$") if err != nil { - klog.V(2).ErrorS(err, "Errored while compiling to check PD or LSSD") + klog.V(4).ErrorS(err, "Errored while compiling to check PD or LSSD") } for _, ssd := range infoList { ssd = strings.TrimSpace(ssd) @@ -322,7 +324,7 @@ func FetchAllLssds() ([]string, error) { } } - klog.V(2).Infof("NVME list %v", diskList) + klog.V(4).Infof("NVME list %v", diskList) return diskList, nil } @@ -358,6 +360,7 @@ func cleanupCache(volumeId string, nodeId string) error { // If volume group doesn't exist then there's nothing to uncache return nil } + reduceVolumeGroup(volumeGroupName, true) mainLvName := getLvName(mainLvSuffix, volumeId) args := []string{ "-an", @@ -404,7 +407,7 @@ func createVg(volumeGroupName string, raidedLocalSsds string) error { if err != nil { return fmt.Errorf("Volume group creation failed %w: %s", err, info) } - klog.Infof("Volume group creation succeeded for %v", volumeGroupName) + klog.V(4).Infof("Volume group creation succeeded for %v", volumeGroupName) args = []string{} info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgscan", args...) @@ -431,8 +434,6 @@ func reduceVolumeGroup(volumeGroupName string, force bool) { func RaidLocalSsds(availableLssds []string) error { args := []string{ "--create", - initialRaidedLocalSsdPath, - "--name", raidedLocalSsdName, "-l" + raidMode, // Force RAIDing as sometime it might fail for caution if there is just 1 LSSD present as 1 LSSD need not be RAIDed @@ -448,7 +449,7 @@ func RaidLocalSsds(availableLssds []string) error { // Validate if Raided successfully isAlreadyRaided, err := IsRaided() if err != nil { - klog.V(2).Infof("Errored while scanning for available raided LocalSSDs err:%v=", err) + klog.V(4).Infof("Errored while scanning for available raided LocalSSDs err:%v=", err) } if !isAlreadyRaided { return fmt.Errorf("failed raiding, raided device not found on scanning") diff --git a/pkg/gce-pd-csi-driver/node.go b/pkg/gce-pd-csi-driver/node.go index 97442c0c8..31bec5b85 100644 --- a/pkg/gce-pd-csi-driver/node.go +++ b/pkg/gce-pd-csi-driver/node.go @@ -338,7 +338,7 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage } configError := ValidateDataCacheConfig(req.GetPublishContext()[common.ContextDataCacheMode], req.GetPublishContext()[common.ContextDataCacheSize], ctx, nodeId) if configError != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("Error validate configuration for Datacache: %v", err.Error())) + return nil, status.Error(codes.Internal, fmt.Sprintf("Error validate configuration for Data Cache: %v", err.Error())) } devicePath, err = setupCaching(devFsPath, req, nodeId) if err != nil { diff --git a/test/e2e/utils/utils.go b/test/e2e/utils/utils.go index 774c07753..f558a27af 100644 --- a/test/e2e/utils/utils.go +++ b/test/e2e/utils/utils.go @@ -67,8 +67,11 @@ func GCEClientAndDriverSetup(instance *remote.InstanceInfo, driverConfig DriverC "--use-instance-api-to-poll-attachment-disk-types=pd-ssd", "--use-instance-api-to-list-volumes-published-nodes", fmt.Sprintf("--fallback-requisite-zones=%s", strings.Join(driverConfig.Zones, ",")), - "--enable-data-cache", - fmt.Sprintf("--node-name=%s", utilcommon.TestNode), + } + + if instance.GetLocalSSD() > 0 { + extra_flags = append(extra_flags, "--enable-data-cache") + extra_flags = append(extra_flags, fmt.Sprintf("--node-name=%s", utilcommon.TestNode)) } extra_flags = append(extra_flags, fmt.Sprintf("--compute-endpoint=%s", driverConfig.ComputeEndpoint)) extra_flags = append(extra_flags, driverConfig.ExtraFlags...) diff --git a/test/remote/instance.go b/test/remote/instance.go index c9c3dd6d5..554e7612e 100644 --- a/test/remote/instance.go +++ b/test/remote/instance.go @@ -80,6 +80,10 @@ func (i *InstanceInfo) GetNodeID() string { return common.CreateNodeID(i.cfg.Project, i.cfg.Zone, i.cfg.Name) } +func (i *InstanceInfo) GetLocalSSD() int64 { + return i.cfg.LocalSSDCount +} + func machineTypeMismatch(curInst *compute.Instance, newInst *compute.Instance) bool { if !strings.Contains(curInst.MachineType, newInst.MachineType) { klog.Infof("Machine type mismatch") From e1d8632df16e78a48bd30f5b83af9a3f5758939c Mon Sep 17 00:00:00 2001 From: Hung Nguyen Date: Wed, 26 Feb 2025 22:57:24 +0000 Subject: [PATCH 09/25] update validation logic for Data Cache to distinguish user and non-user errors when RAIDing fails --- cmd/gce-pd-csi-driver/main.go | 11 ++++++++++- pkg/gce-pd-csi-driver/cache.go | 5 ++--- pkg/gce-pd-csi-driver/gce-pd-driver.go | 15 ++++++++------- pkg/gce-pd-csi-driver/node.go | 24 +++++++++++++++--------- pkg/gce-pd-csi-driver/node_test.go | 6 +++--- 5 files changed, 38 insertions(+), 23 deletions(-) diff --git a/cmd/gce-pd-csi-driver/main.go b/cmd/gce-pd-csi-driver/main.go index 895ed5fa1..81d308aab 100644 --- a/cmd/gce-pd-csi-driver/main.go +++ b/cmd/gce-pd-csi-driver/main.go @@ -229,7 +229,8 @@ func handle() { klog.Fatalf("Failed to set up metadata service: %v", err.Error()) } nsArgs := driver.NodeServerArgs{ - EnableDataCache: *enableDataCacheFlag, + EnableDataCache: *enableDataCacheFlag, + DataCacheEnabledNodePool: isDataCacheEnabledNodePool(ctx, *nodeName), } nodeServer = driver.NewNodeServer(gceDriver, mounter, deviceUtils, meta, statter, nsArgs) if *maxConcurrentFormatAndMount > 0 { @@ -325,6 +326,14 @@ func urlFlag(target **url.URL, name string, usage string) { }) } +func isDataCacheEnabledNodePool(ctx context.Context, nodeName string) bool { + dataCacheLSSDCount, err := driver.GetDataCacheCountFromNodeLabel(ctx, nodeName) + if err != nil || dataCacheLSSDCount == 0 { + return false + } + return true +} + func fetchLssdsForRaiding(lssdCount int) ([]string, error) { allLssds, err := driver.FetchAllLssds() if err != nil { diff --git a/pkg/gce-pd-csi-driver/cache.go b/pkg/gce-pd-csi-driver/cache.go index 3d3bc021d..c23f5d258 100644 --- a/pkg/gce-pd-csi-driver/cache.go +++ b/pkg/gce-pd-csi-driver/cache.go @@ -207,7 +207,7 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str return mainDevicePath, nil } -func ValidateDataCacheConfig(dataCacheMode string, dataCacheSize string, ctx context.Context, nodeName string) error { +func ValidateDataCacheConfig(dataCacheMode string, dataCacheSize string, ctx context.Context) error { if dataCacheMode != "" && dataCacheSize != "" { isAlreadyRaided, err := IsRaided() if err != nil { @@ -218,8 +218,7 @@ func ValidateDataCacheConfig(dataCacheMode string, dataCacheSize string, ctx con } return nil } - klog.V(4).Infof("Data Cache is not enabled for PVC (data-cache-size: %v, data-cache-mode: %v). Please set both these parameters in StorageClass to enable caching", dataCacheSize, dataCacheMode) - return nil + return fmt.Errorf("Data Cache is not enabled for PVC (data-cache-size: %v, data-cache-mode: %v). Please set both parameters in StorageClass to enable caching", dataCacheSize, dataCacheMode) } func GetDataCacheCountFromNodeLabel(ctx context.Context, nodeName string) (int, error) { diff --git a/pkg/gce-pd-csi-driver/gce-pd-driver.go b/pkg/gce-pd-csi-driver/gce-pd-driver.go index 28f6a56c9..e1bb5706c 100644 --- a/pkg/gce-pd-csi-driver/gce-pd-driver.go +++ b/pkg/gce-pd-csi-driver/gce-pd-driver.go @@ -145,13 +145,14 @@ func NewIdentityServer(gceDriver *GCEDriver) *GCEIdentityServer { func NewNodeServer(gceDriver *GCEDriver, mounter *mount.SafeFormatAndMount, deviceUtils deviceutils.DeviceUtils, meta metadataservice.MetadataService, statter mountmanager.Statter, args NodeServerArgs) *GCENodeServer { return &GCENodeServer{ - Driver: gceDriver, - Mounter: mounter, - DeviceUtils: deviceUtils, - MetadataService: meta, - volumeLocks: common.NewVolumeLocks(), - VolumeStatter: statter, - EnableDataCache: args.EnableDataCache, + Driver: gceDriver, + Mounter: mounter, + DeviceUtils: deviceUtils, + MetadataService: meta, + volumeLocks: common.NewVolumeLocks(), + VolumeStatter: statter, + EnableDataCache: args.EnableDataCache, + DataCacheEnabledNodePool: args.DataCacheEnabledNodePool, } } diff --git a/pkg/gce-pd-csi-driver/node.go b/pkg/gce-pd-csi-driver/node.go index 31bec5b85..d6cca666b 100644 --- a/pkg/gce-pd-csi-driver/node.go +++ b/pkg/gce-pd-csi-driver/node.go @@ -42,12 +42,13 @@ import ( ) type GCENodeServer struct { - Driver *GCEDriver - Mounter *mount.SafeFormatAndMount - DeviceUtils deviceutils.DeviceUtils - VolumeStatter mountmanager.Statter - MetadataService metadataservice.MetadataService - EnableDataCache bool + Driver *GCEDriver + Mounter *mount.SafeFormatAndMount + DeviceUtils deviceutils.DeviceUtils + VolumeStatter mountmanager.Statter + MetadataService metadataservice.MetadataService + EnableDataCache bool + DataCacheEnabledNodePool bool // A map storing all volumes with ongoing operations so that additional operations // for that same volume (as defined by VolumeID) return an Aborted error @@ -69,6 +70,8 @@ type GCENodeServer struct { type NodeServerArgs struct { EnableDataCache bool + + DataCacheEnabledNodePool bool } var _ csi.NodeServer = &GCENodeServer{} @@ -328,7 +331,7 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage klog.Infof("Successfully found attached GCE PD %q at device path %s.", volumeKey.Name, devicePath) - if ns.EnableDataCache && req.GetPublishContext()[common.ContextDataCacheSize] != "" { + if ns.EnableDataCache && (req.GetPublishContext()[common.ContextDataCacheSize] != "" || req.GetPublishContext()[common.ContextDataCacheMode] != "") { if len(nodeId) == 0 { return nil, status.Error(codes.InvalidArgument, "NodeStageVolume Node ID must be provided") } @@ -336,9 +339,12 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage if err != nil { klog.Errorf("filepath.EvalSymlinks(%q) failed when trying to create volume group: %v", devicePath, err) } - configError := ValidateDataCacheConfig(req.GetPublishContext()[common.ContextDataCacheMode], req.GetPublishContext()[common.ContextDataCacheSize], ctx, nodeId) + configError := ValidateDataCacheConfig(req.GetPublishContext()[common.ContextDataCacheMode], req.GetPublishContext()[common.ContextDataCacheSize], ctx) if configError != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("Error validate configuration for Data Cache: %v", err.Error())) + if ns.DataCacheEnabledNodePool { + return nil, status.Error(codes.DataLoss, fmt.Sprintf("Error validate configuration for Data Cache: %v", configError.Error())) + } + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Data Cache PVC is requested for an incompatible node pool: %v", configError.Error())) } devicePath, err = setupCaching(devFsPath, req, nodeId) if err != nil { diff --git a/pkg/gce-pd-csi-driver/node_test.go b/pkg/gce-pd-csi-driver/node_test.go index 80e5c28ec..337cd3a8a 100644 --- a/pkg/gce-pd-csi-driver/node_test.go +++ b/pkg/gce-pd-csi-driver/node_test.go @@ -52,7 +52,7 @@ func getTestGCEDriverWithCustomMounter(t *testing.T, mounter *mount.SafeFormatAn func getCustomTestGCEDriver(t *testing.T, mounter *mount.SafeFormatAndMount, deviceUtils deviceutils.DeviceUtils, metaService metadataservice.MetadataService) *GCEDriver { gceDriver := GetGCEDriver() enableDataCache := false - nodeServer := NewNodeServer(gceDriver, mounter, deviceUtils, metaService, mountmanager.NewFakeStatter(mounter), NodeServerArgs{enableDataCache}) + nodeServer := NewNodeServer(gceDriver, mounter, deviceUtils, metaService, mountmanager.NewFakeStatter(mounter), NodeServerArgs{enableDataCache, false /*dataCacheEnableNodePool */}) err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, nil, nil, nodeServer) if err != nil { t.Fatalf("Failed to setup GCE Driver: %v", err) @@ -63,7 +63,7 @@ func getCustomTestGCEDriver(t *testing.T, mounter *mount.SafeFormatAndMount, dev func getTestBlockingMountGCEDriver(t *testing.T, readyToExecute chan chan struct{}) *GCEDriver { gceDriver := GetGCEDriver() mounter := mountmanager.NewFakeSafeBlockingMounter(readyToExecute) - nodeServer := NewNodeServer(gceDriver, mounter, deviceutils.NewFakeDeviceUtils(false), metadataservice.NewFakeService(), mountmanager.NewFakeStatter(mounter), NodeServerArgs{true}) + nodeServer := NewNodeServer(gceDriver, mounter, deviceutils.NewFakeDeviceUtils(false), metadataservice.NewFakeService(), mountmanager.NewFakeStatter(mounter), NodeServerArgs{true, false /*dataCacheEnableNodePool */}) err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, nil, nil, nodeServer) if err != nil { t.Fatalf("Failed to setup GCE Driver: %v", err) @@ -75,7 +75,7 @@ func getTestBlockingFormatAndMountGCEDriver(t *testing.T, readyToExecute chan ch gceDriver := GetGCEDriver() enableDataCache := true mounter := mountmanager.NewFakeSafeBlockingMounter(readyToExecute) - nodeServer := NewNodeServer(gceDriver, mounter, deviceutils.NewFakeDeviceUtils(false), metadataservice.NewFakeService(), mountmanager.NewFakeStatter(mounter), NodeServerArgs{enableDataCache}).WithSerializedFormatAndMount(5*time.Second, 1) + nodeServer := NewNodeServer(gceDriver, mounter, deviceutils.NewFakeDeviceUtils(false), metadataservice.NewFakeService(), mountmanager.NewFakeStatter(mounter), NodeServerArgs{enableDataCache, false /*dataCacheEnableNodePool */}).WithSerializedFormatAndMount(5*time.Second, 1) err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, nil, nil, nodeServer) if err != nil { From 63d56ec398f742bde3f8ec2a96c3c2837268bc38 Mon Sep 17 00:00:00 2001 From: Sunny Date: Thu, 27 Feb 2025 09:08:56 -0800 Subject: [PATCH 10/25] Update error message for end users --- pkg/gce-pd-csi-driver/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/gce-pd-csi-driver/node.go b/pkg/gce-pd-csi-driver/node.go index d6cca666b..534a0ebf7 100644 --- a/pkg/gce-pd-csi-driver/node.go +++ b/pkg/gce-pd-csi-driver/node.go @@ -344,7 +344,7 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage if ns.DataCacheEnabledNodePool { return nil, status.Error(codes.DataLoss, fmt.Sprintf("Error validate configuration for Data Cache: %v", configError.Error())) } - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Data Cache PVC is requested for an incompatible node pool: %v", configError.Error())) + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("The Data Cache PVC is scheduled on an incompatible node pool. Please select a node pool with data cache configured: %v", configError.Error())) } devicePath, err = setupCaching(devFsPath, req, nodeId) if err != nil { From 03d4a849c1d8d0e9ab16c5b081db11a94b090d15 Mon Sep 17 00:00:00 2001 From: Sneha Aradhey Date: Tue, 25 Feb 2025 20:58:35 +0000 Subject: [PATCH 11/25] Fix chunksize bug for large cache size in data cache --- .../base/controller/controller.yaml | 1 + pkg/gce-pd-csi-driver/cache.go | 66 +++++++++++++------ 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/deploy/kubernetes/base/controller/controller.yaml b/deploy/kubernetes/base/controller/controller.yaml index e7e5db9d1..c038bc583 100644 --- a/deploy/kubernetes/base/controller/controller.yaml +++ b/deploy/kubernetes/base/controller/controller.yaml @@ -139,6 +139,7 @@ spec: args: - "--v=5" - "--endpoint=unix:/csi/csi.sock" + - --enable-controller-data-cache env: - name: GOOGLE_APPLICATION_CREDENTIALS value: "/etc/cloud-sa/cloud-sa.json" diff --git a/pkg/gce-pd-csi-driver/cache.go b/pkg/gce-pd-csi-driver/cache.go index c23f5d258..123e36de2 100644 --- a/pkg/gce-pd-csi-driver/cache.go +++ b/pkg/gce-pd-csi-driver/cache.go @@ -3,6 +3,7 @@ package gceGCEDriver import ( "context" "fmt" + "math" "regexp" "strconv" "strings" @@ -16,10 +17,13 @@ import ( ) const ( - cacheSuffix = "csi-fast" - mainLvSuffix = "csi-main" - raidedLocalSsdName = "csi-driver-data-cache" - raidMode = "0" + cacheSuffix = "csi-fast" + mainLvSuffix = "csi-main" + raidedLocalSsdName = "csi-driver-data-cache" + raidMode = "0" + maxAllowedChunks int64 = 1000000 // This is the max allowed chunks for LVM + GiB int64 = 1024 * 1024 * 1024 + KiB int64 = 1024 ) func fetchRAIDedLocalSsdPath() (string, error) { @@ -159,21 +163,30 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str // Validate that cache is setup for required size klog.V(4).Infof("Assuming valid data cache size and mode, resizing cache is not supported") } else { - fastCacheSize := req.GetPublishContext()[common.ContextDataCacheSize] - chunkSize := "960" // Cannot use default chunk size(64KiB) as it errors on maxChunksAllowed. Unit - KiB - args = []string{ - "--yes", - "-n", - cacheLvName, - "-L", - // ConvertGiStringToInt64 converts the input size to GiB so default to "g" for cache size - LVM g|G is GiB. - fastCacheSize + "g", - volumeGroupName, - raidedLocalSsdPath, - } - info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvcreate", args...) + cacheSize := req.GetPublishContext()[common.ContextDataCacheSize] + chunkSize, err := fetchChunkSize(cacheSize) if err != nil { - return mainDevicePath, fmt.Errorf("Errored while creating cache %w: %s", err, info) + klog.Errorf("Errored to fetch cache size, verify the data-cache-size is valid: got %v, error: %q", cacheSize, err) + return mainDevicePath, err + } + // Check if LV exists + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvs", args...) + lvExists := strings.Contains(string(info), cacheLvName) + if !lvExists { + args = []string{ + "--yes", + "-n", + cacheLvName, + "-L", + // ConvertGiStringToInt64 converts the input size to GiB so default to "g" for cache size - LVM g|G is GiB. + cacheSize + "g", + volumeGroupName, + raidedLocalSsdPath, + } + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvcreate", args...) + if err != nil { + return mainDevicePath, fmt.Errorf("Errored while creating cache %w: %s", err, info) + } } // Once caching is setup, link the PD to cache @@ -188,7 +201,7 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str req.GetPublishContext()[common.ContextDataCacheMode], volumeGroupName + "/" + mainLvName, "--chunksize", - string(chunkSize), + chunkSize, "--force", "-y", } @@ -497,6 +510,21 @@ func isCachingSetup(mainLvName string) (error, bool) { return nil, false } +func fetchChunkSize(cacheSize string) (string, error) { + var chunkSize float64 + var maxChunkSize int64 = 1 * GiB // Max allowed chunk size as per LVM documentation + var minChunkSize int64 = 320 * KiB // This is randomly selected, we need a multiple of 32KiB, the default size would be too small for caching https://man7.org/linux/man-pages/man8/lvcreate.8.html (--chunksize) + cacheSizeInt, err := common.ConvertGiStringToInt64(cacheSize) + if err != nil { + return "0", err + } + // Chunksize should be divisible by 32Kib so we need (chunksize/32*1024)*32*1024 + chunkSize = float64(cacheSizeInt) / float64(maxAllowedChunks) + chunkSize = math.Ceil(chunkSize/float64(32*KiB)) * float64(32*KiB) + chunkSize = math.Min(math.Max(chunkSize, float64(minChunkSize)), float64(maxChunkSize)) + return strconv.FormatInt(int64(chunkSize), 10), nil +} + func fetchChunkSizeKiB(cacheSize string) (string, error) { var chunkSize float64 From bc0a5d3da261fa28e94e809253b2c0ddf7d77634 Mon Sep 17 00:00:00 2001 From: Sneha Aradhey Date: Tue, 25 Feb 2025 21:48:50 +0000 Subject: [PATCH 12/25] skip cache clean up for non-data cache PVCs --- .../kubernetes/base/controller/controller.yaml | 2 +- pkg/gce-pd-csi-driver/cache.go | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/deploy/kubernetes/base/controller/controller.yaml b/deploy/kubernetes/base/controller/controller.yaml index c038bc583..5f8713af5 100644 --- a/deploy/kubernetes/base/controller/controller.yaml +++ b/deploy/kubernetes/base/controller/controller.yaml @@ -139,7 +139,7 @@ spec: args: - "--v=5" - "--endpoint=unix:/csi/csi.sock" - - --enable-controller-data-cache + - --enable-data-cache env: - name: GOOGLE_APPLICATION_CREDENTIALS value: "/etc/cloud-sa/cloud-sa.json" diff --git a/pkg/gce-pd-csi-driver/cache.go b/pkg/gce-pd-csi-driver/cache.go index 123e36de2..932d61906 100644 --- a/pkg/gce-pd-csi-driver/cache.go +++ b/pkg/gce-pd-csi-driver/cache.go @@ -369,11 +369,17 @@ func cleanupCache(volumeId string, nodeId string) error { volumeGroupName := getVolumeGroupName(nodeId) if !checkVgExists(volumeGroupName) { + klog.V(4).Infof("Volume group %s not found, no cache clean up needed", volumeGroupName) // If volume group doesn't exist then there's nothing to uncache return nil } reduceVolumeGroup(volumeGroupName, true) mainLvName := getLvName(mainLvSuffix, volumeId) + if !checkLvExists(mainLvName) { + klog.V(4).Infof("Logical volume %s not found, assuming caching wasn't setup for the PVC %s or is cleaned up", mainLvName, volumeId) + // If logical volume doesn't exist then there's nothing to uncache + return nil + } args := []string{ "-an", "/dev/" + volumeGroupName + "/" + mainLvName, @@ -394,6 +400,17 @@ func cleanupCache(volumeId string, nodeId string) error { return nil } +func checkLvExists(lvName string) bool { + args := []string{} + info, err := common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "lvscan", args...) + if err != nil { + klog.Errorf("Errored while checking if logical volume exists for %s %v: %s", lvName, err, info) + return false + } + // Check if the required logical volume already exists + return strings.Contains(string(info), lvName) +} + func getVolumeGroupName(nodePath string) string { nodeSlice := strings.Split(nodePath, "/") nodeId := nodeSlice[len(nodeSlice)-1] From 5233508e8a3a8423fde874af4dc3480b3d29101e Mon Sep 17 00:00:00 2001 From: Sneha Aradhey Date: Wed, 26 Feb 2025 19:05:01 +0000 Subject: [PATCH 13/25] Add unit tests --- pkg/common/parameters.go | 6 ++- pkg/common/utils.go | 7 ++++ pkg/common/utils_test.go | 36 ++++++++++++++++++ pkg/gce-pd-csi-driver/cache.go | 41 ++++++++++++--------- pkg/gce-pd-csi-driver/cache_test.go | 57 +++++++++++++++++++++++++++++ 5 files changed, 127 insertions(+), 20 deletions(-) create mode 100644 pkg/gce-pd-csi-driver/cache_test.go diff --git a/pkg/common/parameters.go b/pkg/common/parameters.go index 5bd8bd6a8..27eae3665 100644 --- a/pkg/common/parameters.go +++ b/pkg/common/parameters.go @@ -260,19 +260,21 @@ func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string] if !enableDataCache { return p, d, fmt.Errorf("data caching enabled: %v; parameters contains invalid option %q", enableDataCache, ParameterKeyDataCacheSize) } - // TODO: need to parse or validate the string paramDataCacheSize, err := ConvertGiStringToInt64(v) if err != nil { return p, d, fmt.Errorf("parameters contain invalid dataCacheSize parameter: %w", err) } + if err := ValidateNonNegativeInt(paramDataCacheSize); err != nil { + return p, d, fmt.Errorf("parameters contains invalid option: %s: %w", ParameterKeyDataCacheSize, err) + } d.DataCacheSize = strconv.FormatInt(paramDataCacheSize, 10) case ParameterKeyDataCacheMode: if !enableDataCache { return p, d, fmt.Errorf("data caching enabled %v; parameters contains invalid option %q", enableDataCache, ParameterKeyDataCacheSize) } if err := ValidateDataCacheMode(v); err != nil { - return p, d, fmt.Errorf("parameters contains invalid option: %w", err) + return p, d, fmt.Errorf("parameters contains invalid option: %s: %w", ParameterKeyDataCacheMode, err) } d.DataCacheMode = v case ParameterKeyResourceTags: diff --git a/pkg/common/utils.go b/pkg/common/utils.go index 73fa32243..8a1229d02 100644 --- a/pkg/common/utils.go +++ b/pkg/common/utils.go @@ -713,6 +713,13 @@ func ValidateDataCacheMode(s string) error { return fmt.Errorf("invalid data-cache-mode %s. Only \"writeback\" and \"writethrough\" is a valid input", s) } +func ValidateNonNegativeInt(n int64) error { + if n <= 0 { + return fmt.Errorf("Input should be set to > 0, got %d", n) + } + return nil +} + // NewLimiter returns a token bucket based request rate limiter after initializing // the passed values for limit, burst (or token bucket) size. If opted for emptyBucket // all initial tokens are reserved for the first burst. diff --git a/pkg/common/utils_test.go b/pkg/common/utils_test.go index e3323905d..dbca3d789 100644 --- a/pkg/common/utils_test.go +++ b/pkg/common/utils_test.go @@ -1823,6 +1823,42 @@ func TestValidateDataCacheMode(t *testing.T) { } +func TestValidateNonNegativeInt(t *testing.T) { + testCases := []struct { + name string + cacheSize int64 + expectError bool + }{ + { + name: "valid input - positive cache size", + cacheSize: 100000, + }, + { + name: "invalid input - cachesize 0", + cacheSize: 0, + expectError: true, + }, + { + name: "invalid input - negative cache size", + cacheSize: -100, + expectError: true, + }, + } + + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + err := ValidateNonNegativeInt(tc.cacheSize) + if err != nil && !tc.expectError { + t.Errorf("Got error %v validate data cache mode %d; expect no error", err, tc.cacheSize) + } + + if err == nil && tc.expectError { + t.Errorf("Got no error validate data cache mode %d; expect an error", tc.cacheSize) + } + } + +} + func TestParseZoneFromURI(t *testing.T) { testcases := []struct { name string diff --git a/pkg/gce-pd-csi-driver/cache.go b/pkg/gce-pd-csi-driver/cache.go index 932d61906..7a8d627ac 100644 --- a/pkg/gce-pd-csi-driver/cache.go +++ b/pkg/gce-pd-csi-driver/cache.go @@ -17,13 +17,18 @@ import ( ) const ( - cacheSuffix = "csi-fast" - mainLvSuffix = "csi-main" - raidedLocalSsdName = "csi-driver-data-cache" - raidMode = "0" - maxAllowedChunks int64 = 1000000 // This is the max allowed chunks for LVM - GiB int64 = 1024 * 1024 * 1024 - KiB int64 = 1024 + cacheSuffix = "csi-fast" + mainLvSuffix = "csi-main" + raidedLocalSsdName = "csi-driver-data-cache" + raidMode = "0" + maxAllowedChunks int64 = 1000000 // This is the max allowed chunks for LVM + GiB float64 = 1024 * 1024 * 1024 + KiB float64 = 1024 +) + +var ( + maxChunkSize float64 = 1 * GiB // Max allowed chunk size as per LVM documentation + minChunkSize float64 = 160 * KiB // This is randomly selected, we need a multiple of 32KiB, the default size would be too small for caching https://man7.org/linux/man-pages/man8/lvcreate.8.html (--chunksize) ) func fetchRAIDedLocalSsdPath() (string, error) { @@ -88,7 +93,7 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str vgNameForPv := strings.TrimSpace(infoSlice[(len(infoSlice) - 1)]) klog.V(4).Infof("Physical volume is part of Volume group: %v", vgNameForPv) if vgNameForPv == volumeGroupName { - klog.V(4).Infof("Physical Volume(PV) already exists in the Volume Group") + klog.V(4).Infof("Physical Volume(PV) already exists in the Volume Group %v", volumeGroupName) } else if vgNameForPv != "VG" && vgNameForPv != "" { info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgchange", []string{"-an", vgNameForPv}...) @@ -164,7 +169,7 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str klog.V(4).Infof("Assuming valid data cache size and mode, resizing cache is not supported") } else { cacheSize := req.GetPublishContext()[common.ContextDataCacheSize] - chunkSize, err := fetchChunkSize(cacheSize) + chunkSize, err := fetchChunkSizeKiB(cacheSize) if err != nil { klog.Errorf("Errored to fetch cache size, verify the data-cache-size is valid: got %v, error: %q", cacheSize, err) return mainDevicePath, err @@ -201,7 +206,7 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str req.GetPublishContext()[common.ContextDataCacheMode], volumeGroupName + "/" + mainLvName, "--chunksize", - chunkSize, + chunkSize, // default unit is KiB "--force", "-y", } @@ -402,7 +407,7 @@ func cleanupCache(volumeId string, nodeId string) error { func checkLvExists(lvName string) bool { args := []string{} - info, err := common.RunCommand("" /* pipedCmd */, "" /* pipedCmdArg */, "lvscan", args...) + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvscan", args...) if err != nil { klog.Errorf("Errored while checking if logical volume exists for %s %v: %s", lvName, err, info) return false @@ -527,19 +532,19 @@ func isCachingSetup(mainLvName string) (error, bool) { return nil, false } -func fetchChunkSize(cacheSize string) (string, error) { +func fetchChunkSizeKiB(cacheSize string) (string, error) { var chunkSize float64 - var maxChunkSize int64 = 1 * GiB // Max allowed chunk size as per LVM documentation - var minChunkSize int64 = 320 * KiB // This is randomly selected, we need a multiple of 32KiB, the default size would be too small for caching https://man7.org/linux/man-pages/man8/lvcreate.8.html (--chunksize) + cacheSizeInt, err := common.ConvertGiStringToInt64(cacheSize) if err != nil { return "0", err } // Chunksize should be divisible by 32Kib so we need (chunksize/32*1024)*32*1024 - chunkSize = float64(cacheSizeInt) / float64(maxAllowedChunks) - chunkSize = math.Ceil(chunkSize/float64(32*KiB)) * float64(32*KiB) - chunkSize = math.Min(math.Max(chunkSize, float64(minChunkSize)), float64(maxChunkSize)) - return strconv.FormatInt(int64(chunkSize), 10), nil + chunkSize = (float64(cacheSizeInt) * GiB) / float64(maxAllowedChunks) + chunkSize = math.Round(chunkSize/(32*KiB)) * (32 * KiB) + chunkSize = math.Min(math.Max(chunkSize, minChunkSize), maxChunkSize) / KiB + // default chunk size unit KiB + return strconv.FormatInt(int64(chunkSize), 10) + "KiB", nil } func fetchChunkSizeKiB(cacheSize string) (string, error) { diff --git a/pkg/gce-pd-csi-driver/cache_test.go b/pkg/gce-pd-csi-driver/cache_test.go new file mode 100644 index 000000000..918f26b0c --- /dev/null +++ b/pkg/gce-pd-csi-driver/cache_test.go @@ -0,0 +1,57 @@ +package gceGCEDriver + +import ( + "testing" +) + +func TestFetchChunkSizeKiB(t *testing.T) { + testCases := []struct { + name string + cacheSize string + expChunkSize string + expErr bool + }{ + { + name: "chunk size is in the allowed range", + cacheSize: "500Gi", + expChunkSize: "512KiB", //range defined in fetchChunkSizeKiB + }, + { + name: "chunk size is set to the range ceil", + cacheSize: "30000000Gi", + expChunkSize: "1048576KiB", //range defined in fetchChunkSizeKiB - max 1GiB + }, + { + name: "chunk size is set to the allowed range floor", + cacheSize: "10Gi", + expChunkSize: "160KiB", //range defined in fetchChunkSizeKiB - min 160 KiB + }, + { + name: "cacheSize set to KiB also sets the chunk size to range floor", + cacheSize: "100Ki", + expChunkSize: "160KiB", //range defined in fetchChunkSizeKiB - min 160 KiB + }, + { + name: "invalid cacheSize", + cacheSize: "fdfsdKi", + expChunkSize: "160KiB", //range defined in fetchChunkSizeKiB - min 160 KiB + expErr: true, + }, + // cacheSize is validated in storage class parameter so assuming invalid cacheSize (like negative, 0) would not be passed to the function + } + + for _, tc := range testCases { + chunkSize, err := fetchChunkSizeKiB(tc.cacheSize) + if err != nil { + if !tc.expErr { + t.Errorf("Errored %s", err) + } + continue + } + if chunkSize != tc.expChunkSize { + t.Errorf("Got %s want %s", chunkSize, tc.expChunkSize) + } + + } + +} From f86f20de4933f0e107265a80a237589fdbd1dba2 Mon Sep 17 00:00:00 2001 From: halimsam Date: Fri, 28 Feb 2025 18:57:53 +0000 Subject: [PATCH 14/25] Only run data cache watcher if a data cache nodepool exist & adding check for VG cleanup --- cmd/gce-pd-csi-driver/main.go | 26 +++++++++++++++----------- pkg/gce-pd-csi-driver/cache.go | 4 ++++ 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/cmd/gce-pd-csi-driver/main.go b/cmd/gce-pd-csi-driver/main.go index 81d308aab..ac5e8acd2 100644 --- a/cmd/gce-pd-csi-driver/main.go +++ b/cmd/gce-pd-csi-driver/main.go @@ -236,14 +236,16 @@ func handle() { if *maxConcurrentFormatAndMount > 0 { nodeServer = nodeServer.WithSerializedFormatAndMount(*formatAndMountTimeout, *maxConcurrentFormatAndMount) } - } - - if *enableDataCacheFlag { - if nodeName == nil || *nodeName == "" { - klog.Errorf("Data Cache enabled, but --node-name not passed") - } - if err := setupDataCache(ctx, *nodeName); err != nil { - klog.Errorf("Data Cache setup failed: %v", err) + if *enableDataCacheFlag { + if nodeName == nil || *nodeName == "" { + klog.Errorf("Data Cache enabled, but --node-name not passed") + } + if nsArgs.DataCacheEnabledNodePool { + if err := setupDataCache(ctx, *nodeName, nodeServer.MetadataService.GetName()); err != nil { + klog.Errorf("Data Cache setup failed: %v", err) + } + go driver.StartWatcher(*nodeName) + } } } @@ -327,9 +329,11 @@ func urlFlag(target **url.URL, name string, usage string) { } func isDataCacheEnabledNodePool(ctx context.Context, nodeName string) bool { - dataCacheLSSDCount, err := driver.GetDataCacheCountFromNodeLabel(ctx, nodeName) - if err != nil || dataCacheLSSDCount == 0 { - return false + if nodeName != common.TestNode { // disregard logic below when E2E testing. + dataCacheLSSDCount, err := driver.GetDataCacheCountFromNodeLabel(ctx, nodeName) + if err != nil || dataCacheLSSDCount == 0 { + return false + } } return true } diff --git a/pkg/gce-pd-csi-driver/cache.go b/pkg/gce-pd-csi-driver/cache.go index 7a8d627ac..2abb6d8bd 100644 --- a/pkg/gce-pd-csi-driver/cache.go +++ b/pkg/gce-pd-csi-driver/cache.go @@ -452,6 +452,10 @@ func createVg(volumeGroupName string, raidedLocalSsds string) error { } func reduceVolumeGroup(volumeGroupName string, force bool) { + if !checkVgExists(volumeGroupName) { + klog.V(2).Infof("Volume group %v not found, no further action needed", volumeGroupName) + return + } args := []string{ "--removemissing", volumeGroupName, From 0c33c22f107f1af51b8031bff85053002791b885 Mon Sep 17 00:00:00 2001 From: Hung Nguyen Date: Fri, 14 Mar 2025 19:18:52 +0000 Subject: [PATCH 15/25] Fix logic bug while checking available LSSDs for RAIDing for Data Cache --- cmd/gce-pd-csi-driver/main.go | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/cmd/gce-pd-csi-driver/main.go b/cmd/gce-pd-csi-driver/main.go index ac5e8acd2..302992ece 100644 --- a/cmd/gce-pd-csi-driver/main.go +++ b/cmd/gce-pd-csi-driver/main.go @@ -349,24 +349,14 @@ func fetchLssdsForRaiding(lssdCount int) ([]string, error) { return nil, fmt.Errorf("Error listing RAIDed LSSDs %v", err) } - unRaidedLssds := []string{} - for _, l := range allLssds { - if !slices.Contains(raidedLssds, l) { - unRaidedLssds = append(unRaidedLssds, l) - } - if len(unRaidedLssds) == lssdCount { - break - } - } - LSSDsWithEmptyMountPoint, err := driver.FetchLSSDsWihtEmptyMountPoint() if err != nil { return nil, fmt.Errorf("Error listing LSSDs with empty mountpoint: %v", err) } // We need to ensure the disks to be used for Data Cache are both unRAIDed & not containing mountpoints for ephemeral storage already - availableLssds := slices.Filter(nil, unRaidedLssds, func(e string) bool { - return slices.Contains(LSSDsWithEmptyMountPoint, e) + availableLssds := slices.Filter(nil, allLssds, func(e string) bool { + return slices.Contains(LSSDsWithEmptyMountPoint, e) && !slices.Contains(raidedLssds, e) }) if len(availableLssds) == 0 { @@ -376,7 +366,8 @@ func fetchLssdsForRaiding(lssdCount int) ([]string, error) { if len(availableLssds) < lssdCount { return nil, fmt.Errorf("Not enough LSSDs available to set up caching. Available LSSDs: %v, wanted LSSDs: %v", len(availableLssds), lssdCount) } - return availableLssds, nil + + return availableLssds[:lssdCount], nil } func setupDataCache(ctx context.Context, nodeName string) error { From 0cb814ad0843a9e38e3fba52b07dd044b13c1075 Mon Sep 17 00:00:00 2001 From: Hung Nguyen Date: Mon, 31 Mar 2025 16:17:19 +0000 Subject: [PATCH 16/25] fix outdated metadata error in watcher --- pkg/gce-pd-csi-driver/cache.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/gce-pd-csi-driver/cache.go b/pkg/gce-pd-csi-driver/cache.go index 2abb6d8bd..f8f7ab583 100644 --- a/pkg/gce-pd-csi-driver/cache.go +++ b/pkg/gce-pd-csi-driver/cache.go @@ -626,6 +626,14 @@ func watchDiskDetaches(watcher *fsnotify.Watcher, nodeName string, errorCh chan case event := <-watcher.Events: // In case of an event i.e. creation or deletion of any new PV, we update the VG metadata. // This might include some non-LVM changes, no harm in updating metadata multiple times. + args := []string{ + "--updatemetadata", + getVolumeGroupName(nodeName), + } + _, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgck", args...) + if err != nil { + klog.Errorf("Error updating volume group's metadata: %v", err) + } reduceVolumeGroup(getVolumeGroupName(nodeName), true) klog.V(2).Infof("disk attach/detach event %#v\n", event) } From 9aca35bc57e185f2701b99d2e927f93aab5c6188 Mon Sep 17 00:00:00 2001 From: Sneha Aradhey Date: Thu, 3 Apr 2025 04:45:27 +0000 Subject: [PATCH 17/25] update cache logic to calculate chunk size based on toatl cache --- pkg/gce-pd-csi-driver/cache.go | 61 +++++++++++++++++++++++++++-- pkg/gce-pd-csi-driver/cache_test.go | 60 ++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 4 deletions(-) diff --git a/pkg/gce-pd-csi-driver/cache.go b/pkg/gce-pd-csi-driver/cache.go index f8f7ab583..403423342 100644 --- a/pkg/gce-pd-csi-driver/cache.go +++ b/pkg/gce-pd-csi-driver/cache.go @@ -169,10 +169,18 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str klog.V(4).Infof("Assuming valid data cache size and mode, resizing cache is not supported") } else { cacheSize := req.GetPublishContext()[common.ContextDataCacheSize] - chunkSize, err := fetchChunkSizeKiB(cacheSize) + maxChunkSizeStr := strconv.FormatInt(int64(maxChunkSize/KiB), 10) + var chunkSize string + cachePvSize, err := fetchPvSizeGiB() if err != nil { - klog.Errorf("Errored to fetch cache size, verify the data-cache-size is valid: got %v, error: %q", cacheSize, err) - return mainDevicePath, err + klog.Errorf("Errored while fetching PV size, got %v, falling back to default chunkSize of %v", err, maxChunkSize) + chunkSize = maxChunkSizeStr + } else { + chunkSize, err = fetchChunkSizeKiB(cachePvSize) + if err != nil { + klog.Errorf("Errored to fetch cache size, verify the data-cache-size is valid: got %v, error: %q", chunkSize, err) + chunkSize = maxChunkSizeStr + } } // Check if LV exists info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvs", args...) @@ -635,7 +643,7 @@ func watchDiskDetaches(watcher *fsnotify.Watcher, nodeName string, errorCh chan klog.Errorf("Error updating volume group's metadata: %v", err) } reduceVolumeGroup(getVolumeGroupName(nodeName), true) - klog.V(2).Infof("disk attach/detach event %#v\n", event) + klog.V(6).Infof("disk attach/detach event %#v\n", event) } } } @@ -667,3 +675,48 @@ func addRaidedLSSDToVg(vgName, lssdPath string) error { } return nil } + +func fetchPvSizeGiB() (string, error) { + args := []string{ + "--select", + "-o", + "--noheadings", + "pv_size", + "--units=b", + } + // RAIDed device is always registered with its /dev/md127 equivalent in VG so cannot check it directly based on the RAIDed LSSD path which could be /dev/md/csi-driver-data-cache + info, err := common.RunCommand("grep" /* pipedCmd */, []string{"/dev/md"} /* pipedCmdArg */, "pvs", args...) + if err != nil { + return "", fmt.Errorf("errored while fetching PV size %v: %s", err, info) + } + infoString := strings.TrimSpace(string(info)) + infoSlice := strings.Fields(infoString) + pvSize, err := fetchNumberGiB(infoSlice) + if err != nil { + return "", fmt.Errorf("Error fetching PV size for cache %v", err) + } + return pvSize, nil + +} + +func fetchNumberGiB(infoSlice []string) (string, error) { + re, err := regexp.Compile("^[0-9]+B$") + if err != nil { + return "", fmt.Errorf("Failed to compile regex match %v", err) + } + var pvSize string + for _, i := range infoSlice { + if re.MatchString(i) { + pvSize, err = strings.TrimSuffix(i, "B"), nil + if err != nil { + return "", fmt.Errorf("Failed to extract PV size %v", err) + } + break + } + } + pvSizeInt, err := strconv.ParseFloat(pvSize, 64) + if err != nil { + return "", fmt.Errorf("Error while fetching PV size for cache %v", err) + } + return strconv.FormatInt(int64(math.Ceil(pvSizeInt/GiB)), 10) + "GiB", nil +} diff --git a/pkg/gce-pd-csi-driver/cache_test.go b/pkg/gce-pd-csi-driver/cache_test.go index 918f26b0c..33fc4e673 100644 --- a/pkg/gce-pd-csi-driver/cache_test.go +++ b/pkg/gce-pd-csi-driver/cache_test.go @@ -55,3 +55,63 @@ func TestFetchChunkSizeKiB(t *testing.T) { } } + +func TestFetchNumberGiB(t *testing.T) { + testCases := []struct { + name string + stringInput []string + expOutput string // Outputs value in GiB + expErr bool + }{ + { + name: "valid input 1", + stringInput: []string{"5000000000B"}, + expOutput: "5GiB", //range defined in fetchChunkSizeKiB + }, + { + name: "valid input 2", + stringInput: []string{"375000000000B"}, // 1 LSSD attached + expOutput: "350GiB", //range defined in fetchChunkSizeKiB + }, + { + name: "valid input 3", + stringInput: []string{"9000000000000B"}, // 24 LSSD attached + expOutput: "8382GiB", //range defined in fetchChunkSizeKiB + }, + { + name: "valid input 4", + stringInput: []string{"Some text before ", "9000000000000B", "Some text after"}, // 24 LSSD attached + expOutput: "8382GiB", //range defined in fetchChunkSizeKiB + }, + { + name: "invalid input 1", + stringInput: []string{"9000000000000"}, + expErr: true, + }, + { + name: "invalid input 2", + stringInput: []string{"A9000000000000B"}, + expErr: true, + }, + { + name: "valid input 5", + stringInput: []string{"900000B"}, // <1GiB gets rounded off to 0GiB + expOutput: "1GiB", + }, + } + + for _, tc := range testCases { + v, err := fetchNumberGiB(tc.stringInput) + if err != nil { + if !tc.expErr { + t.Errorf("Errored %s", err) + } + continue + } + if v != tc.expOutput { + t.Errorf("Got %s want %s", v, tc.expOutput) + } + + } + +} From 5b9665a0c28498e2ab61c0be2c1b5686f5abcba6 Mon Sep 17 00:00:00 2001 From: halimsam Date: Tue, 8 Apr 2025 16:42:19 +0000 Subject: [PATCH 18/25] Update --- deploy/kubernetes/base/controller/controller.yaml | 5 ++++- deploy/kubernetes/base/node_linux/node.yaml | 4 +++- deploy/kubernetes/images/stable-master/image.yaml | 7 +++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/deploy/kubernetes/base/controller/controller.yaml b/deploy/kubernetes/base/controller/controller.yaml index 5f8713af5..7d48a60c4 100644 --- a/deploy/kubernetes/base/controller/controller.yaml +++ b/deploy/kubernetes/base/controller/controller.yaml @@ -139,7 +139,10 @@ spec: args: - "--v=5" - "--endpoint=unix:/csi/csi.sock" - - --enable-data-cache + # - "--enable-data-cache" + - "--enable-data-cache=true" + - "--run-node-service=false" + - "--run-controller-service=true" env: - name: GOOGLE_APPLICATION_CREDENTIALS value: "/etc/cloud-sa/cloud-sa.json" diff --git a/deploy/kubernetes/base/node_linux/node.yaml b/deploy/kubernetes/base/node_linux/node.yaml index b191b2881..73cb8f5e6 100644 --- a/deploy/kubernetes/base/node_linux/node.yaml +++ b/deploy/kubernetes/base/node_linux/node.yaml @@ -46,7 +46,9 @@ spec: - "--v=5" - "--endpoint=unix:/csi/csi.sock" - "--run-controller-service=false" - - "--enable-data-cache" + - "--run-node-service=true" + # - "--enable-data-cache" + - "--enable-data-cache=true" - "--node-name=$(KUBE_NODE_NAME)" securityContext: privileged: true diff --git a/deploy/kubernetes/images/stable-master/image.yaml b/deploy/kubernetes/images/stable-master/image.yaml index 56a0ac6ae..5a5ce5f45 100644 --- a/deploy/kubernetes/images/stable-master/image.yaml +++ b/deploy/kubernetes/images/stable-master/image.yaml @@ -49,8 +49,7 @@ metadata: name: imagetag-gcepd-driver imageTag: name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver - # Don't change stable image without changing pdImagePlaceholder in - # test/k8s-integration/main.go - newName: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver - newTag: "v1.13.2" + # pdImagePlaceholder in test/k8s-integration/main.go is updated automatically with the newTag + newName: gcr.io/samhalim-joonix/gcp-compute-persistent-disk-csi-driver + newTag: "datacache_sam" --- From f04f49634cd6134fadd9410dd4f286b4d6af947b Mon Sep 17 00:00:00 2001 From: halimsam Date: Wed, 30 Apr 2025 03:24:12 +0000 Subject: [PATCH 19/25] remove GKE Data Cache Watcher regular event logs since it's generating alot of no-op logs for customer. --- deploy/kubernetes/base/controller/controller.yaml | 5 +---- deploy/kubernetes/base/node_linux/node.yaml | 4 +--- deploy/kubernetes/images/stable-master/image.yaml | 4 ++-- pkg/gce-pd-csi-driver/cache.go | 3 +-- 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/deploy/kubernetes/base/controller/controller.yaml b/deploy/kubernetes/base/controller/controller.yaml index 7d48a60c4..5f8713af5 100644 --- a/deploy/kubernetes/base/controller/controller.yaml +++ b/deploy/kubernetes/base/controller/controller.yaml @@ -139,10 +139,7 @@ spec: args: - "--v=5" - "--endpoint=unix:/csi/csi.sock" - # - "--enable-data-cache" - - "--enable-data-cache=true" - - "--run-node-service=false" - - "--run-controller-service=true" + - --enable-data-cache env: - name: GOOGLE_APPLICATION_CREDENTIALS value: "/etc/cloud-sa/cloud-sa.json" diff --git a/deploy/kubernetes/base/node_linux/node.yaml b/deploy/kubernetes/base/node_linux/node.yaml index 73cb8f5e6..b191b2881 100644 --- a/deploy/kubernetes/base/node_linux/node.yaml +++ b/deploy/kubernetes/base/node_linux/node.yaml @@ -46,9 +46,7 @@ spec: - "--v=5" - "--endpoint=unix:/csi/csi.sock" - "--run-controller-service=false" - - "--run-node-service=true" - # - "--enable-data-cache" - - "--enable-data-cache=true" + - "--enable-data-cache" - "--node-name=$(KUBE_NODE_NAME)" securityContext: privileged: true diff --git a/deploy/kubernetes/images/stable-master/image.yaml b/deploy/kubernetes/images/stable-master/image.yaml index 5a5ce5f45..abd5c0eca 100644 --- a/deploy/kubernetes/images/stable-master/image.yaml +++ b/deploy/kubernetes/images/stable-master/image.yaml @@ -50,6 +50,6 @@ metadata: imageTag: name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver # pdImagePlaceholder in test/k8s-integration/main.go is updated automatically with the newTag - newName: gcr.io/samhalim-joonix/gcp-compute-persistent-disk-csi-driver - newTag: "datacache_sam" + newName: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver + newTag: "v1.17.2" --- diff --git a/pkg/gce-pd-csi-driver/cache.go b/pkg/gce-pd-csi-driver/cache.go index 403423342..fa087a842 100644 --- a/pkg/gce-pd-csi-driver/cache.go +++ b/pkg/gce-pd-csi-driver/cache.go @@ -631,7 +631,7 @@ func watchDiskDetaches(watcher *fsnotify.Watcher, nodeName string, errorCh chan case err := <-watcher.Errors: errorCh <- fmt.Errorf("disk update event errored: %v", err) // watch for events - case event := <-watcher.Events: + case <-watcher.Events: // In case of an event i.e. creation or deletion of any new PV, we update the VG metadata. // This might include some non-LVM changes, no harm in updating metadata multiple times. args := []string{ @@ -643,7 +643,6 @@ func watchDiskDetaches(watcher *fsnotify.Watcher, nodeName string, errorCh chan klog.Errorf("Error updating volume group's metadata: %v", err) } reduceVolumeGroup(getVolumeGroupName(nodeName), true) - klog.V(6).Infof("disk attach/detach event %#v\n", event) } } } From a420e9d2cae14201d0f75ca03b059f1d2f2df89d Mon Sep 17 00:00:00 2001 From: Engin Akdemir Date: Wed, 4 Jun 2025 20:50:51 +0000 Subject: [PATCH 20/25] Resolving some bad merges, update to e2e test setup, mod vendor update --- go.sum | 6 +- pkg/gce-pd-csi-driver/cache.go | 18 +- pkg/gce-pd-csi-driver/cache_test.go | 8 +- test/e2e/tests/setup_e2e_test.go | 48 +- .../fsnotify/fsnotify/.editorconfig | 12 - .../fsnotify/fsnotify/.gitattributes | 1 - vendor/github.com/fsnotify/fsnotify/AUTHORS | 62 -- vendor/github.com/fsnotify/fsnotify/fen.go | 38 -- .../fsnotify/fsnotify/fsnotify_unsupported.go | 36 -- .../github.com/fsnotify/fsnotify/inotify.go | 351 ----------- .../fsnotify/fsnotify/inotify_poller.go | 187 ------ vendor/github.com/fsnotify/fsnotify/kqueue.go | 535 ---------------- .../fsnotify/fsnotify/open_mode_bsd.go | 12 - .../fsnotify/fsnotify/open_mode_darwin.go | 13 - .../github.com/fsnotify/fsnotify/windows.go | 586 ------------------ vendor/github.com/go-logr/logr/README.md | 1 + vendor/github.com/go-logr/logr/funcr/funcr.go | 169 ++--- 17 files changed, 134 insertions(+), 1949 deletions(-) delete mode 100644 vendor/github.com/fsnotify/fsnotify/.editorconfig delete mode 100644 vendor/github.com/fsnotify/fsnotify/.gitattributes delete mode 100644 vendor/github.com/fsnotify/fsnotify/AUTHORS delete mode 100644 vendor/github.com/fsnotify/fsnotify/fen.go delete mode 100644 vendor/github.com/fsnotify/fsnotify/fsnotify_unsupported.go delete mode 100644 vendor/github.com/fsnotify/fsnotify/inotify.go delete mode 100644 vendor/github.com/fsnotify/fsnotify/inotify_poller.go delete mode 100644 vendor/github.com/fsnotify/fsnotify/kqueue.go delete mode 100644 vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go delete mode 100644 vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go delete mode 100644 vendor/github.com/fsnotify/fsnotify/windows.go diff --git a/go.sum b/go.sum index 0f436e461..2c3135245 100644 --- a/go.sum +++ b/go.sum @@ -1029,8 +1029,6 @@ github.com/frankban/quicktest v1.8.1/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fsouza/fake-gcs-server v0.0.0-20180612165233-e85be23bdaa8/go.mod h1:1/HufuJ+eaDf4KTnYdS6HJMGvMRU8d4cYTuu/1QaBbI= @@ -1073,8 +1071,8 @@ github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= diff --git a/pkg/gce-pd-csi-driver/cache.go b/pkg/gce-pd-csi-driver/cache.go index fa087a842..f8ac87e58 100644 --- a/pkg/gce-pd-csi-driver/cache.go +++ b/pkg/gce-pd-csi-driver/cache.go @@ -9,6 +9,7 @@ import ( "strings" csi "github.com/container-storage-interface/spec/lib/go/csi" + fsnotify "github.com/fsnotify/fsnotify" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -547,22 +548,7 @@ func isCachingSetup(mainLvName string) (error, bool) { func fetchChunkSizeKiB(cacheSize string) (string, error) { var chunkSize float64 - cacheSizeInt, err := common.ConvertGiStringToInt64(cacheSize) - if err != nil { - return "0", err - } - // Chunksize should be divisible by 32Kib so we need (chunksize/32*1024)*32*1024 - chunkSize = (float64(cacheSizeInt) * GiB) / float64(maxAllowedChunks) - chunkSize = math.Round(chunkSize/(32*KiB)) * (32 * KiB) - chunkSize = math.Min(math.Max(chunkSize, minChunkSize), maxChunkSize) / KiB - // default chunk size unit KiB - return strconv.FormatInt(int64(chunkSize), 10) + "KiB", nil -} - -func fetchChunkSizeKiB(cacheSize string) (string, error) { - var chunkSize float64 - - cacheSizeInt, err := common.ConvertGiStringToInt64(cacheSize) + cacheSizeInt, err := strconv.ParseInt(cacheSize, 10, 64) if err != nil { return "0", err } diff --git a/pkg/gce-pd-csi-driver/cache_test.go b/pkg/gce-pd-csi-driver/cache_test.go index 33fc4e673..340c924a1 100644 --- a/pkg/gce-pd-csi-driver/cache_test.go +++ b/pkg/gce-pd-csi-driver/cache_test.go @@ -13,22 +13,22 @@ func TestFetchChunkSizeKiB(t *testing.T) { }{ { name: "chunk size is in the allowed range", - cacheSize: "500Gi", + cacheSize: "500", expChunkSize: "512KiB", //range defined in fetchChunkSizeKiB }, { name: "chunk size is set to the range ceil", - cacheSize: "30000000Gi", + cacheSize: "30000000", expChunkSize: "1048576KiB", //range defined in fetchChunkSizeKiB - max 1GiB }, { name: "chunk size is set to the allowed range floor", - cacheSize: "10Gi", + cacheSize: "100", expChunkSize: "160KiB", //range defined in fetchChunkSizeKiB - min 160 KiB }, { name: "cacheSize set to KiB also sets the chunk size to range floor", - cacheSize: "100Ki", + cacheSize: "1", expChunkSize: "160KiB", //range defined in fetchChunkSizeKiB - min 160 KiB }, { diff --git a/test/e2e/tests/setup_e2e_test.go b/test/e2e/tests/setup_e2e_test.go index 882a32b6e..8641f3346 100644 --- a/test/e2e/tests/setup_e2e_test.go +++ b/test/e2e/tests/setup_e2e_test.go @@ -44,9 +44,10 @@ var ( vmNamePrefix = flag.String("vm-name-prefix", "gce-pd-csi-e2e", "VM name prefix") architecture = flag.String("arch", "amd64", "Architecture pd csi driver build on") minCpuPlatform = flag.String("min-cpu-platform", "rome", "Minimum CPU architecture") + mwMinCpuPlatform = flag.String("min-cpu-platform-mw", "sapphirerapids", "Minimum CPU architecture for multiwriter tests") zones = flag.String("zones", "us-east4-a,us-east4-c", "Zones to run tests in. If there are multiple zones, separate each by comma") - machineType = flag.String("machine-type", "n2-standard-2", "Type of machine to provision instance on") - imageURL = flag.String("image-url", "projects/debian-cloud/global/images/family/debian-11", "OS image url to get image from") + machineType = flag.String("machine-type", "n2d-standard-4", "Type of machine to provision instance on") + imageURL = flag.String("image-url", "projects/ubuntu-os-cloud/global/images/family/ubuntu-minimal-2404-lts-amd64", "OS image url to get image from") runInProw = flag.Bool("run-in-prow", false, "If true, use a Boskos loaned project and special CI service accounts and ssh keys") deleteInstances = flag.Bool("delete-instances", false, "Delete the instances after tests run") cloudtopHost = flag.Bool("cloudtop-host", false, "The local host is cloudtop, a kind of googler machine with special requirements to access GCP") @@ -54,13 +55,15 @@ var ( enableConfidentialCompute = flag.Bool("enable-confidential-compute", false, "Create VMs with confidential compute mode. This uses NVMe devices") // Multi-writer is only supported on M3, C3, and N4 // https://cloud.google.com/compute/docs/disks/sharing-disks-between-vms#hd-multi-writer - hdMachineType = flag.String("hyperdisk-machine-type", "c3-standard-4", "Type of machine to provision instance on") - - testContexts = []*remote.TestContext{} - computeService *compute.Service - computeAlphaService *computealpha.Service - computeBetaService *computebeta.Service - kmsClient *cloudkms.KeyManagementClient + hdMachineType = flag.String("hyperdisk-machine-type", "c3-standard-4", "Type of machine to provision instance on") + hdMinCpuPlatform = flag.String("hyperdisk-min-cpu-platform", "sapphirerapids", "Minimum CPU architecture") + + testContexts = []*remote.TestContext{} + hyperdiskTestContexts = []*remote.TestContext{} + computeService *compute.Service + computeAlphaService *computealpha.Service + computeBetaService *computebeta.Service + kmsClient *cloudkms.KeyManagementClient ) func init() { @@ -78,7 +81,9 @@ var _ = BeforeSuite(func() { numberOfInstancesPerZone := 2 zones := strings.Split(*zones, ",") tcc := make(chan *remote.TestContext, len(zones)*numberOfInstancesPerZone) + hdtcc := make(chan *remote.TestContext, len(zones)) defer close(tcc) + defer close(hdtcc) rand.Seed(time.Now().UnixNano()) @@ -115,6 +120,13 @@ var _ = BeforeSuite(func() { tcc <- NewDefaultTestContext(curZone, strconv.Itoa(randInt)) }(zone, j) } + go func(curZone string) { + wg.Add(1) + defer GinkgoRecover() + defer wg.Done() + hdtcc <- NewTestContext(curZone, *hdMinCpuPlatform, *hdMachineType, "0") + }(zone) + wg.Wait() } for _, zone := range zones { @@ -126,6 +138,11 @@ var _ = BeforeSuite(func() { testContexts = append(testContexts, tc) klog.Infof("Added TestContext for node %s", tc.Instance.GetName()) } + for i := 0; i < len(zones); i++ { + tc := <-hdtcc + hyperdiskTestContexts = append(hyperdiskTestContexts, tc) + klog.Infof("Added TestContext for node %s", tc.Instance.GetName()) + } }) var _ = AfterSuite(func() { @@ -136,6 +153,13 @@ var _ = AfterSuite(func() { tc.Instance.DeleteInstance() } } + for _, mwTc := range hyperdiskTestContexts { + err := remote.TeardownDriverAndClient(mwTc) + Expect(err).To(BeNil(), "Multiwriter Teardown Driver and Client failed with error") + if *deleteInstances { + mwTc.Instance.DeleteInstance() + } + } }) func notEmpty(v string) bool { @@ -220,3 +244,9 @@ func getRandomTestContext() *remote.TestContext { rn := rand.Intn(len(testContexts)) return testContexts[rn] } + +func getRandomMwTestContext() *remote.TestContext { + Expect(hyperdiskTestContexts).ToNot(BeEmpty()) + rn := rand.Intn(len(hyperdiskTestContexts)) + return hyperdiskTestContexts[rn] +} diff --git a/vendor/github.com/fsnotify/fsnotify/.editorconfig b/vendor/github.com/fsnotify/fsnotify/.editorconfig deleted file mode 100644 index fad895851..000000000 --- a/vendor/github.com/fsnotify/fsnotify/.editorconfig +++ /dev/null @@ -1,12 +0,0 @@ -root = true - -[*.go] -indent_style = tab -indent_size = 4 -insert_final_newline = true - -[*.{yml,yaml}] -indent_style = space -indent_size = 2 -insert_final_newline = true -trim_trailing_whitespace = true diff --git a/vendor/github.com/fsnotify/fsnotify/.gitattributes b/vendor/github.com/fsnotify/fsnotify/.gitattributes deleted file mode 100644 index 32f1001be..000000000 --- a/vendor/github.com/fsnotify/fsnotify/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -go.sum linguist-generated diff --git a/vendor/github.com/fsnotify/fsnotify/AUTHORS b/vendor/github.com/fsnotify/fsnotify/AUTHORS deleted file mode 100644 index 6cbabe5ef..000000000 --- a/vendor/github.com/fsnotify/fsnotify/AUTHORS +++ /dev/null @@ -1,62 +0,0 @@ -# Names should be added to this file as -# Name or Organization -# The email address is not required for organizations. - -# You can update this list using the following command: -# -# $ (head -n10 AUTHORS && git shortlog -se | sed -E 's/^\s+[0-9]+\t//') | tee AUTHORS - -# Please keep the list sorted. - -Aaron L -Adrien Bustany -Alexey Kazakov -Amit Krishnan -Anmol Sethi -Bjørn Erik Pedersen -Brian Goff -Bruno Bigras -Caleb Spare -Case Nelson -Chris Howey -Christoffer Buchholz -Daniel Wagner-Hall -Dave Cheney -Eric Lin -Evan Phoenix -Francisco Souza -Gautam Dey -Hari haran -Ichinose Shogo -Johannes Ebke -John C Barstow -Kelvin Fo -Ken-ichirou MATSUZAWA -Matt Layher -Matthias Stone -Nathan Youngman -Nickolai Zeldovich -Oliver Bristow -Patrick -Paul Hammond -Pawel Knap -Pieter Droogendijk -Pratik Shinde -Pursuit92 -Riku Voipio -Rob Figueiredo -Rodrigo Chiossi -Slawek Ligus -Soge Zhang -Tiffany Jernigan -Tilak Sharma -Tobias Klauser -Tom Payne -Travis Cline -Tudor Golubenco -Vahe Khachikyan -Yukang -bronze1man -debrando -henrikedwards -铁哥 diff --git a/vendor/github.com/fsnotify/fsnotify/fen.go b/vendor/github.com/fsnotify/fsnotify/fen.go deleted file mode 100644 index b3ac3d8f5..000000000 --- a/vendor/github.com/fsnotify/fsnotify/fen.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build solaris -// +build solaris - -package fsnotify - -import ( - "errors" -) - -// Watcher watches a set of files, delivering events to a channel. -type Watcher struct { - Events chan Event - Errors chan error -} - -// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. -func NewWatcher() (*Watcher, error) { - return nil, errors.New("FEN based watcher not yet supported for fsnotify\n") -} - -// Close removes all watches and closes the events channel. -func (w *Watcher) Close() error { - return nil -} - -// Add starts watching the named file or directory (non-recursively). -func (w *Watcher) Add(name string) error { - return nil -} - -// Remove stops watching the the named file or directory (non-recursively). -func (w *Watcher) Remove(name string) error { - return nil -} diff --git a/vendor/github.com/fsnotify/fsnotify/fsnotify_unsupported.go b/vendor/github.com/fsnotify/fsnotify/fsnotify_unsupported.go deleted file mode 100644 index 596885598..000000000 --- a/vendor/github.com/fsnotify/fsnotify/fsnotify_unsupported.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !darwin && !dragonfly && !freebsd && !openbsd && !linux && !netbsd && !solaris && !windows -// +build !darwin,!dragonfly,!freebsd,!openbsd,!linux,!netbsd,!solaris,!windows - -package fsnotify - -import ( - "fmt" - "runtime" -) - -// Watcher watches a set of files, delivering events to a channel. -type Watcher struct{} - -// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. -func NewWatcher() (*Watcher, error) { - return nil, fmt.Errorf("fsnotify not supported on %s", runtime.GOOS) -} - -// Close removes all watches and closes the events channel. -func (w *Watcher) Close() error { - return nil -} - -// Add starts watching the named file or directory (non-recursively). -func (w *Watcher) Add(name string) error { - return nil -} - -// Remove stops watching the the named file or directory (non-recursively). -func (w *Watcher) Remove(name string) error { - return nil -} diff --git a/vendor/github.com/fsnotify/fsnotify/inotify.go b/vendor/github.com/fsnotify/fsnotify/inotify.go deleted file mode 100644 index a6d0e0ec8..000000000 --- a/vendor/github.com/fsnotify/fsnotify/inotify.go +++ /dev/null @@ -1,351 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build linux -// +build linux - -package fsnotify - -import ( - "errors" - "fmt" - "io" - "os" - "path/filepath" - "strings" - "sync" - "unsafe" - - "golang.org/x/sys/unix" -) - -// Watcher watches a set of files, delivering events to a channel. -type Watcher struct { - Events chan Event - Errors chan error - mu sync.Mutex // Map access - fd int - poller *fdPoller - watches map[string]*watch // Map of inotify watches (key: path) - paths map[int]string // Map of watched paths (key: watch descriptor) - done chan struct{} // Channel for sending a "quit message" to the reader goroutine - doneResp chan struct{} // Channel to respond to Close -} - -// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. -func NewWatcher() (*Watcher, error) { - // Create inotify fd - fd, errno := unix.InotifyInit1(unix.IN_CLOEXEC) - if fd == -1 { - return nil, errno - } - // Create epoll - poller, err := newFdPoller(fd) - if err != nil { - unix.Close(fd) - return nil, err - } - w := &Watcher{ - fd: fd, - poller: poller, - watches: make(map[string]*watch), - paths: make(map[int]string), - Events: make(chan Event), - Errors: make(chan error), - done: make(chan struct{}), - doneResp: make(chan struct{}), - } - - go w.readEvents() - return w, nil -} - -func (w *Watcher) isClosed() bool { - select { - case <-w.done: - return true - default: - return false - } -} - -// Close removes all watches and closes the events channel. -func (w *Watcher) Close() error { - if w.isClosed() { - return nil - } - - // Send 'close' signal to goroutine, and set the Watcher to closed. - close(w.done) - - // Wake up goroutine - w.poller.wake() - - // Wait for goroutine to close - <-w.doneResp - - return nil -} - -// Add starts watching the named file or directory (non-recursively). -func (w *Watcher) Add(name string) error { - name = filepath.Clean(name) - if w.isClosed() { - return errors.New("inotify instance already closed") - } - - const agnosticEvents = unix.IN_MOVED_TO | unix.IN_MOVED_FROM | - unix.IN_CREATE | unix.IN_ATTRIB | unix.IN_MODIFY | - unix.IN_MOVE_SELF | unix.IN_DELETE | unix.IN_DELETE_SELF - - var flags uint32 = agnosticEvents - - w.mu.Lock() - defer w.mu.Unlock() - watchEntry := w.watches[name] - if watchEntry != nil { - flags |= watchEntry.flags | unix.IN_MASK_ADD - } - wd, errno := unix.InotifyAddWatch(w.fd, name, flags) - if wd == -1 { - return errno - } - - if watchEntry == nil { - w.watches[name] = &watch{wd: uint32(wd), flags: flags} - w.paths[wd] = name - } else { - watchEntry.wd = uint32(wd) - watchEntry.flags = flags - } - - return nil -} - -// Remove stops watching the named file or directory (non-recursively). -func (w *Watcher) Remove(name string) error { - name = filepath.Clean(name) - - // Fetch the watch. - w.mu.Lock() - defer w.mu.Unlock() - watch, ok := w.watches[name] - - // Remove it from inotify. - if !ok { - return fmt.Errorf("can't remove non-existent inotify watch for: %s", name) - } - - // We successfully removed the watch if InotifyRmWatch doesn't return an - // error, we need to clean up our internal state to ensure it matches - // inotify's kernel state. - delete(w.paths, int(watch.wd)) - delete(w.watches, name) - - // inotify_rm_watch will return EINVAL if the file has been deleted; - // the inotify will already have been removed. - // watches and pathes are deleted in ignoreLinux() implicitly and asynchronously - // by calling inotify_rm_watch() below. e.g. readEvents() goroutine receives IN_IGNORE - // so that EINVAL means that the wd is being rm_watch()ed or its file removed - // by another thread and we have not received IN_IGNORE event. - success, errno := unix.InotifyRmWatch(w.fd, watch.wd) - if success == -1 { - // TODO: Perhaps it's not helpful to return an error here in every case. - // the only two possible errors are: - // EBADF, which happens when w.fd is not a valid file descriptor of any kind. - // EINVAL, which is when fd is not an inotify descriptor or wd is not a valid watch descriptor. - // Watch descriptors are invalidated when they are removed explicitly or implicitly; - // explicitly by inotify_rm_watch, implicitly when the file they are watching is deleted. - return errno - } - - return nil -} - -// WatchList returns the directories and files that are being monitered. -func (w *Watcher) WatchList() []string { - w.mu.Lock() - defer w.mu.Unlock() - - entries := make([]string, 0, len(w.watches)) - for pathname := range w.watches { - entries = append(entries, pathname) - } - - return entries -} - -type watch struct { - wd uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall) - flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags) -} - -// readEvents reads from the inotify file descriptor, converts the -// received events into Event objects and sends them via the Events channel -func (w *Watcher) readEvents() { - var ( - buf [unix.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events - n int // Number of bytes read with read() - errno error // Syscall errno - ok bool // For poller.wait - ) - - defer close(w.doneResp) - defer close(w.Errors) - defer close(w.Events) - defer unix.Close(w.fd) - defer w.poller.close() - - for { - // See if we have been closed. - if w.isClosed() { - return - } - - ok, errno = w.poller.wait() - if errno != nil { - select { - case w.Errors <- errno: - case <-w.done: - return - } - continue - } - - if !ok { - continue - } - - n, errno = unix.Read(w.fd, buf[:]) - // If a signal interrupted execution, see if we've been asked to close, and try again. - // http://man7.org/linux/man-pages/man7/signal.7.html : - // "Before Linux 3.8, reads from an inotify(7) file descriptor were not restartable" - if errno == unix.EINTR { - continue - } - - // unix.Read might have been woken up by Close. If so, we're done. - if w.isClosed() { - return - } - - if n < unix.SizeofInotifyEvent { - var err error - if n == 0 { - // If EOF is received. This should really never happen. - err = io.EOF - } else if n < 0 { - // If an error occurred while reading. - err = errno - } else { - // Read was too short. - err = errors.New("notify: short read in readEvents()") - } - select { - case w.Errors <- err: - case <-w.done: - return - } - continue - } - - var offset uint32 - // We don't know how many events we just read into the buffer - // While the offset points to at least one whole event... - for offset <= uint32(n-unix.SizeofInotifyEvent) { - // Point "raw" to the event in the buffer - raw := (*unix.InotifyEvent)(unsafe.Pointer(&buf[offset])) - - mask := uint32(raw.Mask) - nameLen := uint32(raw.Len) - - if mask&unix.IN_Q_OVERFLOW != 0 { - select { - case w.Errors <- ErrEventOverflow: - case <-w.done: - return - } - } - - // If the event happened to the watched directory or the watched file, the kernel - // doesn't append the filename to the event, but we would like to always fill the - // the "Name" field with a valid filename. We retrieve the path of the watch from - // the "paths" map. - w.mu.Lock() - name, ok := w.paths[int(raw.Wd)] - // IN_DELETE_SELF occurs when the file/directory being watched is removed. - // This is a sign to clean up the maps, otherwise we are no longer in sync - // with the inotify kernel state which has already deleted the watch - // automatically. - if ok && mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF { - delete(w.paths, int(raw.Wd)) - delete(w.watches, name) - } - w.mu.Unlock() - - if nameLen > 0 { - // Point "bytes" at the first byte of the filename - bytes := (*[unix.PathMax]byte)(unsafe.Pointer(&buf[offset+unix.SizeofInotifyEvent]))[:nameLen:nameLen] - // The filename is padded with NULL bytes. TrimRight() gets rid of those. - name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000") - } - - event := newEvent(name, mask) - - // Send the events that are not ignored on the events channel - if !event.ignoreLinux(mask) { - select { - case w.Events <- event: - case <-w.done: - return - } - } - - // Move to the next event in the buffer - offset += unix.SizeofInotifyEvent + nameLen - } - } -} - -// Certain types of events can be "ignored" and not sent over the Events -// channel. Such as events marked ignore by the kernel, or MODIFY events -// against files that do not exist. -func (e *Event) ignoreLinux(mask uint32) bool { - // Ignore anything the inotify API says to ignore - if mask&unix.IN_IGNORED == unix.IN_IGNORED { - return true - } - - // If the event is not a DELETE or RENAME, the file must exist. - // Otherwise the event is ignored. - // *Note*: this was put in place because it was seen that a MODIFY - // event was sent after the DELETE. This ignores that MODIFY and - // assumes a DELETE will come or has come if the file doesn't exist. - if !(e.Op&Remove == Remove || e.Op&Rename == Rename) { - _, statErr := os.Lstat(e.Name) - return os.IsNotExist(statErr) - } - return false -} - -// newEvent returns an platform-independent Event based on an inotify mask. -func newEvent(name string, mask uint32) Event { - e := Event{Name: name} - if mask&unix.IN_CREATE == unix.IN_CREATE || mask&unix.IN_MOVED_TO == unix.IN_MOVED_TO { - e.Op |= Create - } - if mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF || mask&unix.IN_DELETE == unix.IN_DELETE { - e.Op |= Remove - } - if mask&unix.IN_MODIFY == unix.IN_MODIFY { - e.Op |= Write - } - if mask&unix.IN_MOVE_SELF == unix.IN_MOVE_SELF || mask&unix.IN_MOVED_FROM == unix.IN_MOVED_FROM { - e.Op |= Rename - } - if mask&unix.IN_ATTRIB == unix.IN_ATTRIB { - e.Op |= Chmod - } - return e -} diff --git a/vendor/github.com/fsnotify/fsnotify/inotify_poller.go b/vendor/github.com/fsnotify/fsnotify/inotify_poller.go deleted file mode 100644 index b572a37c3..000000000 --- a/vendor/github.com/fsnotify/fsnotify/inotify_poller.go +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build linux -// +build linux - -package fsnotify - -import ( - "errors" - - "golang.org/x/sys/unix" -) - -type fdPoller struct { - fd int // File descriptor (as returned by the inotify_init() syscall) - epfd int // Epoll file descriptor - pipe [2]int // Pipe for waking up -} - -func emptyPoller(fd int) *fdPoller { - poller := new(fdPoller) - poller.fd = fd - poller.epfd = -1 - poller.pipe[0] = -1 - poller.pipe[1] = -1 - return poller -} - -// Create a new inotify poller. -// This creates an inotify handler, and an epoll handler. -func newFdPoller(fd int) (*fdPoller, error) { - var errno error - poller := emptyPoller(fd) - defer func() { - if errno != nil { - poller.close() - } - }() - - // Create epoll fd - poller.epfd, errno = unix.EpollCreate1(unix.EPOLL_CLOEXEC) - if poller.epfd == -1 { - return nil, errno - } - // Create pipe; pipe[0] is the read end, pipe[1] the write end. - errno = unix.Pipe2(poller.pipe[:], unix.O_NONBLOCK|unix.O_CLOEXEC) - if errno != nil { - return nil, errno - } - - // Register inotify fd with epoll - event := unix.EpollEvent{ - Fd: int32(poller.fd), - Events: unix.EPOLLIN, - } - errno = unix.EpollCtl(poller.epfd, unix.EPOLL_CTL_ADD, poller.fd, &event) - if errno != nil { - return nil, errno - } - - // Register pipe fd with epoll - event = unix.EpollEvent{ - Fd: int32(poller.pipe[0]), - Events: unix.EPOLLIN, - } - errno = unix.EpollCtl(poller.epfd, unix.EPOLL_CTL_ADD, poller.pipe[0], &event) - if errno != nil { - return nil, errno - } - - return poller, nil -} - -// Wait using epoll. -// Returns true if something is ready to be read, -// false if there is not. -func (poller *fdPoller) wait() (bool, error) { - // 3 possible events per fd, and 2 fds, makes a maximum of 6 events. - // I don't know whether epoll_wait returns the number of events returned, - // or the total number of events ready. - // I decided to catch both by making the buffer one larger than the maximum. - events := make([]unix.EpollEvent, 7) - for { - n, errno := unix.EpollWait(poller.epfd, events, -1) - if n == -1 { - if errno == unix.EINTR { - continue - } - return false, errno - } - if n == 0 { - // If there are no events, try again. - continue - } - if n > 6 { - // This should never happen. More events were returned than should be possible. - return false, errors.New("epoll_wait returned more events than I know what to do with") - } - ready := events[:n] - epollhup := false - epollerr := false - epollin := false - for _, event := range ready { - if event.Fd == int32(poller.fd) { - if event.Events&unix.EPOLLHUP != 0 { - // This should not happen, but if it does, treat it as a wakeup. - epollhup = true - } - if event.Events&unix.EPOLLERR != 0 { - // If an error is waiting on the file descriptor, we should pretend - // something is ready to read, and let unix.Read pick up the error. - epollerr = true - } - if event.Events&unix.EPOLLIN != 0 { - // There is data to read. - epollin = true - } - } - if event.Fd == int32(poller.pipe[0]) { - if event.Events&unix.EPOLLHUP != 0 { - // Write pipe descriptor was closed, by us. This means we're closing down the - // watcher, and we should wake up. - } - if event.Events&unix.EPOLLERR != 0 { - // If an error is waiting on the pipe file descriptor. - // This is an absolute mystery, and should never ever happen. - return false, errors.New("Error on the pipe descriptor.") - } - if event.Events&unix.EPOLLIN != 0 { - // This is a regular wakeup, so we have to clear the buffer. - err := poller.clearWake() - if err != nil { - return false, err - } - } - } - } - - if epollhup || epollerr || epollin { - return true, nil - } - return false, nil - } -} - -// Close the write end of the poller. -func (poller *fdPoller) wake() error { - buf := make([]byte, 1) - n, errno := unix.Write(poller.pipe[1], buf) - if n == -1 { - if errno == unix.EAGAIN { - // Buffer is full, poller will wake. - return nil - } - return errno - } - return nil -} - -func (poller *fdPoller) clearWake() error { - // You have to be woken up a LOT in order to get to 100! - buf := make([]byte, 100) - n, errno := unix.Read(poller.pipe[0], buf) - if n == -1 { - if errno == unix.EAGAIN { - // Buffer is empty, someone else cleared our wake. - return nil - } - return errno - } - return nil -} - -// Close all poller file descriptors, but not the one passed to it. -func (poller *fdPoller) close() { - if poller.pipe[1] != -1 { - unix.Close(poller.pipe[1]) - } - if poller.pipe[0] != -1 { - unix.Close(poller.pipe[0]) - } - if poller.epfd != -1 { - unix.Close(poller.epfd) - } -} diff --git a/vendor/github.com/fsnotify/fsnotify/kqueue.go b/vendor/github.com/fsnotify/fsnotify/kqueue.go deleted file mode 100644 index 6fb8d8532..000000000 --- a/vendor/github.com/fsnotify/fsnotify/kqueue.go +++ /dev/null @@ -1,535 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build freebsd || openbsd || netbsd || dragonfly || darwin -// +build freebsd openbsd netbsd dragonfly darwin - -package fsnotify - -import ( - "errors" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "sync" - "time" - - "golang.org/x/sys/unix" -) - -// Watcher watches a set of files, delivering events to a channel. -type Watcher struct { - Events chan Event - Errors chan error - done chan struct{} // Channel for sending a "quit message" to the reader goroutine - - kq int // File descriptor (as returned by the kqueue() syscall). - - mu sync.Mutex // Protects access to watcher data - watches map[string]int // Map of watched file descriptors (key: path). - externalWatches map[string]bool // Map of watches added by user of the library. - dirFlags map[string]uint32 // Map of watched directories to fflags used in kqueue. - paths map[int]pathInfo // Map file descriptors to path names for processing kqueue events. - fileExists map[string]bool // Keep track of if we know this file exists (to stop duplicate create events). - isClosed bool // Set to true when Close() is first called -} - -type pathInfo struct { - name string - isDir bool -} - -// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. -func NewWatcher() (*Watcher, error) { - kq, err := kqueue() - if err != nil { - return nil, err - } - - w := &Watcher{ - kq: kq, - watches: make(map[string]int), - dirFlags: make(map[string]uint32), - paths: make(map[int]pathInfo), - fileExists: make(map[string]bool), - externalWatches: make(map[string]bool), - Events: make(chan Event), - Errors: make(chan error), - done: make(chan struct{}), - } - - go w.readEvents() - return w, nil -} - -// Close removes all watches and closes the events channel. -func (w *Watcher) Close() error { - w.mu.Lock() - if w.isClosed { - w.mu.Unlock() - return nil - } - w.isClosed = true - - // copy paths to remove while locked - var pathsToRemove = make([]string, 0, len(w.watches)) - for name := range w.watches { - pathsToRemove = append(pathsToRemove, name) - } - w.mu.Unlock() - // unlock before calling Remove, which also locks - - for _, name := range pathsToRemove { - w.Remove(name) - } - - // send a "quit" message to the reader goroutine - close(w.done) - - return nil -} - -// Add starts watching the named file or directory (non-recursively). -func (w *Watcher) Add(name string) error { - w.mu.Lock() - w.externalWatches[name] = true - w.mu.Unlock() - _, err := w.addWatch(name, noteAllEvents) - return err -} - -// Remove stops watching the the named file or directory (non-recursively). -func (w *Watcher) Remove(name string) error { - name = filepath.Clean(name) - w.mu.Lock() - watchfd, ok := w.watches[name] - w.mu.Unlock() - if !ok { - return fmt.Errorf("can't remove non-existent kevent watch for: %s", name) - } - - const registerRemove = unix.EV_DELETE - if err := register(w.kq, []int{watchfd}, registerRemove, 0); err != nil { - return err - } - - unix.Close(watchfd) - - w.mu.Lock() - isDir := w.paths[watchfd].isDir - delete(w.watches, name) - delete(w.paths, watchfd) - delete(w.dirFlags, name) - w.mu.Unlock() - - // Find all watched paths that are in this directory that are not external. - if isDir { - var pathsToRemove []string - w.mu.Lock() - for _, path := range w.paths { - wdir, _ := filepath.Split(path.name) - if filepath.Clean(wdir) == name { - if !w.externalWatches[path.name] { - pathsToRemove = append(pathsToRemove, path.name) - } - } - } - w.mu.Unlock() - for _, name := range pathsToRemove { - // Since these are internal, not much sense in propagating error - // to the user, as that will just confuse them with an error about - // a path they did not explicitly watch themselves. - w.Remove(name) - } - } - - return nil -} - -// WatchList returns the directories and files that are being monitered. -func (w *Watcher) WatchList() []string { - w.mu.Lock() - defer w.mu.Unlock() - - entries := make([]string, 0, len(w.watches)) - for pathname := range w.watches { - entries = append(entries, pathname) - } - - return entries -} - -// Watch all events (except NOTE_EXTEND, NOTE_LINK, NOTE_REVOKE) -const noteAllEvents = unix.NOTE_DELETE | unix.NOTE_WRITE | unix.NOTE_ATTRIB | unix.NOTE_RENAME - -// keventWaitTime to block on each read from kevent -var keventWaitTime = durationToTimespec(100 * time.Millisecond) - -// addWatch adds name to the watched file set. -// The flags are interpreted as described in kevent(2). -// Returns the real path to the file which was added, if any, which may be different from the one passed in the case of symlinks. -func (w *Watcher) addWatch(name string, flags uint32) (string, error) { - var isDir bool - // Make ./name and name equivalent - name = filepath.Clean(name) - - w.mu.Lock() - if w.isClosed { - w.mu.Unlock() - return "", errors.New("kevent instance already closed") - } - watchfd, alreadyWatching := w.watches[name] - // We already have a watch, but we can still override flags. - if alreadyWatching { - isDir = w.paths[watchfd].isDir - } - w.mu.Unlock() - - if !alreadyWatching { - fi, err := os.Lstat(name) - if err != nil { - return "", err - } - - // Don't watch sockets. - if fi.Mode()&os.ModeSocket == os.ModeSocket { - return "", nil - } - - // Don't watch named pipes. - if fi.Mode()&os.ModeNamedPipe == os.ModeNamedPipe { - return "", nil - } - - // Follow Symlinks - // Unfortunately, Linux can add bogus symlinks to watch list without - // issue, and Windows can't do symlinks period (AFAIK). To maintain - // consistency, we will act like everything is fine. There will simply - // be no file events for broken symlinks. - // Hence the returns of nil on errors. - if fi.Mode()&os.ModeSymlink == os.ModeSymlink { - name, err = filepath.EvalSymlinks(name) - if err != nil { - return "", nil - } - - w.mu.Lock() - _, alreadyWatching = w.watches[name] - w.mu.Unlock() - - if alreadyWatching { - return name, nil - } - - fi, err = os.Lstat(name) - if err != nil { - return "", nil - } - } - - watchfd, err = unix.Open(name, openMode, 0700) - if watchfd == -1 { - return "", err - } - - isDir = fi.IsDir() - } - - const registerAdd = unix.EV_ADD | unix.EV_CLEAR | unix.EV_ENABLE - if err := register(w.kq, []int{watchfd}, registerAdd, flags); err != nil { - unix.Close(watchfd) - return "", err - } - - if !alreadyWatching { - w.mu.Lock() - w.watches[name] = watchfd - w.paths[watchfd] = pathInfo{name: name, isDir: isDir} - w.mu.Unlock() - } - - if isDir { - // Watch the directory if it has not been watched before, - // or if it was watched before, but perhaps only a NOTE_DELETE (watchDirectoryFiles) - w.mu.Lock() - - watchDir := (flags&unix.NOTE_WRITE) == unix.NOTE_WRITE && - (!alreadyWatching || (w.dirFlags[name]&unix.NOTE_WRITE) != unix.NOTE_WRITE) - // Store flags so this watch can be updated later - w.dirFlags[name] = flags - w.mu.Unlock() - - if watchDir { - if err := w.watchDirectoryFiles(name); err != nil { - return "", err - } - } - } - return name, nil -} - -// readEvents reads from kqueue and converts the received kevents into -// Event values that it sends down the Events channel. -func (w *Watcher) readEvents() { - eventBuffer := make([]unix.Kevent_t, 10) - -loop: - for { - // See if there is a message on the "done" channel - select { - case <-w.done: - break loop - default: - } - - // Get new events - kevents, err := read(w.kq, eventBuffer, &keventWaitTime) - // EINTR is okay, the syscall was interrupted before timeout expired. - if err != nil && err != unix.EINTR { - select { - case w.Errors <- err: - case <-w.done: - break loop - } - continue - } - - // Flush the events we received to the Events channel - for len(kevents) > 0 { - kevent := &kevents[0] - watchfd := int(kevent.Ident) - mask := uint32(kevent.Fflags) - w.mu.Lock() - path := w.paths[watchfd] - w.mu.Unlock() - event := newEvent(path.name, mask) - - if path.isDir && !(event.Op&Remove == Remove) { - // Double check to make sure the directory exists. This can happen when - // we do a rm -fr on a recursively watched folders and we receive a - // modification event first but the folder has been deleted and later - // receive the delete event - if _, err := os.Lstat(event.Name); os.IsNotExist(err) { - // mark is as delete event - event.Op |= Remove - } - } - - if event.Op&Rename == Rename || event.Op&Remove == Remove { - w.Remove(event.Name) - w.mu.Lock() - delete(w.fileExists, event.Name) - w.mu.Unlock() - } - - if path.isDir && event.Op&Write == Write && !(event.Op&Remove == Remove) { - w.sendDirectoryChangeEvents(event.Name) - } else { - // Send the event on the Events channel. - select { - case w.Events <- event: - case <-w.done: - break loop - } - } - - if event.Op&Remove == Remove { - // Look for a file that may have overwritten this. - // For example, mv f1 f2 will delete f2, then create f2. - if path.isDir { - fileDir := filepath.Clean(event.Name) - w.mu.Lock() - _, found := w.watches[fileDir] - w.mu.Unlock() - if found { - // make sure the directory exists before we watch for changes. When we - // do a recursive watch and perform rm -fr, the parent directory might - // have gone missing, ignore the missing directory and let the - // upcoming delete event remove the watch from the parent directory. - if _, err := os.Lstat(fileDir); err == nil { - w.sendDirectoryChangeEvents(fileDir) - } - } - } else { - filePath := filepath.Clean(event.Name) - if fileInfo, err := os.Lstat(filePath); err == nil { - w.sendFileCreatedEventIfNew(filePath, fileInfo) - } - } - } - - // Move to next event - kevents = kevents[1:] - } - } - - // cleanup - err := unix.Close(w.kq) - if err != nil { - // only way the previous loop breaks is if w.done was closed so we need to async send to w.Errors. - select { - case w.Errors <- err: - default: - } - } - close(w.Events) - close(w.Errors) -} - -// newEvent returns an platform-independent Event based on kqueue Fflags. -func newEvent(name string, mask uint32) Event { - e := Event{Name: name} - if mask&unix.NOTE_DELETE == unix.NOTE_DELETE { - e.Op |= Remove - } - if mask&unix.NOTE_WRITE == unix.NOTE_WRITE { - e.Op |= Write - } - if mask&unix.NOTE_RENAME == unix.NOTE_RENAME { - e.Op |= Rename - } - if mask&unix.NOTE_ATTRIB == unix.NOTE_ATTRIB { - e.Op |= Chmod - } - return e -} - -func newCreateEvent(name string) Event { - return Event{Name: name, Op: Create} -} - -// watchDirectoryFiles to mimic inotify when adding a watch on a directory -func (w *Watcher) watchDirectoryFiles(dirPath string) error { - // Get all files - files, err := ioutil.ReadDir(dirPath) - if err != nil { - return err - } - - for _, fileInfo := range files { - filePath := filepath.Join(dirPath, fileInfo.Name()) - filePath, err = w.internalWatch(filePath, fileInfo) - if err != nil { - return err - } - - w.mu.Lock() - w.fileExists[filePath] = true - w.mu.Unlock() - } - - return nil -} - -// sendDirectoryEvents searches the directory for newly created files -// and sends them over the event channel. This functionality is to have -// the BSD version of fsnotify match Linux inotify which provides a -// create event for files created in a watched directory. -func (w *Watcher) sendDirectoryChangeEvents(dirPath string) { - // Get all files - files, err := ioutil.ReadDir(dirPath) - if err != nil { - select { - case w.Errors <- err: - case <-w.done: - return - } - } - - // Search for new files - for _, fileInfo := range files { - filePath := filepath.Join(dirPath, fileInfo.Name()) - err := w.sendFileCreatedEventIfNew(filePath, fileInfo) - - if err != nil { - return - } - } -} - -// sendFileCreatedEvent sends a create event if the file isn't already being tracked. -func (w *Watcher) sendFileCreatedEventIfNew(filePath string, fileInfo os.FileInfo) (err error) { - w.mu.Lock() - _, doesExist := w.fileExists[filePath] - w.mu.Unlock() - if !doesExist { - // Send create event - select { - case w.Events <- newCreateEvent(filePath): - case <-w.done: - return - } - } - - // like watchDirectoryFiles (but without doing another ReadDir) - filePath, err = w.internalWatch(filePath, fileInfo) - if err != nil { - return err - } - - w.mu.Lock() - w.fileExists[filePath] = true - w.mu.Unlock() - - return nil -} - -func (w *Watcher) internalWatch(name string, fileInfo os.FileInfo) (string, error) { - if fileInfo.IsDir() { - // mimic Linux providing delete events for subdirectories - // but preserve the flags used if currently watching subdirectory - w.mu.Lock() - flags := w.dirFlags[name] - w.mu.Unlock() - - flags |= unix.NOTE_DELETE | unix.NOTE_RENAME - return w.addWatch(name, flags) - } - - // watch file to mimic Linux inotify - return w.addWatch(name, noteAllEvents) -} - -// kqueue creates a new kernel event queue and returns a descriptor. -func kqueue() (kq int, err error) { - kq, err = unix.Kqueue() - if kq == -1 { - return kq, err - } - return kq, nil -} - -// register events with the queue -func register(kq int, fds []int, flags int, fflags uint32) error { - changes := make([]unix.Kevent_t, len(fds)) - - for i, fd := range fds { - // SetKevent converts int to the platform-specific types: - unix.SetKevent(&changes[i], fd, unix.EVFILT_VNODE, flags) - changes[i].Fflags = fflags - } - - // register the events - success, err := unix.Kevent(kq, changes, nil, nil) - if success == -1 { - return err - } - return nil -} - -// read retrieves pending events, or waits until an event occurs. -// A timeout of nil blocks indefinitely, while 0 polls the queue. -func read(kq int, events []unix.Kevent_t, timeout *unix.Timespec) ([]unix.Kevent_t, error) { - n, err := unix.Kevent(kq, nil, events, timeout) - if err != nil { - return nil, err - } - return events[0:n], nil -} - -// durationToTimespec prepares a timeout value -func durationToTimespec(d time.Duration) unix.Timespec { - return unix.NsecToTimespec(d.Nanoseconds()) -} diff --git a/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go b/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go deleted file mode 100644 index 36cc3845b..000000000 --- a/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build freebsd || openbsd || netbsd || dragonfly -// +build freebsd openbsd netbsd dragonfly - -package fsnotify - -import "golang.org/x/sys/unix" - -const openMode = unix.O_NONBLOCK | unix.O_RDONLY | unix.O_CLOEXEC diff --git a/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go b/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go deleted file mode 100644 index 98cd8476f..000000000 --- a/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build darwin -// +build darwin - -package fsnotify - -import "golang.org/x/sys/unix" - -// note: this constant is not defined on BSD -const openMode = unix.O_EVTONLY | unix.O_CLOEXEC diff --git a/vendor/github.com/fsnotify/fsnotify/windows.go b/vendor/github.com/fsnotify/fsnotify/windows.go deleted file mode 100644 index 02ce7deb0..000000000 --- a/vendor/github.com/fsnotify/fsnotify/windows.go +++ /dev/null @@ -1,586 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build windows -// +build windows - -package fsnotify - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "reflect" - "runtime" - "sync" - "syscall" - "unsafe" -) - -// Watcher watches a set of files, delivering events to a channel. -type Watcher struct { - Events chan Event - Errors chan error - isClosed bool // Set to true when Close() is first called - mu sync.Mutex // Map access - port syscall.Handle // Handle to completion port - watches watchMap // Map of watches (key: i-number) - input chan *input // Inputs to the reader are sent on this channel - quit chan chan<- error -} - -// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. -func NewWatcher() (*Watcher, error) { - port, e := syscall.CreateIoCompletionPort(syscall.InvalidHandle, 0, 0, 0) - if e != nil { - return nil, os.NewSyscallError("CreateIoCompletionPort", e) - } - w := &Watcher{ - port: port, - watches: make(watchMap), - input: make(chan *input, 1), - Events: make(chan Event, 50), - Errors: make(chan error), - quit: make(chan chan<- error, 1), - } - go w.readEvents() - return w, nil -} - -// Close removes all watches and closes the events channel. -func (w *Watcher) Close() error { - if w.isClosed { - return nil - } - w.isClosed = true - - // Send "quit" message to the reader goroutine - ch := make(chan error) - w.quit <- ch - if err := w.wakeupReader(); err != nil { - return err - } - return <-ch -} - -// Add starts watching the named file or directory (non-recursively). -func (w *Watcher) Add(name string) error { - if w.isClosed { - return errors.New("watcher already closed") - } - in := &input{ - op: opAddWatch, - path: filepath.Clean(name), - flags: sysFSALLEVENTS, - reply: make(chan error), - } - w.input <- in - if err := w.wakeupReader(); err != nil { - return err - } - return <-in.reply -} - -// Remove stops watching the the named file or directory (non-recursively). -func (w *Watcher) Remove(name string) error { - in := &input{ - op: opRemoveWatch, - path: filepath.Clean(name), - reply: make(chan error), - } - w.input <- in - if err := w.wakeupReader(); err != nil { - return err - } - return <-in.reply -} - -// WatchList returns the directories and files that are being monitered. -func (w *Watcher) WatchList() []string { - w.mu.Lock() - defer w.mu.Unlock() - - entries := make([]string, 0, len(w.watches)) - for _, entry := range w.watches { - for _, watchEntry := range entry { - entries = append(entries, watchEntry.path) - } - } - - return entries -} - -const ( - // Options for AddWatch - sysFSONESHOT = 0x80000000 - sysFSONLYDIR = 0x1000000 - - // Events - sysFSACCESS = 0x1 - sysFSALLEVENTS = 0xfff - sysFSATTRIB = 0x4 - sysFSCLOSE = 0x18 - sysFSCREATE = 0x100 - sysFSDELETE = 0x200 - sysFSDELETESELF = 0x400 - sysFSMODIFY = 0x2 - sysFSMOVE = 0xc0 - sysFSMOVEDFROM = 0x40 - sysFSMOVEDTO = 0x80 - sysFSMOVESELF = 0x800 - - // Special events - sysFSIGNORED = 0x8000 - sysFSQOVERFLOW = 0x4000 -) - -func newEvent(name string, mask uint32) Event { - e := Event{Name: name} - if mask&sysFSCREATE == sysFSCREATE || mask&sysFSMOVEDTO == sysFSMOVEDTO { - e.Op |= Create - } - if mask&sysFSDELETE == sysFSDELETE || mask&sysFSDELETESELF == sysFSDELETESELF { - e.Op |= Remove - } - if mask&sysFSMODIFY == sysFSMODIFY { - e.Op |= Write - } - if mask&sysFSMOVE == sysFSMOVE || mask&sysFSMOVESELF == sysFSMOVESELF || mask&sysFSMOVEDFROM == sysFSMOVEDFROM { - e.Op |= Rename - } - if mask&sysFSATTRIB == sysFSATTRIB { - e.Op |= Chmod - } - return e -} - -const ( - opAddWatch = iota - opRemoveWatch -) - -const ( - provisional uint64 = 1 << (32 + iota) -) - -type input struct { - op int - path string - flags uint32 - reply chan error -} - -type inode struct { - handle syscall.Handle - volume uint32 - index uint64 -} - -type watch struct { - ov syscall.Overlapped - ino *inode // i-number - path string // Directory path - mask uint64 // Directory itself is being watched with these notify flags - names map[string]uint64 // Map of names being watched and their notify flags - rename string // Remembers the old name while renaming a file - buf [4096]byte -} - -type indexMap map[uint64]*watch -type watchMap map[uint32]indexMap - -func (w *Watcher) wakeupReader() error { - e := syscall.PostQueuedCompletionStatus(w.port, 0, 0, nil) - if e != nil { - return os.NewSyscallError("PostQueuedCompletionStatus", e) - } - return nil -} - -func getDir(pathname string) (dir string, err error) { - attr, e := syscall.GetFileAttributes(syscall.StringToUTF16Ptr(pathname)) - if e != nil { - return "", os.NewSyscallError("GetFileAttributes", e) - } - if attr&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 { - dir = pathname - } else { - dir, _ = filepath.Split(pathname) - dir = filepath.Clean(dir) - } - return -} - -func getIno(path string) (ino *inode, err error) { - h, e := syscall.CreateFile(syscall.StringToUTF16Ptr(path), - syscall.FILE_LIST_DIRECTORY, - syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, - nil, syscall.OPEN_EXISTING, - syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OVERLAPPED, 0) - if e != nil { - return nil, os.NewSyscallError("CreateFile", e) - } - var fi syscall.ByHandleFileInformation - if e = syscall.GetFileInformationByHandle(h, &fi); e != nil { - syscall.CloseHandle(h) - return nil, os.NewSyscallError("GetFileInformationByHandle", e) - } - ino = &inode{ - handle: h, - volume: fi.VolumeSerialNumber, - index: uint64(fi.FileIndexHigh)<<32 | uint64(fi.FileIndexLow), - } - return ino, nil -} - -// Must run within the I/O thread. -func (m watchMap) get(ino *inode) *watch { - if i := m[ino.volume]; i != nil { - return i[ino.index] - } - return nil -} - -// Must run within the I/O thread. -func (m watchMap) set(ino *inode, watch *watch) { - i := m[ino.volume] - if i == nil { - i = make(indexMap) - m[ino.volume] = i - } - i[ino.index] = watch -} - -// Must run within the I/O thread. -func (w *Watcher) addWatch(pathname string, flags uint64) error { - dir, err := getDir(pathname) - if err != nil { - return err - } - if flags&sysFSONLYDIR != 0 && pathname != dir { - return nil - } - ino, err := getIno(dir) - if err != nil { - return err - } - w.mu.Lock() - watchEntry := w.watches.get(ino) - w.mu.Unlock() - if watchEntry == nil { - if _, e := syscall.CreateIoCompletionPort(ino.handle, w.port, 0, 0); e != nil { - syscall.CloseHandle(ino.handle) - return os.NewSyscallError("CreateIoCompletionPort", e) - } - watchEntry = &watch{ - ino: ino, - path: dir, - names: make(map[string]uint64), - } - w.mu.Lock() - w.watches.set(ino, watchEntry) - w.mu.Unlock() - flags |= provisional - } else { - syscall.CloseHandle(ino.handle) - } - if pathname == dir { - watchEntry.mask |= flags - } else { - watchEntry.names[filepath.Base(pathname)] |= flags - } - if err = w.startRead(watchEntry); err != nil { - return err - } - if pathname == dir { - watchEntry.mask &= ^provisional - } else { - watchEntry.names[filepath.Base(pathname)] &= ^provisional - } - return nil -} - -// Must run within the I/O thread. -func (w *Watcher) remWatch(pathname string) error { - dir, err := getDir(pathname) - if err != nil { - return err - } - ino, err := getIno(dir) - if err != nil { - return err - } - w.mu.Lock() - watch := w.watches.get(ino) - w.mu.Unlock() - if watch == nil { - return fmt.Errorf("can't remove non-existent watch for: %s", pathname) - } - if pathname == dir { - w.sendEvent(watch.path, watch.mask&sysFSIGNORED) - watch.mask = 0 - } else { - name := filepath.Base(pathname) - w.sendEvent(filepath.Join(watch.path, name), watch.names[name]&sysFSIGNORED) - delete(watch.names, name) - } - return w.startRead(watch) -} - -// Must run within the I/O thread. -func (w *Watcher) deleteWatch(watch *watch) { - for name, mask := range watch.names { - if mask&provisional == 0 { - w.sendEvent(filepath.Join(watch.path, name), mask&sysFSIGNORED) - } - delete(watch.names, name) - } - if watch.mask != 0 { - if watch.mask&provisional == 0 { - w.sendEvent(watch.path, watch.mask&sysFSIGNORED) - } - watch.mask = 0 - } -} - -// Must run within the I/O thread. -func (w *Watcher) startRead(watch *watch) error { - if e := syscall.CancelIo(watch.ino.handle); e != nil { - w.Errors <- os.NewSyscallError("CancelIo", e) - w.deleteWatch(watch) - } - mask := toWindowsFlags(watch.mask) - for _, m := range watch.names { - mask |= toWindowsFlags(m) - } - if mask == 0 { - if e := syscall.CloseHandle(watch.ino.handle); e != nil { - w.Errors <- os.NewSyscallError("CloseHandle", e) - } - w.mu.Lock() - delete(w.watches[watch.ino.volume], watch.ino.index) - w.mu.Unlock() - return nil - } - e := syscall.ReadDirectoryChanges(watch.ino.handle, &watch.buf[0], - uint32(unsafe.Sizeof(watch.buf)), false, mask, nil, &watch.ov, 0) - if e != nil { - err := os.NewSyscallError("ReadDirectoryChanges", e) - if e == syscall.ERROR_ACCESS_DENIED && watch.mask&provisional == 0 { - // Watched directory was probably removed - if w.sendEvent(watch.path, watch.mask&sysFSDELETESELF) { - if watch.mask&sysFSONESHOT != 0 { - watch.mask = 0 - } - } - err = nil - } - w.deleteWatch(watch) - w.startRead(watch) - return err - } - return nil -} - -// readEvents reads from the I/O completion port, converts the -// received events into Event objects and sends them via the Events channel. -// Entry point to the I/O thread. -func (w *Watcher) readEvents() { - var ( - n, key uint32 - ov *syscall.Overlapped - ) - runtime.LockOSThread() - - for { - e := syscall.GetQueuedCompletionStatus(w.port, &n, &key, &ov, syscall.INFINITE) - watch := (*watch)(unsafe.Pointer(ov)) - - if watch == nil { - select { - case ch := <-w.quit: - w.mu.Lock() - var indexes []indexMap - for _, index := range w.watches { - indexes = append(indexes, index) - } - w.mu.Unlock() - for _, index := range indexes { - for _, watch := range index { - w.deleteWatch(watch) - w.startRead(watch) - } - } - var err error - if e := syscall.CloseHandle(w.port); e != nil { - err = os.NewSyscallError("CloseHandle", e) - } - close(w.Events) - close(w.Errors) - ch <- err - return - case in := <-w.input: - switch in.op { - case opAddWatch: - in.reply <- w.addWatch(in.path, uint64(in.flags)) - case opRemoveWatch: - in.reply <- w.remWatch(in.path) - } - default: - } - continue - } - - switch e { - case syscall.ERROR_MORE_DATA: - if watch == nil { - w.Errors <- errors.New("ERROR_MORE_DATA has unexpectedly null lpOverlapped buffer") - } else { - // The i/o succeeded but the buffer is full. - // In theory we should be building up a full packet. - // In practice we can get away with just carrying on. - n = uint32(unsafe.Sizeof(watch.buf)) - } - case syscall.ERROR_ACCESS_DENIED: - // Watched directory was probably removed - w.sendEvent(watch.path, watch.mask&sysFSDELETESELF) - w.deleteWatch(watch) - w.startRead(watch) - continue - case syscall.ERROR_OPERATION_ABORTED: - // CancelIo was called on this handle - continue - default: - w.Errors <- os.NewSyscallError("GetQueuedCompletionPort", e) - continue - case nil: - } - - var offset uint32 - for { - if n == 0 { - w.Events <- newEvent("", sysFSQOVERFLOW) - w.Errors <- errors.New("short read in readEvents()") - break - } - - // Point "raw" to the event in the buffer - raw := (*syscall.FileNotifyInformation)(unsafe.Pointer(&watch.buf[offset])) - // TODO: Consider using unsafe.Slice that is available from go1.17 - // https://stackoverflow.com/questions/51187973/how-to-create-an-array-or-a-slice-from-an-array-unsafe-pointer-in-golang - // instead of using a fixed syscall.MAX_PATH buf, we create a buf that is the size of the path name - size := int(raw.FileNameLength / 2) - var buf []uint16 - sh := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) - sh.Data = uintptr(unsafe.Pointer(&raw.FileName)) - sh.Len = size - sh.Cap = size - name := syscall.UTF16ToString(buf) - fullname := filepath.Join(watch.path, name) - - var mask uint64 - switch raw.Action { - case syscall.FILE_ACTION_REMOVED: - mask = sysFSDELETESELF - case syscall.FILE_ACTION_MODIFIED: - mask = sysFSMODIFY - case syscall.FILE_ACTION_RENAMED_OLD_NAME: - watch.rename = name - case syscall.FILE_ACTION_RENAMED_NEW_NAME: - if watch.names[watch.rename] != 0 { - watch.names[name] |= watch.names[watch.rename] - delete(watch.names, watch.rename) - mask = sysFSMOVESELF - } - } - - sendNameEvent := func() { - if w.sendEvent(fullname, watch.names[name]&mask) { - if watch.names[name]&sysFSONESHOT != 0 { - delete(watch.names, name) - } - } - } - if raw.Action != syscall.FILE_ACTION_RENAMED_NEW_NAME { - sendNameEvent() - } - if raw.Action == syscall.FILE_ACTION_REMOVED { - w.sendEvent(fullname, watch.names[name]&sysFSIGNORED) - delete(watch.names, name) - } - if w.sendEvent(fullname, watch.mask&toFSnotifyFlags(raw.Action)) { - if watch.mask&sysFSONESHOT != 0 { - watch.mask = 0 - } - } - if raw.Action == syscall.FILE_ACTION_RENAMED_NEW_NAME { - fullname = filepath.Join(watch.path, watch.rename) - sendNameEvent() - } - - // Move to the next event in the buffer - if raw.NextEntryOffset == 0 { - break - } - offset += raw.NextEntryOffset - - // Error! - if offset >= n { - w.Errors <- errors.New("Windows system assumed buffer larger than it is, events have likely been missed.") - break - } - } - - if err := w.startRead(watch); err != nil { - w.Errors <- err - } - } -} - -func (w *Watcher) sendEvent(name string, mask uint64) bool { - if mask == 0 { - return false - } - event := newEvent(name, uint32(mask)) - select { - case ch := <-w.quit: - w.quit <- ch - case w.Events <- event: - } - return true -} - -func toWindowsFlags(mask uint64) uint32 { - var m uint32 - if mask&sysFSACCESS != 0 { - m |= syscall.FILE_NOTIFY_CHANGE_LAST_ACCESS - } - if mask&sysFSMODIFY != 0 { - m |= syscall.FILE_NOTIFY_CHANGE_LAST_WRITE - } - if mask&sysFSATTRIB != 0 { - m |= syscall.FILE_NOTIFY_CHANGE_ATTRIBUTES - } - if mask&(sysFSMOVE|sysFSCREATE|sysFSDELETE) != 0 { - m |= syscall.FILE_NOTIFY_CHANGE_FILE_NAME | syscall.FILE_NOTIFY_CHANGE_DIR_NAME - } - return m -} - -func toFSnotifyFlags(action uint32) uint64 { - switch action { - case syscall.FILE_ACTION_ADDED: - return sysFSCREATE - case syscall.FILE_ACTION_REMOVED: - return sysFSDELETE - case syscall.FILE_ACTION_MODIFIED: - return sysFSMODIFY - case syscall.FILE_ACTION_RENAMED_OLD_NAME: - return sysFSMOVEDFROM - case syscall.FILE_ACTION_RENAMED_NEW_NAME: - return sysFSMOVEDTO - } - return 0 -} diff --git a/vendor/github.com/go-logr/logr/README.md b/vendor/github.com/go-logr/logr/README.md index 8969526a6..7c7f0c69c 100644 --- a/vendor/github.com/go-logr/logr/README.md +++ b/vendor/github.com/go-logr/logr/README.md @@ -1,6 +1,7 @@ # A minimal logging API for Go [![Go Reference](https://pkg.go.dev/badge/github.com/go-logr/logr.svg)](https://pkg.go.dev/github.com/go-logr/logr) +[![Go Report Card](https://goreportcard.com/badge/github.com/go-logr/logr)](https://goreportcard.com/report/github.com/go-logr/logr) [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/go-logr/logr/badge)](https://securityscorecards.dev/viewer/?platform=github.com&org=go-logr&repo=logr) logr offers an(other) opinion on how Go programs and libraries can do logging diff --git a/vendor/github.com/go-logr/logr/funcr/funcr.go b/vendor/github.com/go-logr/logr/funcr/funcr.go index fb2f866f4..30568e768 100644 --- a/vendor/github.com/go-logr/logr/funcr/funcr.go +++ b/vendor/github.com/go-logr/logr/funcr/funcr.go @@ -236,15 +236,14 @@ func newFormatter(opts Options, outfmt outputFormat) Formatter { // implementation. It should be constructed with NewFormatter. Some of // its methods directly implement logr.LogSink. type Formatter struct { - outputFormat outputFormat - prefix string - values []any - valuesStr string - parentValuesStr string - depth int - opts *Options - group string // for slog groups - groupDepth int + outputFormat outputFormat + prefix string + values []any + valuesStr string + depth int + opts *Options + groupName string // for slog groups + groups []groupDef } // outputFormat indicates which outputFormat to use. @@ -257,6 +256,13 @@ const ( outputJSON ) +// groupDef represents a saved group. The values may be empty, but we don't +// know if we need to render the group until the final record is rendered. +type groupDef struct { + name string + values string +} + // PseudoStruct is a list of key-value pairs that gets logged as a struct. type PseudoStruct []any @@ -264,76 +270,102 @@ type PseudoStruct []any func (f Formatter) render(builtins, args []any) string { // Empirically bytes.Buffer is faster than strings.Builder for this. buf := bytes.NewBuffer(make([]byte, 0, 1024)) + if f.outputFormat == outputJSON { - buf.WriteByte('{') // for the whole line + buf.WriteByte('{') // for the whole record } + // Render builtins vals := builtins if hook := f.opts.RenderBuiltinsHook; hook != nil { vals = hook(f.sanitize(vals)) } - f.flatten(buf, vals, false, false) // keys are ours, no need to escape + f.flatten(buf, vals, false) // keys are ours, no need to escape continuing := len(builtins) > 0 - if f.parentValuesStr != "" { - if continuing { - buf.WriteByte(f.comma()) + // Turn the inner-most group into a string + argsStr := func() string { + buf := bytes.NewBuffer(make([]byte, 0, 1024)) + + vals = args + if hook := f.opts.RenderArgsHook; hook != nil { + vals = hook(f.sanitize(vals)) } - buf.WriteString(f.parentValuesStr) - continuing = true - } + f.flatten(buf, vals, true) // escape user-provided keys - groupDepth := f.groupDepth - if f.group != "" { - if f.valuesStr != "" || len(args) != 0 { - if continuing { - buf.WriteByte(f.comma()) - } - buf.WriteString(f.quoted(f.group, true)) // escape user-provided keys - buf.WriteByte(f.colon()) - buf.WriteByte('{') // for the group - continuing = false - } else { - // The group was empty - groupDepth-- + return buf.String() + }() + + // Render the stack of groups from the inside out. + bodyStr := f.renderGroup(f.groupName, f.valuesStr, argsStr) + for i := len(f.groups) - 1; i >= 0; i-- { + grp := &f.groups[i] + if grp.values == "" && bodyStr == "" { + // no contents, so we must elide the whole group + continue } + bodyStr = f.renderGroup(grp.name, grp.values, bodyStr) } - if f.valuesStr != "" { + if bodyStr != "" { if continuing { buf.WriteByte(f.comma()) } - buf.WriteString(f.valuesStr) - continuing = true + buf.WriteString(bodyStr) } - vals = args - if hook := f.opts.RenderArgsHook; hook != nil { - vals = hook(f.sanitize(vals)) + if f.outputFormat == outputJSON { + buf.WriteByte('}') // for the whole record } - f.flatten(buf, vals, continuing, true) // escape user-provided keys - for i := 0; i < groupDepth; i++ { - buf.WriteByte('}') // for the groups + return buf.String() +} + +// renderGroup returns a string representation of the named group with rendered +// values and args. If the name is empty, this will return the values and args, +// joined. If the name is not empty, this will return a single key-value pair, +// where the value is a grouping of the values and args. If the values and +// args are both empty, this will return an empty string, even if the name was +// specified. +func (f Formatter) renderGroup(name string, values string, args string) string { + buf := bytes.NewBuffer(make([]byte, 0, 1024)) + + needClosingBrace := false + if name != "" && (values != "" || args != "") { + buf.WriteString(f.quoted(name, true)) // escape user-provided keys + buf.WriteByte(f.colon()) + buf.WriteByte('{') + needClosingBrace = true } - if f.outputFormat == outputJSON { - buf.WriteByte('}') // for the whole line + continuing := false + if values != "" { + buf.WriteString(values) + continuing = true + } + + if args != "" { + if continuing { + buf.WriteByte(f.comma()) + } + buf.WriteString(args) + } + + if needClosingBrace { + buf.WriteByte('}') } return buf.String() } -// flatten renders a list of key-value pairs into a buffer. If continuing is -// true, it assumes that the buffer has previous values and will emit a -// separator (which depends on the output format) before the first pair it -// writes. If escapeKeys is true, the keys are assumed to have -// non-JSON-compatible characters in them and must be evaluated for escapes. +// flatten renders a list of key-value pairs into a buffer. If escapeKeys is +// true, the keys are assumed to have non-JSON-compatible characters in them +// and must be evaluated for escapes. // // This function returns a potentially modified version of kvList, which // ensures that there is a value for every key (adding a value if needed) and // that each key is a string (substituting a key if needed). -func (f Formatter) flatten(buf *bytes.Buffer, kvList []any, continuing bool, escapeKeys bool) []any { +func (f Formatter) flatten(buf *bytes.Buffer, kvList []any, escapeKeys bool) []any { // This logic overlaps with sanitize() but saves one type-cast per key, // which can be measurable. if len(kvList)%2 != 0 { @@ -354,7 +386,7 @@ func (f Formatter) flatten(buf *bytes.Buffer, kvList []any, continuing bool, esc } v := kvList[i+1] - if i > 0 || continuing { + if i > 0 { if f.outputFormat == outputJSON { buf.WriteByte(f.comma()) } else { @@ -766,46 +798,17 @@ func (f Formatter) sanitize(kvList []any) []any { // startGroup opens a new group scope (basically a sub-struct), which locks all // the current saved values and starts them anew. This is needed to satisfy // slog. -func (f *Formatter) startGroup(group string) { +func (f *Formatter) startGroup(name string) { // Unnamed groups are just inlined. - if group == "" { + if name == "" { return } - // Any saved values can no longer be changed. - buf := bytes.NewBuffer(make([]byte, 0, 1024)) - continuing := false - - if f.parentValuesStr != "" { - buf.WriteString(f.parentValuesStr) - continuing = true - } - - if f.group != "" && f.valuesStr != "" { - if continuing { - buf.WriteByte(f.comma()) - } - buf.WriteString(f.quoted(f.group, true)) // escape user-provided keys - buf.WriteByte(f.colon()) - buf.WriteByte('{') // for the group - continuing = false - } - - if f.valuesStr != "" { - if continuing { - buf.WriteByte(f.comma()) - } - buf.WriteString(f.valuesStr) - } - - // NOTE: We don't close the scope here - that's done later, when a log line - // is actually rendered (because we have N scopes to close). - - f.parentValuesStr = buf.String() + n := len(f.groups) + f.groups = append(f.groups[:n:n], groupDef{f.groupName, f.valuesStr}) // Start collecting new values. - f.group = group - f.groupDepth++ + f.groupName = name f.valuesStr = "" f.values = nil } @@ -900,7 +903,7 @@ func (f *Formatter) AddValues(kvList []any) { // Pre-render values, so we don't have to do it on each Info/Error call. buf := bytes.NewBuffer(make([]byte, 0, 1024)) - f.flatten(buf, vals, false, true) // escape user-provided keys + f.flatten(buf, vals, true) // escape user-provided keys f.valuesStr = buf.String() } From 3ffb6afb9373fb587edbfedf3e869f7d4de8df28 Mon Sep 17 00:00:00 2001 From: Engin Akdemir Date: Wed, 4 Jun 2025 21:27:36 +0000 Subject: [PATCH 21/25] Fix main.go bad merge from the latest commit hash of Data cache PRs --- cmd/gce-pd-csi-driver/main.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/gce-pd-csi-driver/main.go b/cmd/gce-pd-csi-driver/main.go index 302992ece..243e7acb8 100644 --- a/cmd/gce-pd-csi-driver/main.go +++ b/cmd/gce-pd-csi-driver/main.go @@ -370,7 +370,7 @@ func fetchLssdsForRaiding(lssdCount int) ([]string, error) { return availableLssds[:lssdCount], nil } -func setupDataCache(ctx context.Context, nodeName string) error { +func setupDataCache(ctx context.Context, nodeName string, nodeId string) error { isAlreadyRaided, err := driver.IsRaided() if err != nil { klog.V(4).Infof("Errored while scanning for available LocalSSDs err:%v; continuing Raiding", err) @@ -400,6 +400,11 @@ func setupDataCache(ctx context.Context, nodeName string) error { return fmt.Errorf("Failed to Raid local SSDs, unable to setup Data Cache, got error %v", err) } + // Initializing data cache node (VG checks w/ raided lssd) + if err := driver.InitializeDataCacheNode(nodeId); err != nil { + return err + } + klog.V(4).Infof("LSSD caching is setup for the Data Cache enabled node %s", nodeName) return nil } From f22f0432ba8931981783189893fa21160db1bd84 Mon Sep 17 00:00:00 2001 From: Engin Akdemir Date: Wed, 11 Jun 2025 17:52:29 +0000 Subject: [PATCH 22/25] Check for volume group existing before updating vg metadata Reduce new image tag to below 1.14, bump it from 1.13.2 to 1.13.3 Replace return with continue so we don't break out of infinite loop --- deploy/kubernetes/images/stable-master/image.yaml | 2 +- pkg/gce-pd-csi-driver/cache.go | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/deploy/kubernetes/images/stable-master/image.yaml b/deploy/kubernetes/images/stable-master/image.yaml index abd5c0eca..5d7e74b83 100644 --- a/deploy/kubernetes/images/stable-master/image.yaml +++ b/deploy/kubernetes/images/stable-master/image.yaml @@ -51,5 +51,5 @@ imageTag: name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver # pdImagePlaceholder in test/k8s-integration/main.go is updated automatically with the newTag newName: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver - newTag: "v1.17.2" + newTag: "v1.13.3" --- diff --git a/pkg/gce-pd-csi-driver/cache.go b/pkg/gce-pd-csi-driver/cache.go index f8ac87e58..b5b4c5bf9 100644 --- a/pkg/gce-pd-csi-driver/cache.go +++ b/pkg/gce-pd-csi-driver/cache.go @@ -618,17 +618,23 @@ func watchDiskDetaches(watcher *fsnotify.Watcher, nodeName string, errorCh chan errorCh <- fmt.Errorf("disk update event errored: %v", err) // watch for events case <-watcher.Events: + vgName := getVolumeGroupName(nodeName) + if !checkVgExists(vgName) { + // If the volume group doesn't exist, there's nothing to update. + // Continue to the next event. + continue + } // In case of an event i.e. creation or deletion of any new PV, we update the VG metadata. // This might include some non-LVM changes, no harm in updating metadata multiple times. args := []string{ "--updatemetadata", - getVolumeGroupName(nodeName), + vgName, } _, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgck", args...) if err != nil { klog.Errorf("Error updating volume group's metadata: %v", err) } - reduceVolumeGroup(getVolumeGroupName(nodeName), true) + reduceVolumeGroup(vgName, true) } } } From daa3ff55a2cf1f6aa04729f505bd3e62880c3383 Mon Sep 17 00:00:00 2001 From: Engin Akdemir Date: Mon, 30 Jun 2025 18:13:44 +0000 Subject: [PATCH 23/25] Disabling data cache watcher by default if node details are not available. Conditionally enable data cache on test node pools to fix e2e tests --- cmd/gce-pd-csi-driver/main.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/cmd/gce-pd-csi-driver/main.go b/cmd/gce-pd-csi-driver/main.go index 243e7acb8..9afca6892 100644 --- a/cmd/gce-pd-csi-driver/main.go +++ b/cmd/gce-pd-csi-driver/main.go @@ -228,9 +228,13 @@ func handle() { if err != nil { klog.Fatalf("Failed to set up metadata service: %v", err.Error()) } + isDataCacheEnabledNodePool, err := isDataCacheEnabledNodePool(ctx, *nodeName) + if err != nil { + klog.Fatalf("Failed to get node info from API server: %v", err.Error()) + } nsArgs := driver.NodeServerArgs{ EnableDataCache: *enableDataCacheFlag, - DataCacheEnabledNodePool: isDataCacheEnabledNodePool(ctx, *nodeName), + DataCacheEnabledNodePool: isDataCacheEnabledNodePool, } nodeServer = driver.NewNodeServer(gceDriver, mounter, deviceUtils, meta, statter, nsArgs) if *maxConcurrentFormatAndMount > 0 { @@ -328,14 +332,17 @@ func urlFlag(target **url.URL, name string, usage string) { }) } -func isDataCacheEnabledNodePool(ctx context.Context, nodeName string) bool { +func isDataCacheEnabledNodePool(ctx context.Context, nodeName string) (bool, error) { + if !*enableDataCacheFlag { + return false, nil + } if nodeName != common.TestNode { // disregard logic below when E2E testing. dataCacheLSSDCount, err := driver.GetDataCacheCountFromNodeLabel(ctx, nodeName) - if err != nil || dataCacheLSSDCount == 0 { - return false - } + return dataCacheLSSDCount != 0, err + } else if nodeName == common.TestNode { + return true, nil } - return true + return false, nil } func fetchLssdsForRaiding(lssdCount int) ([]string, error) { From 21a8ecee1ccb25b8f457192c6bf209f75fcfbb95 Mon Sep 17 00:00:00 2001 From: Engin Akdemir Date: Fri, 18 Jul 2025 19:14:01 +0000 Subject: [PATCH 24/25] Fix typo and use strings.Fields for whitespace splitting to fix issues with strings.Split in case of multiple consecutive spaces Check nodeName empty string before feature check Remove log line that also removed on later release branches Minor changes to bring cache.go into the same state in release-1.17 branch --- cmd/gce-pd-csi-driver/main.go | 4 +-- pkg/gce-pd-csi-driver/cache.go | 60 +++++++++++++++++++++++----------- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/cmd/gce-pd-csi-driver/main.go b/cmd/gce-pd-csi-driver/main.go index 9afca6892..d3c0f5a8f 100644 --- a/cmd/gce-pd-csi-driver/main.go +++ b/cmd/gce-pd-csi-driver/main.go @@ -333,7 +333,7 @@ func urlFlag(target **url.URL, name string, usage string) { } func isDataCacheEnabledNodePool(ctx context.Context, nodeName string) (bool, error) { - if !*enableDataCacheFlag { + if !*enableDataCacheFlag || nodeName == "" { return false, nil } if nodeName != common.TestNode { // disregard logic below when E2E testing. @@ -356,7 +356,7 @@ func fetchLssdsForRaiding(lssdCount int) ([]string, error) { return nil, fmt.Errorf("Error listing RAIDed LSSDs %v", err) } - LSSDsWithEmptyMountPoint, err := driver.FetchLSSDsWihtEmptyMountPoint() + LSSDsWithEmptyMountPoint, err := driver.FetchLSSDsWithEmptyMountPoint() if err != nil { return nil, fmt.Errorf("Error listing LSSDs with empty mountpoint: %v", err) } diff --git a/pkg/gce-pd-csi-driver/cache.go b/pkg/gce-pd-csi-driver/cache.go index b5b4c5bf9..fae7efd02 100644 --- a/pkg/gce-pd-csi-driver/cache.go +++ b/pkg/gce-pd-csi-driver/cache.go @@ -7,10 +7,15 @@ import ( "regexp" "strconv" "strings" + "time" csi "github.com/container-storage-interface/spec/lib/go/csi" fsnotify "github.com/fsnotify/fsnotify" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/klog/v2" @@ -42,7 +47,7 @@ func fetchRAIDedLocalSsdPath() (string, error) { return "", fmt.Errorf("Error getting RAIDed device path for Data Cache %v, output:%v", err, string(info)) } infoString := strings.TrimSpace(string(info)) - infoSlice := strings.Split(infoString, " ") + infoSlice := strings.Fields(infoString) // We want to get the second element in the array (sample: ARRAY /dev/md126 metadata=1.2 name=csi-driver-data-cache UUID=*), // which is the path to the RAIDed device @@ -162,7 +167,7 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str } err, isCached := isCachingSetup(mainLvName) if err != nil { - klog.Errorf("faild to check if caching ius setup for LV, continuing to setup caching.") + klog.Errorf("failed to check if caching is setup for LV, continuing to setup caching.") } cacheLvName := getLvName(cacheSuffix, volumeId) if isCached { @@ -199,6 +204,9 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str } info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvcreate", args...) if err != nil { + if strings.Contains(err.Error(), "insufficient free space") { + return mainDevicePath, status.Error(codes.InvalidArgument, fmt.Sprintf("Error setting up cache: %v", err.Error())) + } return mainDevicePath, fmt.Errorf("Errored while creating cache %w: %s", err, info) } } @@ -215,7 +223,7 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str req.GetPublishContext()[common.ContextDataCacheMode], volumeGroupName + "/" + mainLvName, "--chunksize", - chunkSize, // default unit is KiB + chunkSize, "--force", "-y", } @@ -250,8 +258,6 @@ func ValidateDataCacheConfig(dataCacheMode string, dataCacheSize string, ctx con func GetDataCacheCountFromNodeLabel(ctx context.Context, nodeName string) (int, error) { cfg, err := rest.InClusterConfig() - // We want to capture API errors with node label fetching, so return -1 - // in those cases instead of 0. if err != nil { return 0, err } @@ -259,9 +265,8 @@ func GetDataCacheCountFromNodeLabel(ctx context.Context, nodeName string) (int, if err != nil { return 0, err } - node, err := kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + node, err := getNodeWithRetry(ctx, kubeClient, nodeName) if err != nil { - // We could retry, but this error will also crashloop the driver which may be as good a way to retry as any. return 0, err } if val, found := node.GetLabels()[fmt.Sprintf(common.NodeLabelPrefix, common.DataCacheLssdCountLabel)]; found { @@ -272,10 +277,33 @@ func GetDataCacheCountFromNodeLabel(ctx context.Context, nodeName string) (int, klog.V(4).Infof("Number of local SSDs requested for Data Cache: %v", dataCacheCount) return dataCacheCount, nil } - // This will be returned for a non-Data-Cache node pool return 0, nil } +func getNodeWithRetry(ctx context.Context, kubeClient *kubernetes.Clientset, nodeName string) (*v1.Node, error) { + var nodeObj *v1.Node + backoff := wait.Backoff{ + Duration: 1 * time.Second, + Factor: 2.0, + Steps: 5, + } + err := wait.ExponentialBackoffWithContext(ctx, backoff, func() (bool, error) { + node, err := kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + klog.Warningf("Error getting node %s: %v, retrying...\n", nodeName, err) + return false, nil + } + nodeObj = node + klog.V(4).Infof("Successfully retrieved node info %s\n", nodeName) + return true, nil + }) + + if err != nil { + klog.Errorf("Failed to get node %s after retries: %v\n", nodeName, err) + } + return nodeObj, err +} + func FetchRaidedLssdCountForDatacache() (int, error) { raidedPath, err := fetchRAIDedLocalSsdPath() if err != nil { @@ -342,7 +370,7 @@ func FetchAllLssds() ([]string, error) { for _, ssd := range infoList { ssd = strings.TrimSpace(ssd) if strings.HasPrefix(ssd, "/dev/nvme") { - ssdDetails := strings.Split(ssd, " ") + ssdDetails := strings.Fields(ssd) lssd := re.MatchString(ssdDetails[1]) if lssd { diskList = append(diskList, strings.TrimSpace(ssdDetails[0])) @@ -355,7 +383,7 @@ func FetchAllLssds() ([]string, error) { return diskList, nil } -func FetchLSSDsWihtEmptyMountPoint() ([]string, error) { +func FetchLSSDsWithEmptyMountPoint() ([]string, error) { info, err := common.RunCommand("grep", []string{"-E", `^\S+\s*$`} /* pipeCmdArg */, "lsblk", []string{"-o", "NAME,MOUNTPOINT", "-pdn"}...) if err != nil { return nil, fmt.Errorf("Error while fetching disks with no mount point: %v; err:%v", info, err) @@ -376,6 +404,7 @@ func checkVgExists(volumeGroupName string) bool { return false } // Check if the required volume group already exists + klog.Infof("check vg exists output: %v, volumeGroupName: %v", string(info), volumeGroupName) return strings.Contains(string(info), volumeGroupName) } @@ -462,7 +491,6 @@ func createVg(volumeGroupName string, raidedLocalSsds string) error { func reduceVolumeGroup(volumeGroupName string, force bool) { if !checkVgExists(volumeGroupName) { - klog.V(2).Infof("Volume group %v not found, no further action needed", volumeGroupName) return } args := []string{ @@ -618,23 +646,17 @@ func watchDiskDetaches(watcher *fsnotify.Watcher, nodeName string, errorCh chan errorCh <- fmt.Errorf("disk update event errored: %v", err) // watch for events case <-watcher.Events: - vgName := getVolumeGroupName(nodeName) - if !checkVgExists(vgName) { - // If the volume group doesn't exist, there's nothing to update. - // Continue to the next event. - continue - } // In case of an event i.e. creation or deletion of any new PV, we update the VG metadata. // This might include some non-LVM changes, no harm in updating metadata multiple times. args := []string{ "--updatemetadata", - vgName, + getVolumeGroupName(nodeName), } _, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgck", args...) if err != nil { klog.Errorf("Error updating volume group's metadata: %v", err) } - reduceVolumeGroup(vgName, true) + reduceVolumeGroup(getVolumeGroupName(nodeName), true) } } } From 9ecd66d4a143727b9596fab4498f8dea137d46c2 Mon Sep 17 00:00:00 2001 From: Engin Akdemir Date: Wed, 23 Jul 2025 17:42:32 +0000 Subject: [PATCH 25/25] Revert the default value for isDataCacheEnabledNodePool to return true --- cmd/gce-pd-csi-driver/main.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cmd/gce-pd-csi-driver/main.go b/cmd/gce-pd-csi-driver/main.go index d3c0f5a8f..88711cca7 100644 --- a/cmd/gce-pd-csi-driver/main.go +++ b/cmd/gce-pd-csi-driver/main.go @@ -333,16 +333,14 @@ func urlFlag(target **url.URL, name string, usage string) { } func isDataCacheEnabledNodePool(ctx context.Context, nodeName string) (bool, error) { - if !*enableDataCacheFlag || nodeName == "" { + if !*enableDataCacheFlag { return false, nil } - if nodeName != common.TestNode { // disregard logic below when E2E testing. + if len(nodeName) > 0 && nodeName != common.TestNode { // disregard logic below when E2E testing. dataCacheLSSDCount, err := driver.GetDataCacheCountFromNodeLabel(ctx, nodeName) return dataCacheLSSDCount != 0, err - } else if nodeName == common.TestNode { - return true, nil } - return false, nil + return true, nil } func fetchLssdsForRaiding(lssdCount int) ([]string, error) {