diff --git a/Dockerfile b/Dockerfile index 50e4b1ffc..5136146b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ FROM gke.gcr.io/debian-base:bookworm-v1.0.4-gke.2 as debian # Install necessary dependencies # google_nvme_id script depends on the following packages: nvme-cli, xxd, bash -RUN clean-install util-linux e2fsprogs mount ca-certificates udev xfsprogs nvme-cli xxd bash +RUN clean-install util-linux e2fsprogs mount ca-certificates udev xfsprogs nvme-cli xxd bash kmod lvm2 mdadm # Since we're leveraging apt to pull in dependencies, we use `gcr.io/distroless/base` because it includes glibc. FROM gcr.io/distroless/base-debian12 as distroless-base @@ -56,6 +56,35 @@ COPY --from=debian /sbin/e2fsck /sbin/e2fsck COPY --from=debian /sbin/fsck /sbin/fsck COPY --from=debian /sbin/fsck* /sbin/ COPY --from=debian /sbin/fsck.xfs /sbin/fsck.xfs +# Add dependencies for LVM +COPY --from=debian /etc/lvm /lvm-tmp/lvm +COPY --from=debian /lib/systemd/system/blk-availability.service /lib/systemd/system/blk-availability.service +COPY --from=debian /lib/systemd/system/lvm2-lvmpolld.service /lib/systemd/system/lvm2-lvmpolld.service +COPY --from=debian /lib/systemd/system/lvm2-lvmpolld.socket /lib/systemd/system/lvm2-lvmpolld.socket +COPY --from=debian /lib/systemd/system/lvm2-monitor.service /lib/systemd/system/lvm2-monitor.service +COPY --from=debian /lib/udev/rules.d/56-lvm.rules /lib/udev/rules.d/56-lvm.rules +COPY --from=debian /sbin/fsadm /sbin/fsadm +COPY --from=debian /sbin/lvm /sbin/lvm +COPY --from=debian /sbin/lvmdump /sbin/lvmdump +COPY --from=debian /sbin/lvmpolld /sbin/lvmpolld +COPY --from=debian /usr/lib/tmpfiles.d /usr/lib/tmpfiles.d +COPY --from=debian /usr/lib/tmpfiles.d/lvm2.conf /usr/lib/tmpfiles.d/lvm2.conf +COPY --from=debian /sbin/lv* /sbin/ +COPY --from=debian /sbin/pv* /sbin/ +COPY --from=debian /sbin/vg* /sbin/ +COPY --from=debian /bin/lsblk /bin/lsblk +COPY --from=debian /sbin/modprobe /sbin/modprobe +COPY --from=debian /lib/udev /lib/udev +COPY --from=debian /lib/udev/rules.d /lib/udev/rules.d +COPY --from=debian /lib/udev/rules.d/55-dm.rules /lib/udev/rules.d/55-dm.rules +COPY --from=debian /lib/udev/rules.d/60-persistent-storage-dm.rules /lib/udev/rules.d/60-persistent-storage-dm.rules +COPY --from=debian /lib/udev/rules.d/95-dm-notify.rules /lib/udev/rules.d/95-dm-notify.rules +COPY --from=debian /sbin/blkdeactivate /sbin/blkdeactivate +COPY --from=debian /sbin/dmsetup /sbin/dmsetup +COPY --from=debian /sbin/dmstats /sbin/dmstats +COPY --from=debian /bin/ls /bin/ls +# End of dependencies for LVM +COPY --from=debian /sbin/mdadm /sbin/mdadm COPY --from=debian /sbin/mke2fs /sbin/mke2fs COPY --from=debian /sbin/mkfs* /sbin/ COPY --from=debian /sbin/resize2fs /sbin/resize2fs @@ -71,14 +100,20 @@ COPY --from=debian /bin/date /bin/date COPY --from=debian /bin/grep /bin/grep COPY --from=debian /bin/sed /bin/sed COPY --from=debian /bin/ln /bin/ln +COPY --from=debian /bin/cp /bin/cp COPY --from=debian /bin/udevadm /bin/udevadm # Copy shared libraries into distroless base. COPY --from=debian /lib/${LIB_DIR_PREFIX}-linux-gnu/libselinux.so.1 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libdl.so.2 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libpthread.so.0 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libtinfo.so.6 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libe2p.so.2 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libcom_err.so.2 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libdevmapper.so.1.02.1 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libm.so.6 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libc.so.6 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libdevmapper-event.so.1.02.1 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libext2fs.so.2 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libgcc_s.so.1 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/liblzma.so.5 \ @@ -99,11 +134,17 @@ COPY --from=debian /lib/${LIB_DIR_PREFIX}-linux-gnu/libselinux.so.1 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libzstd.so.1 /lib/${LIB_DIR_PREFIX}-linux-gnu/ COPY --from=debian /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libblkid.so.1 \ + /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libsmartcols.so.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libbsd.so.0 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libinih.so.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libmount.so.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libudev.so.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libuuid.so.1 \ + /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libzstd.so.1 \ + /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libaio.so.1 \ + /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libgcrypt.so.20 \ + /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libsystemd.so.0 \ + /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/liblz4.so.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libacl.so.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libattr.so.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libedit.so.2 \ @@ -130,4 +171,5 @@ RUN /print-missing-deps.sh # Final build stage, create the real Docker image with ENTRYPOINT FROM output-image -ENTRYPOINT ["/gce-pd-csi-driver"] +COPY --from=builder /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/initialize-driver.sh /initialize-driver.sh +ENTRYPOINT ["/initialize-driver.sh"] diff --git a/cmd/gce-pd-csi-driver/main.go b/cmd/gce-pd-csi-driver/main.go index 3c7808498..88711cca7 100644 --- a/cmd/gce-pd-csi-driver/main.go +++ b/cmd/gce-pd-csi-driver/main.go @@ -29,7 +29,6 @@ import ( "k8s.io/klog/v2" "k8s.io/utils/strings/slices" - "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/deviceutils" gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute" @@ -72,6 +71,8 @@ var ( formatAndMountTimeout = flag.Duration("format-and-mount-timeout", 1*time.Minute, "The maximum duration of a format and mount operation before another such operation will be started. Used only if --serialize-format-and-mount") fallbackRequisiteZonesFlag = flag.String("fallback-requisite-zones", "", "Comma separated list of requisite zones that will be used if there are not sufficient zones present in requisite topologies when provisioning a disk") enableStoragePoolsFlag = flag.Bool("enable-storage-pools", false, "If set to true, the CSI Driver will allow volumes to be provisioned in Storage Pools") + enableDataCacheFlag = flag.Bool("enable-data-cache", false, "If set to true, the CSI Driver will allow volumes to be provisioned with Data Cache configuration") + nodeName = flag.String("node-name", "", "The node this driver is running on") multiZoneVolumeHandleDiskTypesFlag = flag.String("multi-zone-volume-handle-disk-types", "", "Comma separated list of allowed disk types that can use the multi-zone volumeHandle. Used only if --multi-zone-volume-handle-enable") multiZoneVolumeHandleEnableFlag = flag.Bool("multi-zone-volume-handle-enable", false, "If set to true, the multi-zone volumeHandle feature will be enabled") @@ -122,7 +123,7 @@ func handle() { if version == "" { klog.Fatalf("version must be set at compile time") } - klog.V(2).Infof("Driver vendor version %v", version) + klog.V(4).Infof("Driver vendor version %v", version) // Start tracing as soon as possible if *enableOtelTracing { @@ -209,7 +210,7 @@ func handle() { } initialBackoffDuration := time.Duration(*errorBackoffInitialDurationMs) * time.Millisecond maxBackoffDuration := time.Duration(*errorBackoffMaxDurationMs) * time.Millisecond - controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration, fallbackRequisiteZones, *enableStoragePoolsFlag, multiZoneVolumeHandleConfig, listVolumesConfig) + controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration, fallbackRequisiteZones, *enableStoragePoolsFlag, *enableDataCacheFlag, multiZoneVolumeHandleConfig, listVolumesConfig) } else if *cloudConfigFilePath != "" { klog.Warningf("controller service is disabled but cloud config given - it has no effect") } @@ -227,10 +228,29 @@ func handle() { if err != nil { klog.Fatalf("Failed to set up metadata service: %v", err.Error()) } - nodeServer = driver.NewNodeServer(gceDriver, mounter, deviceUtils, meta, statter) + isDataCacheEnabledNodePool, err := isDataCacheEnabledNodePool(ctx, *nodeName) + if err != nil { + klog.Fatalf("Failed to get node info from API server: %v", err.Error()) + } + nsArgs := driver.NodeServerArgs{ + EnableDataCache: *enableDataCacheFlag, + DataCacheEnabledNodePool: isDataCacheEnabledNodePool, + } + nodeServer = driver.NewNodeServer(gceDriver, mounter, deviceUtils, meta, statter, nsArgs) if *maxConcurrentFormatAndMount > 0 { nodeServer = nodeServer.WithSerializedFormatAndMount(*formatAndMountTimeout, *maxConcurrentFormatAndMount) } + if *enableDataCacheFlag { + if nodeName == nil || *nodeName == "" { + klog.Errorf("Data Cache enabled, but --node-name not passed") + } + if nsArgs.DataCacheEnabledNodePool { + if err := setupDataCache(ctx, *nodeName, nodeServer.MetadataService.GetName()); err != nil { + klog.Errorf("Data Cache setup failed: %v", err) + } + go driver.StartWatcher(*nodeName) + } + } } err = gceDriver.SetupGCEDriver(driverName, version, extraVolumeLabels, extraTags, identityServer, controllerServer, nodeServer) @@ -311,3 +331,85 @@ func urlFlag(target **url.URL, name string, usage string) { return err }) } + +func isDataCacheEnabledNodePool(ctx context.Context, nodeName string) (bool, error) { + if !*enableDataCacheFlag { + return false, nil + } + if len(nodeName) > 0 && nodeName != common.TestNode { // disregard logic below when E2E testing. + dataCacheLSSDCount, err := driver.GetDataCacheCountFromNodeLabel(ctx, nodeName) + return dataCacheLSSDCount != 0, err + } + return true, nil +} + +func fetchLssdsForRaiding(lssdCount int) ([]string, error) { + allLssds, err := driver.FetchAllLssds() + if err != nil { + return nil, fmt.Errorf("Error listing all LSSDs %v", err) + } + + raidedLssds, err := driver.FetchRaidedLssds() + if err != nil { + return nil, fmt.Errorf("Error listing RAIDed LSSDs %v", err) + } + + LSSDsWithEmptyMountPoint, err := driver.FetchLSSDsWithEmptyMountPoint() + if err != nil { + return nil, fmt.Errorf("Error listing LSSDs with empty mountpoint: %v", err) + } + + // We need to ensure the disks to be used for Data Cache are both unRAIDed & not containing mountpoints for ephemeral storage already + availableLssds := slices.Filter(nil, allLssds, func(e string) bool { + return slices.Contains(LSSDsWithEmptyMountPoint, e) && !slices.Contains(raidedLssds, e) + }) + + if len(availableLssds) == 0 { + return nil, fmt.Errorf("No LSSDs available to set up caching") + } + + if len(availableLssds) < lssdCount { + return nil, fmt.Errorf("Not enough LSSDs available to set up caching. Available LSSDs: %v, wanted LSSDs: %v", len(availableLssds), lssdCount) + } + + return availableLssds[:lssdCount], nil +} + +func setupDataCache(ctx context.Context, nodeName string, nodeId string) error { + isAlreadyRaided, err := driver.IsRaided() + if err != nil { + klog.V(4).Infof("Errored while scanning for available LocalSSDs err:%v; continuing Raiding", err) + } else if isAlreadyRaided { + klog.V(4).Infof("Local SSDs are already RAIDed. Skipping Data Cache setup.") + return nil + } + + lssdCount := common.LocalSSDCountForDataCache + if nodeName != common.TestNode { + var err error + lssdCount, err = driver.GetDataCacheCountFromNodeLabel(ctx, nodeName) + if err != nil { + return err + } + if lssdCount == 0 { + klog.V(4).Infof("Data Cache is not enabled on node %v, so skipping caching setup", nodeName) + return nil + } + } + lssdNames, err := fetchLssdsForRaiding(lssdCount) + if err != nil { + klog.Fatalf("Failed to get sufficient SSDs for Data Cache's caching setup: %v", err) + } + klog.V(4).Infof("Raiding local ssds to setup Data Cache: %v", lssdNames) + if err := driver.RaidLocalSsds(lssdNames); err != nil { + return fmt.Errorf("Failed to Raid local SSDs, unable to setup Data Cache, got error %v", err) + } + + // Initializing data cache node (VG checks w/ raided lssd) + if err := driver.InitializeDataCacheNode(nodeId); err != nil { + return err + } + + klog.V(4).Infof("LSSD caching is setup for the Data Cache enabled node %s", nodeName) + return nil +} diff --git a/deploy/kubernetes/base/controller/cluster_setup.yaml b/deploy/kubernetes/base/controller/cluster_setup.yaml index e22c46b55..804cc5e48 100644 --- a/deploy/kubernetes/base/controller/cluster_setup.yaml +++ b/deploy/kubernetes/base/controller/cluster_setup.yaml @@ -199,6 +199,9 @@ rules: verbs: ['use'] resourceNames: - csi-gce-pd-node-psp + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list"] --- kind: ClusterRole diff --git a/deploy/kubernetes/base/controller/controller.yaml b/deploy/kubernetes/base/controller/controller.yaml index e7e5db9d1..5f8713af5 100644 --- a/deploy/kubernetes/base/controller/controller.yaml +++ b/deploy/kubernetes/base/controller/controller.yaml @@ -139,6 +139,7 @@ spec: args: - "--v=5" - "--endpoint=unix:/csi/csi.sock" + - --enable-data-cache env: - name: GOOGLE_APPLICATION_CREDENTIALS value: "/etc/cloud-sa/cloud-sa.json" diff --git a/deploy/kubernetes/base/node_linux/node.yaml b/deploy/kubernetes/base/node_linux/node.yaml index ebf7779cf..b191b2881 100644 --- a/deploy/kubernetes/base/node_linux/node.yaml +++ b/deploy/kubernetes/base/node_linux/node.yaml @@ -46,8 +46,15 @@ spec: - "--v=5" - "--endpoint=unix:/csi/csi.sock" - "--run-controller-service=false" + - "--enable-data-cache" + - "--node-name=$(KUBE_NODE_NAME)" securityContext: privileged: true + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName volumeMounts: - name: kubelet-dir mountPath: /var/lib/kubelet @@ -66,6 +73,8 @@ spec: mountPath: /run/udev - name: sys mountPath: /sys + - name: lib-modules + mountPath: /lib/modules volumes: - name: registration-dir hostPath: @@ -101,6 +110,10 @@ spec: hostPath: path: /sys type: Directory + - name: lib-modules + hostPath: + path: /lib/modules + type: Directory # https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ # See "special case". This will tolerate everything. Node component should # be scheduled on all nodes. diff --git a/deploy/kubernetes/images/stable-master/image.yaml b/deploy/kubernetes/images/stable-master/image.yaml index 56a0ac6ae..5d7e74b83 100644 --- a/deploy/kubernetes/images/stable-master/image.yaml +++ b/deploy/kubernetes/images/stable-master/image.yaml @@ -49,8 +49,7 @@ metadata: name: imagetag-gcepd-driver imageTag: name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver - # Don't change stable image without changing pdImagePlaceholder in - # test/k8s-integration/main.go + # pdImagePlaceholder in test/k8s-integration/main.go is updated automatically with the newTag newName: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver - newTag: "v1.13.2" + newTag: "v1.13.3" --- diff --git a/go.mod b/go.mod index ab0f5fb53..0494ba324 100644 --- a/go.mod +++ b/go.mod @@ -57,8 +57,8 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful v2.9.5+incompatible // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect diff --git a/go.sum b/go.sum index 9bfabf281..2c3135245 100644 --- a/go.sum +++ b/go.sum @@ -1029,8 +1029,8 @@ github.com/frankban/quicktest v1.8.1/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fsouza/fake-gcs-server v0.0.0-20180612165233-e85be23bdaa8/go.mod h1:1/HufuJ+eaDf4KTnYdS6HJMGvMRU8d4cYTuu/1QaBbI= github.com/fsouza/fake-gcs-server v1.19.4/go.mod h1:I0/88nHCASqJJ5M7zVF0zKODkYTcuXFW5J5yajsNJnE= github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= @@ -1071,8 +1071,8 @@ github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= diff --git a/initialize-driver.sh b/initialize-driver.sh new file mode 100755 index 000000000..fe5a615c8 --- /dev/null +++ b/initialize-driver.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +/bin/cp -r /lvm-tmp/lvm /etc/ +/bin/sed -i -e "s/.*allow_mixed_block_sizes = 0.*/ allow_mixed_block_sizes = 1/" /etc/lvm/lvm.conf +/bin/sed -i -e "s/.*udev_sync = 1.*/ udev_sync = 0/" /etc/lvm/lvm.conf +/bin/sed -i -e "s/.*udev_rules = 1.*/ udev_rules = 0/" /etc/lvm/lvm.conf +/bin/sed -i -e "s/.*locking_dir = .*/ locking_dir = \"\/tmp\"/" /etc/lvm/lvm.conf + +/gce-pd-csi-driver "$@" \ No newline at end of file diff --git a/pkg/common/constants.go b/pkg/common/constants.go index 278063b19..104a50b47 100644 --- a/pkg/common/constants.go +++ b/pkg/common/constants.go @@ -32,6 +32,25 @@ const ( // Label that is set on a disk when it is used by a 'multi-zone' VolumeHandle MultiZoneLabel = "goog-gke-multi-zone" + + // Data cache mode + DataCacheModeWriteBack = "writeback" + DataCacheModeWriteThrough = "writethrough" + + ContextDataCacheSize = "data-cache-size" + ContextDataCacheMode = "data-cache-mode" + + // Keys in the publish context + ContexLocalSsdCacheSize = "local-ssd-cache-size" + // Node name for E2E tests + TestNode = "test-node-csi-e2e" + + // Default LSSD count for datacache E2E tests + LocalSSDCountForDataCache = 2 + + // Node label for Data Cache (only applicable to GKE nodes) + NodeLabelPrefix = "cloud.google.com/%s" + DataCacheLssdCountLabel = "gke-data-cache-disk" ) // doc https://cloud.google.com/compute/docs/disks/hyperdisks#max-total-disks-per-vm diff --git a/pkg/common/parameters.go b/pkg/common/parameters.go index 2a1fa4d13..27eae3665 100644 --- a/pkg/common/parameters.go +++ b/pkg/common/parameters.go @@ -18,6 +18,7 @@ package common import ( "fmt" + "strconv" "strings" ) @@ -32,8 +33,13 @@ const ( ParameterAvailabilityClass = "availability-class" ParameterKeyEnableConfidentialCompute = "enable-confidential-storage" ParameterKeyStoragePools = "storage-pools" - ParameterKeyResourceTags = "resource-tags" - ParameterKeyEnableMultiZoneProvisioning = "enable-multi-zone-provisioning" + + // Parameters for Data Cache + ParameterKeyDataCacheSize = "data-cache-size" + ParameterKeyDataCacheMode = "data-cache-mode" + ParameterKeyResourceTags = "resource-tags" + ParameterKeyEnableMultiZoneProvisioning = "enable-multi-zone-provisioning" + ParameterHdHADiskType = "hyperdisk-balanced-high-availability" // Parameters for VolumeSnapshotClass ParameterKeyStorageLocations = "storage-locations" @@ -69,6 +75,15 @@ const ( tagKeyCreatedForSnapshotContentName = "kubernetes.io/created-for/volumesnapshotcontent/name" ) +type DataCacheParameters struct { + // Values: {string} in int64 form + // Default: "" + DataCacheSize string + // Values: writethrough, writeback + // Default: writethrough + DataCacheMode string +} + // DiskParameters contains normalized and defaulted disk parameters type DiskParameters struct { // Values: pd-standard, pd-balanced, pd-ssd, or any other PD disk type. Not validated. @@ -135,7 +150,8 @@ type ParameterProcessor struct { // put them into a well defined struct making sure to default unspecified fields. // extraVolumeLabels are added as labels; if there are also labels specified in // parameters, any matching extraVolumeLabels will be overridden. -func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string]string, extraVolumeLabels map[string]string, extraTags map[string]string) (DiskParameters, error) { +func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string]string, extraVolumeLabels map[string]string, enableDataCache bool, extraTags map[string]string) (DiskParameters, DataCacheParameters, error) { + p := DiskParameters{ DiskType: "pd-standard", // Default ReplicationType: replicationTypeNone, // Default @@ -145,6 +161,12 @@ func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string] ResourceTags: make(map[string]string), // Default } + // Set data cache mode default + d := DataCacheParameters{} + if enableDataCache && parameters[ParameterKeyDataCacheSize] != "" { + d.DataCacheMode = DataCacheModeWriteThrough + } + for k, v := range extraVolumeLabels { p.Labels[k] = v } @@ -179,7 +201,7 @@ func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string] case ParameterKeyLabels: paramLabels, err := ConvertLabelsStringToMap(v) if err != nil { - return p, fmt.Errorf("parameters contain invalid labels parameter: %w", err) + return p, d, fmt.Errorf("parameters contain invalid labels parameter: %w", err) } // Override any existing labels with those from this parameter. for labelKey, labelValue := range paramLabels { @@ -188,19 +210,25 @@ func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string] case ParameterKeyProvisionedIOPSOnCreate: paramProvisionedIOPSOnCreate, err := ConvertStringToInt64(v) if err != nil { - return p, fmt.Errorf("parameters contain invalid provisionedIOPSOnCreate parameter: %w", err) + return p, d, fmt.Errorf("parameters contain invalid provisionedIOPSOnCreate parameter: %w", err) } p.ProvisionedIOPSOnCreate = paramProvisionedIOPSOnCreate case ParameterKeyProvisionedThroughputOnCreate: paramProvisionedThroughputOnCreate, err := ConvertMiStringToInt64(v) if err != nil { - return p, fmt.Errorf("parameters contain invalid provisionedThroughputOnCreate parameter: %w", err) + return p, d, fmt.Errorf("parameters contain invalid provisionedThroughputOnCreate parameter: %w", err) + } + if paramProvisionedThroughputOnCreate < 0 { + return p, d, fmt.Errorf("parameter provisionedThroughputOnCreate cannot be negative") + } + if paramProvisionedThroughputOnCreate < 0 { + return p, d, fmt.Errorf("parameter provisionedThroughputOnCreate cannot be negative") } p.ProvisionedThroughputOnCreate = paramProvisionedThroughputOnCreate case ParameterAvailabilityClass: paramAvailabilityClass, err := ConvertStringToAvailabilityClass(v) if err != nil { - return p, fmt.Errorf("parameters contain invalid availability class parameter: %w", err) + return p, d, fmt.Errorf("parameters contain invalid availability class parameter: %w", err) } if paramAvailabilityClass == ParameterRegionalHardFailoverClass { p.ForceAttach = true @@ -208,37 +236,58 @@ func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string] case ParameterKeyEnableConfidentialCompute: paramEnableConfidentialCompute, err := ConvertStringToBool(v) if err != nil { - return p, fmt.Errorf("parameters contain invalid value for enable-confidential-storage parameter: %w", err) + return p, d, fmt.Errorf("parameters contain invalid value for enable-confidential-storage parameter: %w", err) } if paramEnableConfidentialCompute { // DiskEncryptionKmsKey is needed to enable confidentialStorage if val, ok := parameters[ParameterKeyDiskEncryptionKmsKey]; !ok || !isValidDiskEncryptionKmsKey(val) { - return p, fmt.Errorf("Valid %v is required to enable ConfidentialStorage", ParameterKeyDiskEncryptionKmsKey) + return p, d, fmt.Errorf("Valid %v is required to enable ConfidentialStorage", ParameterKeyDiskEncryptionKmsKey) } } p.EnableConfidentialCompute = paramEnableConfidentialCompute case ParameterKeyStoragePools: if !pp.EnableStoragePools { - return p, fmt.Errorf("parameters contains invalid option %q", ParameterKeyStoragePools) + return p, d, fmt.Errorf("parameters contains invalid option %q", ParameterKeyStoragePools) } storagePools, err := ParseStoragePools(v) if err != nil { - return p, fmt.Errorf("parameters contains invalid value for %s parameter %q: %w", ParameterKeyStoragePools, v, err) + return p, d, fmt.Errorf("parameters contains invalid value for %s parameter %q: %w", ParameterKeyStoragePools, v, err) } p.StoragePools = storagePools + case ParameterKeyDataCacheSize: + if !enableDataCache { + return p, d, fmt.Errorf("data caching enabled: %v; parameters contains invalid option %q", enableDataCache, ParameterKeyDataCacheSize) + } + + paramDataCacheSize, err := ConvertGiStringToInt64(v) + if err != nil { + return p, d, fmt.Errorf("parameters contain invalid dataCacheSize parameter: %w", err) + } + if err := ValidateNonNegativeInt(paramDataCacheSize); err != nil { + return p, d, fmt.Errorf("parameters contains invalid option: %s: %w", ParameterKeyDataCacheSize, err) + } + d.DataCacheSize = strconv.FormatInt(paramDataCacheSize, 10) + case ParameterKeyDataCacheMode: + if !enableDataCache { + return p, d, fmt.Errorf("data caching enabled %v; parameters contains invalid option %q", enableDataCache, ParameterKeyDataCacheSize) + } + if err := ValidateDataCacheMode(v); err != nil { + return p, d, fmt.Errorf("parameters contains invalid option: %s: %w", ParameterKeyDataCacheMode, err) + } + d.DataCacheMode = v case ParameterKeyResourceTags: if err := extractResourceTagsParameter(v, p.ResourceTags); err != nil { - return p, err + return p, d, err } case ParameterKeyEnableMultiZoneProvisioning: if !pp.EnableMultiZone { - return p, fmt.Errorf("parameters contains invalid option %q", ParameterKeyEnableMultiZoneProvisioning) + return p, d, fmt.Errorf("parameters contains invalid option %q", ParameterKeyEnableMultiZoneProvisioning) } paramEnableMultiZoneProvisioning, err := ConvertStringToBool(v) if err != nil { - return p, fmt.Errorf("parameters contain invalid value for %s parameter: %w", ParameterKeyEnableMultiZoneProvisioning, err) + return p, d, fmt.Errorf("parameters contain invalid value for %s parameter: %w", ParameterKeyEnableMultiZoneProvisioning, err) } p.MultiZoneProvisioning = paramEnableMultiZoneProvisioning @@ -246,13 +295,13 @@ func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string] p.Labels[MultiZoneLabel] = "true" } default: - return p, fmt.Errorf("parameters contains invalid option %q", k) + return p, d, fmt.Errorf("parameters contains invalid option %q", k) } } if len(p.Tags) > 0 { p.Tags[tagKeyCreatedBy] = pp.DriverName } - return p, nil + return p, d, nil } func ExtractAndDefaultSnapshotParameters(parameters map[string]string, driverName string, extraTags map[string]string) (SnapshotParameters, error) { diff --git a/pkg/common/parameters_test.go b/pkg/common/parameters_test.go index 75f1c2a16..64cd4da08 100644 --- a/pkg/common/parameters_test.go +++ b/pkg/common/parameters_test.go @@ -25,14 +25,17 @@ import ( func TestExtractAndDefaultParameters(t *testing.T) { tests := []struct { - name string - parameters map[string]string - labels map[string]string - enableStoragePools bool - enableMultiZone bool - extraTags map[string]string - expectParams DiskParameters - expectErr bool + name string + parameters map[string]string + labels map[string]string + enableStoragePools bool + enableDataCache bool + enableMultiZone bool + enableHdHA bool + extraTags map[string]string + expectParams DiskParameters + expectDataCacheParams DataCacheParameters + expectErr bool }{ { name: "defaults", @@ -351,6 +354,55 @@ func TestExtractAndDefaultParameters(t *testing.T) { labels: map[string]string{}, expectErr: true, }, + { + name: "data cache parameters - set default cache mode", + enableDataCache: true, + parameters: map[string]string{ParameterKeyType: "pd-balanced", ParameterKeyReplicationType: "none", ParameterKeyDiskEncryptionKmsKey: "foo/key", ParameterKeyLabels: "key1=value1,key2=value2", ParameterKeyDataCacheSize: "1234Gi"}, + labels: map[string]string{}, + expectParams: DiskParameters{ + DiskType: "pd-balanced", + ReplicationType: "none", + DiskEncryptionKMSKey: "foo/key", + Tags: map[string]string{}, + Labels: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + ResourceTags: map[string]string{}, + }, + expectDataCacheParams: DataCacheParameters{ + DataCacheMode: DataCacheModeWriteThrough, + DataCacheSize: "1234", + }, + }, + { + name: "data cache parameters", + enableDataCache: true, + parameters: map[string]string{ParameterKeyType: "pd-balanced", ParameterKeyReplicationType: "none", ParameterKeyDiskEncryptionKmsKey: "foo/key", ParameterKeyLabels: "key1=value1,key2=value2", ParameterKeyDataCacheSize: "1234Gi", ParameterKeyDataCacheMode: DataCacheModeWriteBack}, + labels: map[string]string{}, + expectParams: DiskParameters{ + DiskType: "pd-balanced", + ReplicationType: "none", + DiskEncryptionKMSKey: "foo/key", + Tags: map[string]string{}, + Labels: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + ResourceTags: map[string]string{}, + }, + expectDataCacheParams: DataCacheParameters{ + DataCacheMode: DataCacheModeWriteBack, + DataCacheSize: "1234", + }, + }, + { + name: "data cache parameters - enableDataCache is false", + enableDataCache: false, + parameters: map[string]string{ParameterKeyType: "pd-balanced", ParameterKeyReplicationType: "none", ParameterKeyDiskEncryptionKmsKey: "foo/key", ParameterKeyLabels: "key1=value1,key2=value2", ParameterKeyDataCacheSize: "1234Gi", ParameterKeyDataCacheMode: DataCacheModeWriteBack}, + labels: map[string]string{}, + expectErr: true, + }, { name: "multi-zone-enable parameters, multi-zone label is set, multi-zone feature enabled", parameters: map[string]string{ParameterKeyType: "hyperdisk-ml", ParameterKeyEnableMultiZoneProvisioning: "true"}, @@ -397,7 +449,7 @@ func TestExtractAndDefaultParameters(t *testing.T) { EnableStoragePools: tc.enableStoragePools, EnableMultiZone: tc.enableMultiZone, } - p, err := pp.ExtractAndDefaultParameters(tc.parameters, tc.labels, tc.extraTags) + p, d, err := pp.ExtractAndDefaultParameters(tc.parameters, tc.labels, tc.enableDataCache, tc.extraTags) if gotErr := err != nil; gotErr != tc.expectErr { t.Fatalf("ExtractAndDefaultParameters(%+v) = %v; expectedErr: %v", tc.parameters, err, tc.expectErr) } @@ -408,6 +460,10 @@ func TestExtractAndDefaultParameters(t *testing.T) { if diff := cmp.Diff(tc.expectParams, p); diff != "" { t.Errorf("ExtractAndDefaultParameters(%+v): -want, +got \n%s", tc.parameters, diff) } + + if diff := cmp.Diff(tc.expectDataCacheParams, d); diff != "" { + t.Errorf("ExtractAndDefaultParameters(%+v) for data cache params: -want, +got \n%s", tc.parameters, diff) + } }) } } diff --git a/pkg/common/runcmd.go b/pkg/common/runcmd.go new file mode 100644 index 000000000..39457dfb1 --- /dev/null +++ b/pkg/common/runcmd.go @@ -0,0 +1,75 @@ +package common + +import ( + "fmt" + "os/exec" + "strings" + + "k8s.io/klog/v2" +) + +const ( + // Error thrown by exec cmd.Run() when process spawned by cmd.Start() completes before cmd.Wait() is called (see - k/k issue #103753) + errNoChildProcesses = "wait: no child processes" +) + +// RunCommand wraps a k8s exec to deal with the no child process error. Same as exec.CombinedOutput. +// On error, the output is included so callers don't need to echo it again. + +func RunCommand(pipeCmd string, pipeCmdArg []string, cmd1 string, execCmdArgs ...string) ([]byte, error) { + execCmd1 := exec.Command(cmd1, execCmdArgs...) + + if pipeCmd != "" { + output, err := execPipeCommand(pipeCmd, pipeCmdArg, execCmd1) + if err != nil { + return nil, fmt.Errorf("%s %s failed here: %w; output: %s", pipeCmd, pipeCmdArg, err, string(output)) + } + return output, nil + } + output, err := execCmd1.CombinedOutput() + if err != nil { + err = checkError(err, *execCmd1) + return nil, fmt.Errorf("%s %s failed here 2: %w; output: %s", cmd1, strings.Join(execCmdArgs, " "), err, string(output)) + } + + return output, nil +} + +func checkError(err error, execCmd exec.Cmd) error { + if err.Error() == errNoChildProcesses { + if execCmd.ProcessState.Success() { + // If the process succeeded, this can be ignored, see k/k issue #103753 + return nil + } + // Get actual error + klog.Infof("Errored here") + err = &exec.ExitError{ProcessState: execCmd.ProcessState} + } + return err +} +func execPipeCommand(pipeCmd string, pipeCmdArg []string, execCmd1 *exec.Cmd) ([]byte, error) { + + execPipeCmd := exec.Command(pipeCmd, pipeCmdArg...) + stdoutPipe, err := execCmd1.StdoutPipe() + if err != nil { + klog.Errorf("failed command %v: got error:%v", execCmd1, err) + } + err = execCmd1.Start() + if err != nil { + klog.Infof("errored running command %v; error %v; ", execCmd1, err) + } + defer stdoutPipe.Close() + + execPipeCmd.Stdin = stdoutPipe + output, err := execPipeCmd.CombinedOutput() + if err != nil { + // Some commands (such as grep) will return an error with exit status of 1 + if len(output) == 0 && err.(*exec.ExitError).ExitCode() == 1 { + return output, nil + } + err = checkError(err, *execPipeCmd) + return nil, fmt.Errorf("%s failed: %w; output: %s", execPipeCmd, err, string(output)) + } + + return output, nil +} diff --git a/pkg/common/utils.go b/pkg/common/utils.go index 1789dfd0c..8a1229d02 100644 --- a/pkg/common/utils.go +++ b/pkg/common/utils.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "hash/fnv" "net/http" "regexp" "slices" @@ -78,6 +79,7 @@ const ( // Full or partial URL of the zone resource, in the format: // projects/{project}/zones/{zone} zoneURIPattern = "projects/[^/]+/zones/([^/]+)$" + alphanums = "bcdfghjklmnpqrstvwxz2456789" ) var ( @@ -101,6 +103,8 @@ var ( http.StatusConflict: codes.FailedPrecondition, } + validDataCacheMode = []string{DataCacheModeWriteBack, DataCacheModeWriteThrough} + // Regular expressions for validating parent_id, key and value of a resource tag. regexParent = regexp.MustCompile(`(^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$)`) regexKey = regexp.MustCompile(`^[a-zA-Z0-9]([0-9A-Za-z_.-]{0,61}[a-zA-Z0-9])?$`) @@ -392,6 +396,15 @@ func ConvertMiStringToInt64(str string) (int64, error) { return volumehelpers.RoundUpToMiB(quantity) } +// ConvertGiStringToInt64 converts a GiB string to int64 +func ConvertGiStringToInt64(str string) (int64, error) { + quantity, err := resource.ParseQuantity(str) + if err != nil { + return -1, err + } + return volumehelpers.RoundUpToGiB(quantity) +} + // ConvertStringToBool converts a string to a boolean. func ConvertStringToBool(str string) (bool, error) { switch strings.ToLower(str) { @@ -684,6 +697,29 @@ func VolumeIdAsMultiZone(volumeId string) (string, error) { return strings.Join(splitId, "/"), nil } +func StringInSlice(s string, list []string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} + +func ValidateDataCacheMode(s string) error { + if StringInSlice(s, validDataCacheMode) { + return nil + } + return fmt.Errorf("invalid data-cache-mode %s. Only \"writeback\" and \"writethrough\" is a valid input", s) +} + +func ValidateNonNegativeInt(n int64) error { + if n <= 0 { + return fmt.Errorf("Input should be set to > 0, got %d", n) + } + return nil +} + // NewLimiter returns a token bucket based request rate limiter after initializing // the passed values for limit, burst (or token bucket) size. If opted for emptyBucket // all initial tokens are reserved for the first burst. @@ -710,3 +746,16 @@ func MapNumber(num int64) int64 { } return 0 } + +// shortString is inspired by k8s.io/apimachinery/pkg/util/rand.SafeEncodeString, but takes data from a hash. +func ShortString(s string) string { + hasher := fnv.New128a() + hasher.Write([]byte(s)) + sum := hasher.Sum([]byte{}) + const sz = 8 + short := make([]byte, sz) + for i := 0; i < sz; i++ { + short[i] = alphanums[int(sum[i])%len(alphanums)] + } + return string(short) +} diff --git a/pkg/common/utils_test.go b/pkg/common/utils_test.go index 0df0ed1d1..dbca3d789 100644 --- a/pkg/common/utils_test.go +++ b/pkg/common/utils_test.go @@ -1044,6 +1044,108 @@ func TestConvertMiStringToInt64(t *testing.T) { } } +func TestConvertGiStringToInt64(t *testing.T) { + tests := []struct { + desc string + inputStr string + expInt64 int64 + expectError bool + }{ + { + desc: "valid number string", + inputStr: "10000", + expInt64: 1, + expectError: false, + }, + { + desc: "round Ki to GiB", + inputStr: "1000000Ki", + expInt64: 1, + expectError: false, + }, + { + desc: "round k to GiB", + inputStr: "1000000k", + expInt64: 1, + expectError: false, + }, + { + desc: "round Mi to GiB", + inputStr: "1000Mi", + expInt64: 1, + expectError: false, + }, + { + desc: "round M to GiB", + inputStr: "1000M", + expInt64: 1, + expectError: false, + }, + { + desc: "round G to GiB", + inputStr: "1000G", + expInt64: 932, + expectError: false, + }, + { + desc: "round Gi to GiB - most common case", + inputStr: "1234Gi", + expInt64: 1234, + expectError: false, + }, + { + desc: "round decimal to GiB", + inputStr: "1.2Gi", + expInt64: 2, + expectError: false, + }, + { + desc: "round big value to GiB", + inputStr: "8191Pi", + expInt64: 8588886016, + expectError: false, + }, + { + desc: "invalid empty string", + inputStr: "", + expInt64: 0, + expectError: true, + }, + { + desc: "invalid KiB string", + inputStr: "10KiB", + expInt64: 10000, + expectError: true, + }, + { + desc: "invalid GB string", + inputStr: "10GB", + expInt64: 0, + expectError: true, + }, + { + desc: "invalid string", + inputStr: "ew%65", + expInt64: 0, + expectError: true, + }, + } + for _, tc := range tests { + t.Run(tc.desc, func(t *testing.T) { + actualInt64, err := ConvertGiStringToInt64(tc.inputStr) + if err != nil && !tc.expectError { + t.Errorf("Got error %v converting string to int64 %s; expect no error", err, tc.inputStr) + } + if err == nil && tc.expectError { + t.Errorf("Got no error converting string to int64 %s; expect an error", tc.inputStr) + } + if err == nil && actualInt64 != tc.expInt64 { + t.Errorf("Got %d for converting string to int64; expect %d", actualInt64, tc.expInt64) + } + }) + } +} + func TestConvertStringToBool(t *testing.T) { tests := []struct { desc string @@ -1657,6 +1759,106 @@ func TestUnorderedSlicesEqual(t *testing.T) { } } +func TestStringInSlice(t *testing.T) { + testCases := []struct { + name string + inputStr string + inputSlice []string + expectedInSlice bool + }{ + { + name: "string is in the slice", + inputStr: "in slice", + inputSlice: []string{"in slice", "other string"}, + expectedInSlice: true, + }, + { + name: "string is NOT in the slice", + inputStr: "not in slice", + inputSlice: []string{"other string"}, + }, + } + + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + actualResult := StringInSlice(tc.inputStr, tc.inputSlice) + if actualResult != tc.expectedInSlice { + t.Errorf("Expect value is %v but got %v. inputStr is %s, inputSlice is %v", tc.expectedInSlice, actualResult, tc.inputStr, tc.inputSlice) + } + } +} + +func TestValidateDataCacheMode(t *testing.T) { + testCases := []struct { + name string + inputStr string + expectError bool + }{ + { + name: "valid input - writethrough", + inputStr: "writethrough", + }, + { + name: "valid input - writeback", + inputStr: "writeback", + }, + { + name: "invalid input", + inputStr: "write-back not valid", + expectError: true, + }, + } + + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + err := ValidateDataCacheMode(tc.inputStr) + if err != nil && !tc.expectError { + t.Errorf("Got error %v validate data cache mode %s; expect no error", err, tc.inputStr) + } + + if err == nil && tc.expectError { + t.Errorf("Got no error validate data cache mode %s; expect an error", tc.inputStr) + } + } + +} + +func TestValidateNonNegativeInt(t *testing.T) { + testCases := []struct { + name string + cacheSize int64 + expectError bool + }{ + { + name: "valid input - positive cache size", + cacheSize: 100000, + }, + { + name: "invalid input - cachesize 0", + cacheSize: 0, + expectError: true, + }, + { + name: "invalid input - negative cache size", + cacheSize: -100, + expectError: true, + }, + } + + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + err := ValidateNonNegativeInt(tc.cacheSize) + if err != nil && !tc.expectError { + t.Errorf("Got error %v validate data cache mode %d; expect no error", err, tc.cacheSize) + } + + if err == nil && tc.expectError { + t.Errorf("Got no error validate data cache mode %d; expect an error", tc.cacheSize) + } + } + +} + func TestParseZoneFromURI(t *testing.T) { testcases := []struct { name string diff --git a/pkg/deviceutils/device-utils.go b/pkg/deviceutils/device-utils.go index c9ce89885..096e157a3 100644 --- a/pkg/deviceutils/device-utils.go +++ b/pkg/deviceutils/device-utils.go @@ -231,7 +231,7 @@ func (m *deviceUtils) VerifyDevicePath(devicePaths []string, deviceName string) devicePath, innerErr = existingDevicePath(devicePaths) if innerErr != nil { e := fmt.Errorf("for disk %s failed to check for existing device path: %w", deviceName, innerErr) - klog.Errorf(e.Error()) + klog.Errorf("Error: %s", e.Error()) return false, e } @@ -243,7 +243,7 @@ func (m *deviceUtils) VerifyDevicePath(devicePaths []string, deviceName string) innerErr := udevadmTriggerForDiskIfExists(deviceName) if innerErr != nil { e := fmt.Errorf("for disk %s failed to trigger udevadm fix of non existent device path: %w", deviceName, innerErr) - klog.Errorf(e.Error()) + klog.Errorf("Error: %s", e.Error()) return false, e } // Go to next retry loop to get the deviceName again after @@ -256,7 +256,7 @@ func (m *deviceUtils) VerifyDevicePath(devicePaths []string, deviceName string) devFsPath, innerErr := filepath.EvalSymlinks(devicePath) if innerErr != nil { e := fmt.Errorf("filepath.EvalSymlinks(%q) failed: %w", devicePath, innerErr) - klog.Errorf(e.Error()) + klog.Errorf("Error: %s", e.Error()) return false, e } klog.V(4).Infof("For disk %s the /dev/* path is %s for disk/by-id path %s", deviceName, devFsPath, devicePath) @@ -264,7 +264,7 @@ func (m *deviceUtils) VerifyDevicePath(devicePaths []string, deviceName string) devFsSerial, innerErr := getDevFsSerial(devFsPath) if innerErr != nil { e := fmt.Errorf("couldn't get serial number for disk %s at device path %s: %w", deviceName, devFsPath, innerErr) - klog.Errorf(e.Error()) + klog.Errorf("Error: %s", e.Error()) return false, e } klog.V(4).Infof("For disk %s, device path %s, found serial number %s", deviceName, devFsPath, devFsSerial) @@ -281,7 +281,7 @@ func (m *deviceUtils) VerifyDevicePath(devicePaths []string, deviceName string) innerErr = udevadmTriggerForDiskIfExists(deviceName) if innerErr != nil { e := fmt.Errorf("failed to trigger udevadm fix of misconfigured disk for %q: %w", deviceName, innerErr) - klog.Errorf(e.Error()) + klog.Errorf("Error: %s", e.Error()) return false, e } // Go to next retry loop to get the deviceName again after diff --git a/pkg/gce-pd-csi-driver/cache.go b/pkg/gce-pd-csi-driver/cache.go new file mode 100644 index 000000000..fae7efd02 --- /dev/null +++ b/pkg/gce-pd-csi-driver/cache.go @@ -0,0 +1,735 @@ +package gceGCEDriver + +import ( + "context" + "fmt" + "math" + "regexp" + "strconv" + "strings" + "time" + + csi "github.com/container-storage-interface/spec/lib/go/csi" + fsnotify "github.com/fsnotify/fsnotify" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" +) + +const ( + cacheSuffix = "csi-fast" + mainLvSuffix = "csi-main" + raidedLocalSsdName = "csi-driver-data-cache" + raidMode = "0" + maxAllowedChunks int64 = 1000000 // This is the max allowed chunks for LVM + GiB float64 = 1024 * 1024 * 1024 + KiB float64 = 1024 +) + +var ( + maxChunkSize float64 = 1 * GiB // Max allowed chunk size as per LVM documentation + minChunkSize float64 = 160 * KiB // This is randomly selected, we need a multiple of 32KiB, the default size would be too small for caching https://man7.org/linux/man-pages/man8/lvcreate.8.html (--chunksize) +) + +func fetchRAIDedLocalSsdPath() (string, error) { + args := []string{ + "--detail", + "--scan", + } + info, err := common.RunCommand("grep", []string{raidedLocalSsdName}, "mdadm", args...) + if err != nil || len(info) == 0 { + return "", fmt.Errorf("Error getting RAIDed device path for Data Cache %v, output:%v", err, string(info)) + } + infoString := strings.TrimSpace(string(info)) + infoSlice := strings.Fields(infoString) + + // We want to get the second element in the array (sample: ARRAY /dev/md126 metadata=1.2 name=csi-driver-data-cache UUID=*), + // which is the path to the RAIDed device + return infoSlice[1], nil +} + +func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId string) (string, error) { + + // The device path may have changed after rebooting, so we need to fetch the path again + raidedLocalSsdPath, err := fetchRAIDedLocalSsdPath() + if err != nil { + return "", err + } + + volumeId := req.GetVolumeId() + volumeGroupName := getVolumeGroupName(nodeId) + mainDevicePath := "/dev/" + volumeGroupName + "/" + getLvName(mainLvSuffix, volumeId) + mainLvName := getLvName(mainLvSuffix, volumeId) + klog.V(4).Infof("Volume group available on node %v ", volumeGroupName) + vgExists := checkVgExists(volumeGroupName) + if vgExists { + // Clean up Volume Group before adding the PD + reduceVolumeGroup(volumeGroupName, true) + } else { + err := createVg(volumeGroupName, raidedLocalSsdPath) + if err != nil { + return mainDevicePath, err + } + } + + // Check if the Physical Volume(PV) is part of some other volume group + args := []string{ + "--select", + "pv_name=" + devicePath, + "-o", + "vg_name", + } + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "pvs", args...) + if err != nil { + klog.Errorf("errored while checking physical volume details %v: %s", err, info) + // On error info contains the error message which we cannot use for further steps + info = nil + } + + infoString := strings.TrimSpace(strings.ReplaceAll(string(info), "\n", " ")) + infoString = strings.ReplaceAll(infoString, ".", "") + infoString = strings.ReplaceAll(infoString, "\"", "") + infoSlice := strings.Split(strings.TrimSpace(infoString), " ") + vgNameForPv := strings.TrimSpace(infoSlice[(len(infoSlice) - 1)]) + klog.V(4).Infof("Physical volume is part of Volume group: %v", vgNameForPv) + if vgNameForPv == volumeGroupName { + klog.V(4).Infof("Physical Volume(PV) already exists in the Volume Group %v", volumeGroupName) + } else if vgNameForPv != "VG" && vgNameForPv != "" { + + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgchange", []string{"-an", vgNameForPv}...) + if err != nil { + klog.Errorf("Errored while deactivating VG %v: err: %v: %s", vgNameForPv, err, info) + } + // CLean up volume group to remove any dangling PV refrences + reduceVolumeGroup(vgNameForPv, false) + _, isCached := isCachingSetup(mainLvName) + // We will continue to uncache even if it errors to check caching as it is not a terminal issue. + + if isCached { + // Uncache LV + args = []string{ + "--uncache", + vgNameForPv + "/" + mainLvName, + "--force", + "-y", // force remove cache without flushing data + } + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvconvert", args...) + if err != nil { + return "", fmt.Errorf("errored while uncaching main LV. %v: %s", err, info) + } + // CLean up volume group to remove any dangling PV refrences + reduceVolumeGroup(vgNameForPv, false) + } + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgmerge", []string{volumeGroupName, vgNameForPv}...) + if err != nil { + return "", fmt.Errorf("Errored while merging the PV Volume group %s into %s %v: %s", vgNameForPv, volumeGroupName, err, info) + } + + } else { + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgextend", []string{volumeGroupName, devicePath}...) + if err != nil { + return "", fmt.Errorf("Errored while extending Volume group to add PV %v, error: %v: %s", devicePath, err, info) + } + } + + // Create LV if not already created + args = []string{ + "--select", + "vg_name=" + volumeGroupName, + "-o", + "lv_name", + } + lvList, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvs", args...) + if err != nil { + return mainDevicePath, fmt.Errorf("Errored while checking logical volume for the device %s %w: %s", devicePath, err, info) + } + if !strings.Contains(string(lvList), mainLvName) { + args = []string{ + "--yes", + "-n", + mainLvName, + "-l", + "100%PVS", // Use 100% of the PV + volumeGroupName, + devicePath, + } + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvcreate", args...) + if err != nil { + return mainDevicePath, fmt.Errorf("Errored setting up logical volume for the volume %s %w: %s", devicePath, err, info) + } + + } + err, isCached := isCachingSetup(mainLvName) + if err != nil { + klog.Errorf("failed to check if caching is setup for LV, continuing to setup caching.") + } + cacheLvName := getLvName(cacheSuffix, volumeId) + if isCached { + // Validate that cache is setup for required size + klog.V(4).Infof("Assuming valid data cache size and mode, resizing cache is not supported") + } else { + cacheSize := req.GetPublishContext()[common.ContextDataCacheSize] + maxChunkSizeStr := strconv.FormatInt(int64(maxChunkSize/KiB), 10) + var chunkSize string + cachePvSize, err := fetchPvSizeGiB() + if err != nil { + klog.Errorf("Errored while fetching PV size, got %v, falling back to default chunkSize of %v", err, maxChunkSize) + chunkSize = maxChunkSizeStr + } else { + chunkSize, err = fetchChunkSizeKiB(cachePvSize) + if err != nil { + klog.Errorf("Errored to fetch cache size, verify the data-cache-size is valid: got %v, error: %q", chunkSize, err) + chunkSize = maxChunkSizeStr + } + } + // Check if LV exists + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvs", args...) + lvExists := strings.Contains(string(info), cacheLvName) + if !lvExists { + args = []string{ + "--yes", + "-n", + cacheLvName, + "-L", + // ConvertGiStringToInt64 converts the input size to GiB so default to "g" for cache size - LVM g|G is GiB. + cacheSize + "g", + volumeGroupName, + raidedLocalSsdPath, + } + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvcreate", args...) + if err != nil { + if strings.Contains(err.Error(), "insufficient free space") { + return mainDevicePath, status.Error(codes.InvalidArgument, fmt.Sprintf("Error setting up cache: %v", err.Error())) + } + return mainDevicePath, fmt.Errorf("Errored while creating cache %w: %s", err, info) + } + } + + // Once caching is setup, link the PD to cache + args = []string{ + "--type", + "cache", + "--cachevol", + cacheLvName, + "--zero", + "y", + "--cachemode", + req.GetPublishContext()[common.ContextDataCacheMode], + volumeGroupName + "/" + mainLvName, + "--chunksize", + chunkSize, + "--force", + "-y", + } + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvconvert", args...) + if err != nil { + return mainDevicePath, fmt.Errorf("Errored while setting up caching for volume %s %w: %s", devicePath, err, info) + } + } + + // activate all the LVs in the Volume group + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgchange", []string{"-ay", volumeGroupName}...) + if err != nil { + // The logical volumes would not be accessible if the group is not activated + return mainDevicePath, fmt.Errorf("Failed to activate volume group %v %v:%s", volumeGroupName, err, info) + } + return mainDevicePath, nil +} + +func ValidateDataCacheConfig(dataCacheMode string, dataCacheSize string, ctx context.Context) error { + if dataCacheMode != "" && dataCacheSize != "" { + isAlreadyRaided, err := IsRaided() + if err != nil { + return fmt.Errorf("Local SSDs are not setup for caching; got error: %v", err) + } + if !isAlreadyRaided { + return fmt.Errorf("Local SSDs are not setup for caching") + } + return nil + } + return fmt.Errorf("Data Cache is not enabled for PVC (data-cache-size: %v, data-cache-mode: %v). Please set both parameters in StorageClass to enable caching", dataCacheSize, dataCacheMode) +} + +func GetDataCacheCountFromNodeLabel(ctx context.Context, nodeName string) (int, error) { + cfg, err := rest.InClusterConfig() + if err != nil { + return 0, err + } + kubeClient, err := kubernetes.NewForConfig(cfg) + if err != nil { + return 0, err + } + node, err := getNodeWithRetry(ctx, kubeClient, nodeName) + if err != nil { + return 0, err + } + if val, found := node.GetLabels()[fmt.Sprintf(common.NodeLabelPrefix, common.DataCacheLssdCountLabel)]; found { + dataCacheCount, err := strconv.Atoi(val) + if err != nil { + return 0, fmt.Errorf("Error getting Data Cache's LSSD count from node label: %v", err) + } + klog.V(4).Infof("Number of local SSDs requested for Data Cache: %v", dataCacheCount) + return dataCacheCount, nil + } + return 0, nil +} + +func getNodeWithRetry(ctx context.Context, kubeClient *kubernetes.Clientset, nodeName string) (*v1.Node, error) { + var nodeObj *v1.Node + backoff := wait.Backoff{ + Duration: 1 * time.Second, + Factor: 2.0, + Steps: 5, + } + err := wait.ExponentialBackoffWithContext(ctx, backoff, func() (bool, error) { + node, err := kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + klog.Warningf("Error getting node %s: %v, retrying...\n", nodeName, err) + return false, nil + } + nodeObj = node + klog.V(4).Infof("Successfully retrieved node info %s\n", nodeName) + return true, nil + }) + + if err != nil { + klog.Errorf("Failed to get node %s after retries: %v\n", nodeName, err) + } + return nodeObj, err +} + +func FetchRaidedLssdCountForDatacache() (int, error) { + raidedPath, err := fetchRAIDedLocalSsdPath() + if err != nil { + return 0, err + } + args := []string{ + "--detail", + raidedPath, + } + info, err := common.RunCommand("grep", []string{"Raid Devices"}, "mdadm", args...) + if err != nil { + return 0, fmt.Errorf("Error getting RAIDed devices for Data Cache") + } + if len(info) != 0 { + raidedDeviceInfo := strings.Split(strings.TrimSpace(string(info)), ":") + // raidedDeviceInfo should be in "Raid Devices : X" format + raidedDeviceCount, _ := strconv.Atoi(strings.TrimSpace(raidedDeviceInfo[1])) + return raidedDeviceCount, nil + } + return 0, nil +} + +func FetchRaidedLssds() ([]string, error) { + raidedLssdList := []string{} + + args := []string{ + "--detail", + "--scan", + "--export", + } + + info, err := common.RunCommand("grep", []string{"/dev"}, "mdadm", args...) + if err != nil { + return nil, fmt.Errorf("error fetching RAIDed LSSDs: %v; err:%v", info, err) + } + + if len(info) != 0 { + infoList := strings.Split(strings.TrimSpace(string(info)), "\n") + for _, ssd := range infoList { + ssdInfo := strings.TrimSpace(ssd) + // SSD name comes after "=" on each output line (e.g. MD_DEVICE_dev_nvme3n1_DEV=/dev/nvme3n1) + ssdName := strings.Split(ssdInfo, "=")[1] + raidedLssdList = append(raidedLssdList, ssdName) + } + } + + klog.V(4).Infof("Raided NVME list %v", raidedLssdList) + + return raidedLssdList, nil +} + +func FetchAllLssds() ([]string, error) { + diskList := []string{} + + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipeCmdArg */, "lsblk", []string{"-o", "NAME,MODEL", "-p", "-d", "-n"}...) + if err != nil { + return nil, fmt.Errorf("errored while fetching NVME disks info: %v; err:%v", info, err) + } + infoList := strings.Split(strings.TrimSpace(string(info)), "\n") + re, err := regexp.Compile("nvme_card([0-9]+)?$") + if err != nil { + klog.V(4).ErrorS(err, "Errored while compiling to check PD or LSSD") + } + for _, ssd := range infoList { + ssd = strings.TrimSpace(ssd) + if strings.HasPrefix(ssd, "/dev/nvme") { + ssdDetails := strings.Fields(ssd) + lssd := re.MatchString(ssdDetails[1]) + if lssd { + diskList = append(diskList, strings.TrimSpace(ssdDetails[0])) + } + } + } + + klog.V(4).Infof("NVME list %v", diskList) + + return diskList, nil +} + +func FetchLSSDsWithEmptyMountPoint() ([]string, error) { + info, err := common.RunCommand("grep", []string{"-E", `^\S+\s*$`} /* pipeCmdArg */, "lsblk", []string{"-o", "NAME,MOUNTPOINT", "-pdn"}...) + if err != nil { + return nil, fmt.Errorf("Error while fetching disks with no mount point: %v; err:%v", info, err) + } + infoList := strings.Split(string(info), "\n") + diskList := []string{} + for _, ssd := range infoList { + diskList = append(diskList, strings.TrimSpace(ssd)) + } + return diskList, nil +} + +func checkVgExists(volumeGroupName string) bool { + args := []string{} + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgscan", args...) + if err != nil { + klog.Errorf("Errored while checking if volume group exists %v: %s", err, info) + return false + } + // Check if the required volume group already exists + klog.Infof("check vg exists output: %v, volumeGroupName: %v", string(info), volumeGroupName) + return strings.Contains(string(info), volumeGroupName) +} + +func cleanupCache(volumeId string, nodeId string) error { + + volumeGroupName := getVolumeGroupName(nodeId) + if !checkVgExists(volumeGroupName) { + klog.V(4).Infof("Volume group %s not found, no cache clean up needed", volumeGroupName) + // If volume group doesn't exist then there's nothing to uncache + return nil + } + reduceVolumeGroup(volumeGroupName, true) + mainLvName := getLvName(mainLvSuffix, volumeId) + if !checkLvExists(mainLvName) { + klog.V(4).Infof("Logical volume %s not found, assuming caching wasn't setup for the PVC %s or is cleaned up", mainLvName, volumeId) + // If logical volume doesn't exist then there's nothing to uncache + return nil + } + args := []string{ + "-an", + "/dev/" + volumeGroupName + "/" + mainLvName, + } + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvchange", args...) + if err != nil { + return fmt.Errorf("Failed to deactivate volume for uncaching %s %v: %s", volumeId, err, info) + } + args = []string{ + "--uncache", + volumeGroupName + "/" + mainLvName, + "-y", + } + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvconvert", args...) + if err != nil { + return fmt.Errorf("Failed to uncache volume %s %w: %s", volumeId, err, info) + } + return nil +} + +func checkLvExists(lvName string) bool { + args := []string{} + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "lvscan", args...) + if err != nil { + klog.Errorf("Errored while checking if logical volume exists for %s %v: %s", lvName, err, info) + return false + } + // Check if the required logical volume already exists + return strings.Contains(string(info), lvName) +} + +func getVolumeGroupName(nodePath string) string { + nodeSlice := strings.Split(nodePath, "/") + nodeId := nodeSlice[len(nodeSlice)-1] + nodeHash := common.ShortString(nodeId) + return fmt.Sprintf("csi-vg-%s", nodeHash) +} + +func getLvName(suffix string, volumeId string) string { + pvcNameStringSlice := strings.Split(volumeId, "/") + pvcName := pvcNameStringSlice[len(pvcNameStringSlice)-1] + return fmt.Sprintf("%s-%s", suffix, pvcName) +} + +func createVg(volumeGroupName string, raidedLocalSsds string) error { + args := []string{ + "--zero", + "y", + volumeGroupName, + raidedLocalSsds, + "-v", + } + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgcreate", args...) + if err != nil { + return fmt.Errorf("Volume group creation failed %w: %s", err, info) + } + klog.V(4).Infof("Volume group creation succeeded for %v", volumeGroupName) + + args = []string{} + info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgscan", args...) + if err != nil { + klog.Errorf("Failed to scan for volume group post creation, continuing: %v: %s", err, info) + } + return nil +} + +func reduceVolumeGroup(volumeGroupName string, force bool) { + if !checkVgExists(volumeGroupName) { + return + } + args := []string{ + "--removemissing", + volumeGroupName, + } + if force { + args = append(args, "--force") + } + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgreduce", args...) + if err != nil { + klog.Errorf("Errored while cleaning up volume group %v: %s", err, info) + } +} + +func RaidLocalSsds(availableLssds []string) error { + args := []string{ + "--create", + raidedLocalSsdName, + "-l" + raidMode, + // Force RAIDing as sometime it might fail for caution if there is just 1 LSSD present as 1 LSSD need not be RAIDed + "--force", + "-n", + strconv.Itoa(len(availableLssds)), + } + args = append(args, availableLssds...) + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipeCmdArg */, "mdadm", args...) + if err != nil { + return fmt.Errorf("errored while RAIDing LSSDs info: %v; err:%v", info, err) + } + // Validate if Raided successfully + isAlreadyRaided, err := IsRaided() + if err != nil { + klog.V(4).Infof("Errored while scanning for available raided LocalSSDs err:%v=", err) + } + if !isAlreadyRaided { + return fmt.Errorf("failed raiding, raided device not found on scanning") + } + + raidedDataCacheCount, err := FetchRaidedLssdCountForDatacache() + if err != nil { + return err + } + if raidedDataCacheCount != len(availableLssds) { + return fmt.Errorf("Local SSDs reserved do not match the requested count") + } + return nil +} + +func IsRaided() (bool, error) { + args := []string{ + "--detail", + "--scan", + } + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipeCmdArg */, "mdadm", args...) + if err != nil { + return false, fmt.Errorf("errored while scanning for raided LSSD %v: %s", err, info) + } + if info != nil && strings.Contains(string(info), raidedLocalSsdName) { + return true, nil + } + return false, nil +} + +func isCachingSetup(mainLvName string) (error, bool) { + // Verify caching is setup for PD + args := []string{ + "--select", + "lv_name=" + mainLvName, + "-o", + "pool_lv", + } + poolName, err := common.RunCommand("" /* pipedCmd */, nil /* pipeCmdArg */, "lvs", args...) + if err != nil { + return fmt.Errorf("Failed to check if caching is setup %w", err), false + } + if strings.Contains(string(poolName), "csi-fast") { + return nil, true + } + return nil, false +} + +func fetchChunkSizeKiB(cacheSize string) (string, error) { + var chunkSize float64 + + cacheSizeInt, err := strconv.ParseInt(cacheSize, 10, 64) + if err != nil { + return "0", err + } + // Chunksize should be divisible by 32Kib so we need (chunksize/32*1024)*32*1024 + chunkSize = (float64(cacheSizeInt) * GiB) / float64(maxAllowedChunks) + chunkSize = math.Round(chunkSize/(32*KiB)) * (32 * KiB) + chunkSize = math.Min(math.Max(chunkSize, minChunkSize), maxChunkSize) / KiB + // default chunk size unit KiB + return strconv.FormatInt(int64(chunkSize), 10) + "KiB", nil +} + +func InitializeDataCacheNode(nodeId string) error { + raidedLocalSsdPath, err := fetchRAIDedLocalSsdPath() + if err != nil { + return err + } + volumeGroupName := getVolumeGroupName(nodeId) + + vgExists := checkVgExists(volumeGroupName) + // Check if the required volume group already exists + if vgExists { + // Clean up Volume Group before adding the PD + reduceVolumeGroup(volumeGroupName, true) + + // validate that raidedLSSD is part of VG + err = validateRaidedLSSDinVG(volumeGroupName, raidedLocalSsdPath) + if err != nil { + return fmt.Errorf("failed validate local ssd in vg %v: %v", volumeGroupName, err) + } + } else { + err := createVg(volumeGroupName, raidedLocalSsdPath) + if err != nil { + return err + } + } + return nil +} + +func StartWatcher(nodeName string) { + dirToWatch := "/dev/" + watcher, err := fsnotify.NewWatcher() + if err != nil { + klog.V(2).ErrorS(err, "errored while creating watcher") + } + klog.V(2).Infof("Watcher started for directory %v", dirToWatch) + defer watcher.Close() + + // out of the box fsnotify can watch a single file, or a single directory + if err := watcher.Add(dirToWatch); err != nil { + klog.V(2).ErrorS(err, "errored while adding watcher directory") + } + errorCh := make(chan error, 1) + // Handle the error received from the watcher goroutine + go watchDiskDetaches(watcher, nodeName, errorCh) + + select { + case err := <-errorCh: + klog.Errorf("watcher encountered an error: %v", err) + } +} + +func watchDiskDetaches(watcher *fsnotify.Watcher, nodeName string, errorCh chan error) error { + for { + select { + // watch for errors + case err := <-watcher.Errors: + errorCh <- fmt.Errorf("disk update event errored: %v", err) + // watch for events + case <-watcher.Events: + // In case of an event i.e. creation or deletion of any new PV, we update the VG metadata. + // This might include some non-LVM changes, no harm in updating metadata multiple times. + args := []string{ + "--updatemetadata", + getVolumeGroupName(nodeName), + } + _, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgck", args...) + if err != nil { + klog.Errorf("Error updating volume group's metadata: %v", err) + } + reduceVolumeGroup(getVolumeGroupName(nodeName), true) + } + } +} + +func validateRaidedLSSDinVG(vgName string, lssdPath string) error { + args := []string{ + "--noheadings", + "-o", + "pv_name", + "--select", + "vg_name=" + vgName, + } + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "pvs", args...) + if err != nil { + return fmt.Errorf("errored while checking physical volume details %v: %s", err, info) + // On error info contains the error message which we cannot use for further steps + } + + if !strings.Contains(string(info), lssdPath) { + return addRaidedLSSDToVg(vgName, lssdPath) + } + return nil +} + +func addRaidedLSSDToVg(vgName, lssdPath string) error { + info, err := common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgextend", []string{vgName, lssdPath}...) + if err != nil { + return fmt.Errorf("errored while extending VGs %v: %s", err, info) + } + return nil +} + +func fetchPvSizeGiB() (string, error) { + args := []string{ + "--select", + "-o", + "--noheadings", + "pv_size", + "--units=b", + } + // RAIDed device is always registered with its /dev/md127 equivalent in VG so cannot check it directly based on the RAIDed LSSD path which could be /dev/md/csi-driver-data-cache + info, err := common.RunCommand("grep" /* pipedCmd */, []string{"/dev/md"} /* pipedCmdArg */, "pvs", args...) + if err != nil { + return "", fmt.Errorf("errored while fetching PV size %v: %s", err, info) + } + infoString := strings.TrimSpace(string(info)) + infoSlice := strings.Fields(infoString) + pvSize, err := fetchNumberGiB(infoSlice) + if err != nil { + return "", fmt.Errorf("Error fetching PV size for cache %v", err) + } + return pvSize, nil + +} + +func fetchNumberGiB(infoSlice []string) (string, error) { + re, err := regexp.Compile("^[0-9]+B$") + if err != nil { + return "", fmt.Errorf("Failed to compile regex match %v", err) + } + var pvSize string + for _, i := range infoSlice { + if re.MatchString(i) { + pvSize, err = strings.TrimSuffix(i, "B"), nil + if err != nil { + return "", fmt.Errorf("Failed to extract PV size %v", err) + } + break + } + } + pvSizeInt, err := strconv.ParseFloat(pvSize, 64) + if err != nil { + return "", fmt.Errorf("Error while fetching PV size for cache %v", err) + } + return strconv.FormatInt(int64(math.Ceil(pvSizeInt/GiB)), 10) + "GiB", nil +} diff --git a/pkg/gce-pd-csi-driver/cache_test.go b/pkg/gce-pd-csi-driver/cache_test.go new file mode 100644 index 000000000..340c924a1 --- /dev/null +++ b/pkg/gce-pd-csi-driver/cache_test.go @@ -0,0 +1,117 @@ +package gceGCEDriver + +import ( + "testing" +) + +func TestFetchChunkSizeKiB(t *testing.T) { + testCases := []struct { + name string + cacheSize string + expChunkSize string + expErr bool + }{ + { + name: "chunk size is in the allowed range", + cacheSize: "500", + expChunkSize: "512KiB", //range defined in fetchChunkSizeKiB + }, + { + name: "chunk size is set to the range ceil", + cacheSize: "30000000", + expChunkSize: "1048576KiB", //range defined in fetchChunkSizeKiB - max 1GiB + }, + { + name: "chunk size is set to the allowed range floor", + cacheSize: "100", + expChunkSize: "160KiB", //range defined in fetchChunkSizeKiB - min 160 KiB + }, + { + name: "cacheSize set to KiB also sets the chunk size to range floor", + cacheSize: "1", + expChunkSize: "160KiB", //range defined in fetchChunkSizeKiB - min 160 KiB + }, + { + name: "invalid cacheSize", + cacheSize: "fdfsdKi", + expChunkSize: "160KiB", //range defined in fetchChunkSizeKiB - min 160 KiB + expErr: true, + }, + // cacheSize is validated in storage class parameter so assuming invalid cacheSize (like negative, 0) would not be passed to the function + } + + for _, tc := range testCases { + chunkSize, err := fetchChunkSizeKiB(tc.cacheSize) + if err != nil { + if !tc.expErr { + t.Errorf("Errored %s", err) + } + continue + } + if chunkSize != tc.expChunkSize { + t.Errorf("Got %s want %s", chunkSize, tc.expChunkSize) + } + + } + +} + +func TestFetchNumberGiB(t *testing.T) { + testCases := []struct { + name string + stringInput []string + expOutput string // Outputs value in GiB + expErr bool + }{ + { + name: "valid input 1", + stringInput: []string{"5000000000B"}, + expOutput: "5GiB", //range defined in fetchChunkSizeKiB + }, + { + name: "valid input 2", + stringInput: []string{"375000000000B"}, // 1 LSSD attached + expOutput: "350GiB", //range defined in fetchChunkSizeKiB + }, + { + name: "valid input 3", + stringInput: []string{"9000000000000B"}, // 24 LSSD attached + expOutput: "8382GiB", //range defined in fetchChunkSizeKiB + }, + { + name: "valid input 4", + stringInput: []string{"Some text before ", "9000000000000B", "Some text after"}, // 24 LSSD attached + expOutput: "8382GiB", //range defined in fetchChunkSizeKiB + }, + { + name: "invalid input 1", + stringInput: []string{"9000000000000"}, + expErr: true, + }, + { + name: "invalid input 2", + stringInput: []string{"A9000000000000B"}, + expErr: true, + }, + { + name: "valid input 5", + stringInput: []string{"900000B"}, // <1GiB gets rounded off to 0GiB + expOutput: "1GiB", + }, + } + + for _, tc := range testCases { + v, err := fetchNumberGiB(tc.stringInput) + if err != nil { + if !tc.expErr { + t.Errorf("Errored %s", err) + } + continue + } + if v != tc.expOutput { + t.Errorf("Got %s want %s", v, tc.expOutput) + } + + } + +} diff --git a/pkg/gce-pd-csi-driver/controller.go b/pkg/gce-pd-csi-driver/controller.go index a2d57000d..7a5f6cbc0 100644 --- a/pkg/gce-pd-csi-driver/controller.go +++ b/pkg/gce-pd-csi-driver/controller.go @@ -105,6 +105,9 @@ type GCEControllerServer struct { // If set to true, the CSI Driver will allow volumes to be provisioned in Storage Pools. enableStoragePools bool + // If set to true, the CSI Driver will allow volumes to be provisioned with data cache configuration + enableDataCache bool + multiZoneVolumeHandleConfig MultiZoneVolumeHandleConfig listVolumesConfig ListVolumesConfig @@ -298,6 +301,14 @@ func (gceCS *GCEControllerServer) createVolumeInternal(ctx context.Context, req defer func() { gceCS.Metrics.RecordOperationErrorMetrics("CreateVolume", err, diskTypeForMetric, enableConfidentialCompute, enableStoragePools) }() + + // Apply Parameters (case-insensitive). We leave validation of + // the values to the cloud provider. + params, dataCacheParams, err := gceCS.parameterProcessor().ExtractAndDefaultParameters(req.GetParameters(), gceCS.Driver.extraVolumeLabels, gceCS.enableDataCache, gceCS.Driver.extraTags) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "failed to extract parameters: %v", err.Error()) + } + // Validate arguments volumeCapabilities := req.GetVolumeCapabilities() capacityRange := req.GetCapacityRange() @@ -320,7 +331,6 @@ func (gceCS *GCEControllerServer) createVolumeInternal(ctx context.Context, req // Apply Parameters (case-insensitive). We leave validation of // the values to the cloud provider. - params, err := gceCS.parameterProcessor().ExtractAndDefaultParameters(req.GetParameters(), gceCS.Driver.extraVolumeLabels, gceCS.Driver.extraTags) diskTypeForMetric = params.DiskType enableConfidentialCompute = strconv.FormatBool(params.EnableConfidentialCompute) hasStoragePools := len(params.StoragePools) > 0 @@ -359,11 +369,11 @@ func (gceCS *GCEControllerServer) createVolumeInternal(ctx context.Context, req if gceCS.multiZoneVolumeHandleConfig.Enable && params.MultiZoneProvisioning { // Create multi-zone disk, that may have up to N disks. - return gceCS.createMultiZoneDisk(ctx, req, params) + return gceCS.createMultiZoneDisk(ctx, req, params, dataCacheParams, gceCS.enableDataCache) } // Create single device zonal or regional disk - return gceCS.createSingleDeviceDisk(ctx, req, params) + return gceCS.createSingleDeviceDisk(ctx, req, params, dataCacheParams, gceCS.enableDataCache) } func (gceCS *GCEControllerServer) getSupportedZonesForPDType(ctx context.Context, zones []string, diskType string) ([]string, error) { @@ -418,7 +428,7 @@ func (gceCS *GCEControllerServer) getMultiZoneProvisioningZones(ctx context.Cont return combinedZones.List(), nil } -func (gceCS *GCEControllerServer) createMultiZoneDisk(ctx context.Context, req *csi.CreateVolumeRequest, params common.DiskParameters) (*csi.CreateVolumeResponse, error) { +func (gceCS *GCEControllerServer) createMultiZoneDisk(ctx context.Context, req *csi.CreateVolumeRequest, params common.DiskParameters, dataCacheParams common.DataCacheParameters, enableDataCache bool) (*csi.CreateVolumeResponse, error) { // Determine the zones that are needed. var err error @@ -464,7 +474,7 @@ func (gceCS *GCEControllerServer) createMultiZoneDisk(ctx context.Context, req * // Use the first response as a template volumeId := fmt.Sprintf("projects/%s/zones/%s/disks/%s", gceCS.CloudProvider.GetDefaultProject(), common.MultiZoneValue, req.GetName()) klog.V(4).Infof("CreateVolume succeeded for multi-zone disks in zones %s: %v", zones, multiZoneVolKey) - return generateCreateVolumeResponseWithVolumeId(createdDisks[0], zones, params, volumeId), nil + return generateCreateVolumeResponseWithVolumeId(createdDisks[0], zones, params, dataCacheParams, enableDataCache, volumeId), nil } func (gceCS *GCEControllerServer) getZonesWithDiskNameAndType(ctx context.Context, name string, diskType string) ([]string, error) { @@ -508,7 +518,7 @@ func (gceCS *GCEControllerServer) updateAccessModeIfNecessary(ctx context.Contex return gceCS.CloudProvider.SetDiskAccessMode(ctx, project, volKey, readOnlyManyAccessMode) } -func (gceCS *GCEControllerServer) createSingleDeviceDisk(ctx context.Context, req *csi.CreateVolumeRequest, params common.DiskParameters) (*csi.CreateVolumeResponse, error) { +func (gceCS *GCEControllerServer) createSingleDeviceDisk(ctx context.Context, req *csi.CreateVolumeRequest, params common.DiskParameters, dataCacheParams common.DataCacheParameters, enableDataCache bool) (*csi.CreateVolumeResponse, error) { var err error var locationTopReq *locationRequirements if useVolumeCloning(req) { @@ -560,7 +570,7 @@ func (gceCS *GCEControllerServer) createSingleDeviceDisk(ctx context.Context, re return nil, common.LoggedError("CreateVolume failed: %v", err) } - return generateCreateVolumeResponseWithVolumeId(disk, zones, params, volumeID), err + return generateCreateVolumeResponseWithVolumeId(disk, zones, params, dataCacheParams, enableDataCache, volumeID), err } func (gceCS *GCEControllerServer) createSingleDisk(ctx context.Context, req *csi.CreateVolumeRequest, params common.DiskParameters, volKey *meta.Key, zones []string) (*gce.CloudDisk, error) { @@ -962,6 +972,15 @@ func (gceCS *GCEControllerServer) executeControllerPublishVolume(ctx context.Con PublishContext: nil, } + // Set data cache publish context + if gceCS.enableDataCache && req.GetVolumeContext() != nil { + if req.GetVolumeContext()[common.ContextDataCacheSize] != "" { + pubVolResp.PublishContext = map[string]string{} + pubVolResp.PublishContext[common.ContextDataCacheSize] = req.GetVolumeContext()[common.ContextDataCacheSize] + pubVolResp.PublishContext[common.ContextDataCacheMode] = req.GetVolumeContext()[common.ContextDataCacheMode] + } + } + instanceZone, instanceName, err := common.NodeIDToZoneAndName(nodeID) if err != nil { return nil, status.Errorf(codes.NotFound, "could not split nodeID: %v", err.Error()), nil @@ -1232,10 +1251,11 @@ func (gceCS *GCEControllerServer) ValidateVolumeCapabilities(ctx context.Context } // Validate the disk parameters match the disk we GET - params, err := gceCS.parameterProcessor().ExtractAndDefaultParameters(req.GetParameters(), gceCS.Driver.extraVolumeLabels, gceCS.Driver.extraTags) + params, _, err := gceCS.parameterProcessor().ExtractAndDefaultParameters(req.GetParameters(), gceCS.Driver.extraVolumeLabels, gceCS.enableDataCache, gceCS.Driver.extraTags) if err != nil { return nil, status.Errorf(codes.InvalidArgument, "failed to extract parameters: %v", err.Error()) } + if err := gce.ValidateDiskParameters(disk, params); err != nil { return generateFailedValidationMessage("Parameters %v do not match given disk %s: %v", req.GetParameters(), disk.GetName(), err.Error()), nil } @@ -2270,7 +2290,7 @@ func extractVolumeContext(context map[string]string) (*PDCSIContext, error) { return info, nil } -func generateCreateVolumeResponseWithVolumeId(disk *gce.CloudDisk, zones []string, params common.DiskParameters, volumeId string) *csi.CreateVolumeResponse { +func generateCreateVolumeResponseWithVolumeId(disk *gce.CloudDisk, zones []string, params common.DiskParameters, dataCacheParams common.DataCacheParameters, enableDataCache bool, volumeId string) *csi.CreateVolumeResponse { tops := []*csi.Topology{} for _, zone := range zones { tops = append(tops, &csi.Topology{ @@ -2286,6 +2306,14 @@ func generateCreateVolumeResponseWithVolumeId(disk *gce.CloudDisk, zones []strin AccessibleTopology: tops, }, } + // Set data cache volume context + if enableDataCache && dataCacheParams != (common.DataCacheParameters{}) { + if createResp.Volume.VolumeContext == nil { + createResp.Volume.VolumeContext = map[string]string{} + } + createResp.Volume.VolumeContext[common.ContextDataCacheMode] = dataCacheParams.DataCacheMode + createResp.Volume.VolumeContext[common.ContextDataCacheSize] = dataCacheParams.DataCacheSize + } snapshotID := disk.GetSnapshotId() imageID := disk.GetImageId() diskID := disk.GetSourceDiskId() diff --git a/pkg/gce-pd-csi-driver/gce-pd-driver.go b/pkg/gce-pd-csi-driver/gce-pd-driver.go index ba1a0c0ba..e1bb5706c 100644 --- a/pkg/gce-pd-csi-driver/gce-pd-driver.go +++ b/pkg/gce-pd-csi-driver/gce-pd-driver.go @@ -143,18 +143,20 @@ func NewIdentityServer(gceDriver *GCEDriver) *GCEIdentityServer { } } -func NewNodeServer(gceDriver *GCEDriver, mounter *mount.SafeFormatAndMount, deviceUtils deviceutils.DeviceUtils, meta metadataservice.MetadataService, statter mountmanager.Statter) *GCENodeServer { +func NewNodeServer(gceDriver *GCEDriver, mounter *mount.SafeFormatAndMount, deviceUtils deviceutils.DeviceUtils, meta metadataservice.MetadataService, statter mountmanager.Statter, args NodeServerArgs) *GCENodeServer { return &GCENodeServer{ - Driver: gceDriver, - Mounter: mounter, - DeviceUtils: deviceUtils, - MetadataService: meta, - volumeLocks: common.NewVolumeLocks(), - VolumeStatter: statter, + Driver: gceDriver, + Mounter: mounter, + DeviceUtils: deviceUtils, + MetadataService: meta, + volumeLocks: common.NewVolumeLocks(), + VolumeStatter: statter, + EnableDataCache: args.EnableDataCache, + DataCacheEnabledNodePool: args.DataCacheEnabledNodePool, } } -func NewControllerServer(gceDriver *GCEDriver, cloudProvider gce.GCECompute, errorBackoffInitialDuration, errorBackoffMaxDuration time.Duration, fallbackRequisiteZones []string, enableStoragePools bool, multiZoneVolumeHandleConfig MultiZoneVolumeHandleConfig, listVolumesConfig ListVolumesConfig) *GCEControllerServer { +func NewControllerServer(gceDriver *GCEDriver, cloudProvider gce.GCECompute, errorBackoffInitialDuration, errorBackoffMaxDuration time.Duration, fallbackRequisiteZones []string, enableStoragePools bool, enableDataCache bool, multiZoneVolumeHandleConfig MultiZoneVolumeHandleConfig, listVolumesConfig ListVolumesConfig) *GCEControllerServer { return &GCEControllerServer{ Driver: gceDriver, CloudProvider: cloudProvider, @@ -163,6 +165,7 @@ func NewControllerServer(gceDriver *GCEDriver, cloudProvider gce.GCECompute, err errorBackoff: newCsiErrorBackoff(errorBackoffInitialDuration, errorBackoffMaxDuration), fallbackRequisiteZones: fallbackRequisiteZones, enableStoragePools: enableStoragePools, + enableDataCache: enableDataCache, multiZoneVolumeHandleConfig: multiZoneVolumeHandleConfig, listVolumesConfig: listVolumesConfig, } diff --git a/pkg/gce-pd-csi-driver/gce-pd-driver_test.go b/pkg/gce-pd-csi-driver/gce-pd-driver_test.go index 15306c141..3789b7bc6 100644 --- a/pkg/gce-pd-csi-driver/gce-pd-driver_test.go +++ b/pkg/gce-pd-csi-driver/gce-pd-driver_test.go @@ -41,17 +41,23 @@ func initBlockingGCEDriver(t *testing.T, cloudDisks []*gce.CloudDisk, readyToExe return initGCEDriverWithCloudProvider(t, fakeBlockingBlockProvider) } -func initGCEDriverWithCloudProvider(t *testing.T, cloudProvider gce.GCECompute) *GCEDriver { - vendorVersion := "test-vendor" +func controllerServerForTest(cloudProvider gce.GCECompute) *GCEControllerServer { gceDriver := GetGCEDriver() errorBackoffInitialDuration := 200 * time.Millisecond errorBackoffMaxDuration := 5 * time.Minute fallbackRequisiteZones := []string{} enableStoragePools := false + enableDataCache := false multiZoneVolumeHandleConfig := MultiZoneVolumeHandleConfig{} listVolumesConfig := ListVolumesConfig{} - controllerServer := NewControllerServer(gceDriver, cloudProvider, errorBackoffInitialDuration, errorBackoffMaxDuration, fallbackRequisiteZones, enableStoragePools, multiZoneVolumeHandleConfig, listVolumesConfig) + return NewControllerServer(gceDriver, cloudProvider, errorBackoffInitialDuration, errorBackoffMaxDuration, fallbackRequisiteZones, enableStoragePools, enableDataCache, multiZoneVolumeHandleConfig, listVolumesConfig) +} + +func initGCEDriverWithCloudProvider(t *testing.T, cloudProvider gce.GCECompute) *GCEDriver { + vendorVersion := "test-vendor" + gceDriver := GetGCEDriver() + controllerServer := controllerServerForTest(cloudProvider) err := gceDriver.SetupGCEDriver(driver, vendorVersion, nil, nil, nil, controllerServer, nil) if err != nil { t.Fatalf("Failed to setup GCE Driver: %v", err) diff --git a/pkg/gce-pd-csi-driver/node.go b/pkg/gce-pd-csi-driver/node.go index b10471e3d..534a0ebf7 100644 --- a/pkg/gce-pd-csi-driver/node.go +++ b/pkg/gce-pd-csi-driver/node.go @@ -42,11 +42,13 @@ import ( ) type GCENodeServer struct { - Driver *GCEDriver - Mounter *mount.SafeFormatAndMount - DeviceUtils deviceutils.DeviceUtils - VolumeStatter mountmanager.Statter - MetadataService metadataservice.MetadataService + Driver *GCEDriver + Mounter *mount.SafeFormatAndMount + DeviceUtils deviceutils.DeviceUtils + VolumeStatter mountmanager.Statter + MetadataService metadataservice.MetadataService + EnableDataCache bool + DataCacheEnabledNodePool bool // A map storing all volumes with ongoing operations so that additional operations // for that same volume (as defined by VolumeID) return an Aborted error @@ -60,6 +62,16 @@ type GCENodeServer struct { // been observed). formatAndMountSemaphore chan any formatAndMountTimeout time.Duration + + // Embed UnimplementedNodeServer to ensure the driver returns Unimplemented for any + // new RPC methods that might be introduced in future versions of the spec. + csi.UnimplementedNodeServer +} + +type NodeServerArgs struct { + EnableDataCache bool + + DataCacheEnabledNodePool bool } var _ csi.NodeServer = &GCENodeServer{} @@ -279,6 +291,7 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage volumeID := req.GetVolumeId() stagingTargetPath := req.GetStagingTargetPath() volumeCapability := req.GetVolumeCapability() + nodeId := ns.MetadataService.GetName() if len(volumeID) == 0 { return nil, status.Error(codes.InvalidArgument, "NodeStageVolume Volume ID must be provided") } @@ -312,12 +325,32 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage partition = part } devicePath, err := getDevicePath(ns, volumeID, partition) - if err != nil { return nil, status.Error(codes.Internal, fmt.Sprintf("Error when getting device path: %v", err.Error())) } - klog.V(4).Infof("Successfully found attached GCE PD %q at device path %s.", volumeKey.Name, devicePath) + klog.Infof("Successfully found attached GCE PD %q at device path %s.", volumeKey.Name, devicePath) + + if ns.EnableDataCache && (req.GetPublishContext()[common.ContextDataCacheSize] != "" || req.GetPublishContext()[common.ContextDataCacheMode] != "") { + if len(nodeId) == 0 { + return nil, status.Error(codes.InvalidArgument, "NodeStageVolume Node ID must be provided") + } + devFsPath, err := filepath.EvalSymlinks(devicePath) + if err != nil { + klog.Errorf("filepath.EvalSymlinks(%q) failed when trying to create volume group: %v", devicePath, err) + } + configError := ValidateDataCacheConfig(req.GetPublishContext()[common.ContextDataCacheMode], req.GetPublishContext()[common.ContextDataCacheSize], ctx) + if configError != nil { + if ns.DataCacheEnabledNodePool { + return nil, status.Error(codes.DataLoss, fmt.Sprintf("Error validate configuration for Data Cache: %v", configError.Error())) + } + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("The Data Cache PVC is scheduled on an incompatible node pool. Please select a node pool with data cache configured: %v", configError.Error())) + } + devicePath, err = setupCaching(devFsPath, req, nodeId) + if err != nil { + return nil, status.Error(codes.DataLoss, fmt.Sprintf("Error setting up cache: %v", err.Error())) + } + } // Part 2: Check if mount already exists at stagingTargetPath if ns.isVolumePathMounted(stagingTargetPath) { @@ -465,6 +498,15 @@ func (ns *GCENodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUns } } + // The NodeUnstageVolume does not have any volume or publish context, we need to get the info from LVM locally + // Check if cache group cache-{volumeID} exist in LVM + if ns.EnableDataCache { + nodeId := ns.MetadataService.GetName() + err := cleanupCache(volumeID, nodeId) + if err != nil { + return nil, status.Errorf(codes.DataLoss, "Failed to cleanup cache for volume %s: %v", volumeID, err) + } + } klog.V(4).Infof("NodeUnstageVolume succeeded on %v from %s", volumeID, stagingTargetPath) return &csi.NodeUnstageVolumeResponse{}, nil } diff --git a/pkg/gce-pd-csi-driver/node_test.go b/pkg/gce-pd-csi-driver/node_test.go index 7e59ea7e0..337cd3a8a 100644 --- a/pkg/gce-pd-csi-driver/node_test.go +++ b/pkg/gce-pd-csi-driver/node_test.go @@ -51,7 +51,8 @@ func getTestGCEDriverWithCustomMounter(t *testing.T, mounter *mount.SafeFormatAn func getCustomTestGCEDriver(t *testing.T, mounter *mount.SafeFormatAndMount, deviceUtils deviceutils.DeviceUtils, metaService metadataservice.MetadataService) *GCEDriver { gceDriver := GetGCEDriver() - nodeServer := NewNodeServer(gceDriver, mounter, deviceUtils, metaService, mountmanager.NewFakeStatter(mounter)) + enableDataCache := false + nodeServer := NewNodeServer(gceDriver, mounter, deviceUtils, metaService, mountmanager.NewFakeStatter(mounter), NodeServerArgs{enableDataCache, false /*dataCacheEnableNodePool */}) err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, nil, nil, nodeServer) if err != nil { t.Fatalf("Failed to setup GCE Driver: %v", err) @@ -62,7 +63,7 @@ func getCustomTestGCEDriver(t *testing.T, mounter *mount.SafeFormatAndMount, dev func getTestBlockingMountGCEDriver(t *testing.T, readyToExecute chan chan struct{}) *GCEDriver { gceDriver := GetGCEDriver() mounter := mountmanager.NewFakeSafeBlockingMounter(readyToExecute) - nodeServer := NewNodeServer(gceDriver, mounter, deviceutils.NewFakeDeviceUtils(false), metadataservice.NewFakeService(), mountmanager.NewFakeStatter(mounter)) + nodeServer := NewNodeServer(gceDriver, mounter, deviceutils.NewFakeDeviceUtils(false), metadataservice.NewFakeService(), mountmanager.NewFakeStatter(mounter), NodeServerArgs{true, false /*dataCacheEnableNodePool */}) err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, nil, nil, nodeServer) if err != nil { t.Fatalf("Failed to setup GCE Driver: %v", err) @@ -72,8 +73,9 @@ func getTestBlockingMountGCEDriver(t *testing.T, readyToExecute chan chan struct func getTestBlockingFormatAndMountGCEDriver(t *testing.T, readyToExecute chan chan struct{}) *GCEDriver { gceDriver := GetGCEDriver() + enableDataCache := true mounter := mountmanager.NewFakeSafeBlockingMounter(readyToExecute) - nodeServer := NewNodeServer(gceDriver, mounter, deviceutils.NewFakeDeviceUtils(false), metadataservice.NewFakeService(), mountmanager.NewFakeStatter(mounter)).WithSerializedFormatAndMount(5*time.Second, 1) + nodeServer := NewNodeServer(gceDriver, mounter, deviceutils.NewFakeDeviceUtils(false), metadataservice.NewFakeService(), mountmanager.NewFakeStatter(mounter), NodeServerArgs{enableDataCache, false /*dataCacheEnableNodePool */}).WithSerializedFormatAndMount(5*time.Second, 1) err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, nil, nil, nodeServer) if err != nil { diff --git a/test/e2e/tests/multi_zone_e2e_test.go b/test/e2e/tests/multi_zone_e2e_test.go index fb1ea32dd..456c781e6 100644 --- a/test/e2e/tests/multi_zone_e2e_test.go +++ b/test/e2e/tests/multi_zone_e2e_test.go @@ -332,7 +332,7 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { }() // Attach Disk - err := testAttachWriteReadDetach(underSpecifiedID, snapshotVolName, tc0.Instance, controllerClient, false /* readOnly */) + err := testAttachWriteReadDetach(underSpecifiedID, snapshotVolName, tc0.Instance, controllerClient, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle") // Create Snapshot @@ -413,11 +413,11 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { Expect(disk2.AccessMode).To(Equal("READ_ONLY_MANY")) // Attach Disk to node1 and validate contents - err = testAttachWriteReadDetach(volID, volName, tc0.Instance, tc0.Client, true /* readonly */) + err = testAttachWriteReadDetach(volID, volName, tc0.Instance, tc0.Client, true /* readonly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to attach/read/detach on vol1") // Attach Disk to node1 and validate contents - err = testAttachWriteReadDetach(volID, volName, tc1.Instance, tc1.Client, true /* readonly */) + err = testAttachWriteReadDetach(volID, volName, tc1.Instance, tc1.Client, true /* readonly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to attach/read/detach on vol2") disk1, err = computeService.Disks.Get(p, zones[0], volName).Do() @@ -479,7 +479,7 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { }() // Attach Disk - err := testAttachWriteReadDetach(underSpecifiedID, snapshotVolName, tc0.Instance, controllerClient, false /* readOnly */) + err := testAttachWriteReadDetach(underSpecifiedID, snapshotVolName, tc0.Instance, controllerClient, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle") // Create Snapshot @@ -551,11 +551,11 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { Expect(disk2.AccessMode).To(Equal("READ_ONLY_MANY")) // Attach Disk to node1 and validate contents - err = testAttachWriteReadDetach(volID, volName, tc0.Instance, tc0.Client, true /* readonly */) + err = testAttachWriteReadDetach(volID, volName, tc0.Instance, tc0.Client, true /* readonly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to attach/read/detach on vol1") // Attach Disk to node1 and validate contents - err = testAttachWriteReadDetach(volID, volName, tc1.Instance, tc1.Client, true /* readonly */) + err = testAttachWriteReadDetach(volID, volName, tc1.Instance, tc1.Client, true /* readonly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to attach/read/detach on vol2") disk1, err = computeService.Disks.Get(p, zones[0], volName).Do() @@ -617,7 +617,7 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { }() // Attach Disk - err := testAttachWriteReadDetach(underSpecifiedID, snapshotVolName, tc0.Instance, controllerClient, false /* readOnly */) + err := testAttachWriteReadDetach(underSpecifiedID, snapshotVolName, tc0.Instance, controllerClient, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle") // Create Disk Image @@ -707,11 +707,11 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { Expect(disk2.AccessMode).To(Equal("READ_ONLY_MANY")) // Attach Disk to node1 - err = testAttachWriteReadDetach(volID, volName, tc0.Instance, tc0.Client, true /* readonly */) + err = testAttachWriteReadDetach(volID, volName, tc0.Instance, tc0.Client, true /* readonly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to attach/read/detach on vol1") // Attach Disk to node1 - err = testAttachWriteReadDetach(volID, volName, tc1.Instance, tc1.Client, true /* readonly */) + err = testAttachWriteReadDetach(volID, volName, tc1.Instance, tc1.Client, true /* readonly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to attach/read/detach on vol2") }) @@ -806,18 +806,18 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { volID0 := fmt.Sprintf("projects/%s/zones/%s/disks/%s", p, zones[0], volName) volID1 := fmt.Sprintf("projects/%s/zones/%s/disks/%s", p, zones[1], volName) - err = testAttachWriteReadDetach(volID0, volName, tc0.Instance, tc0.Client, false /* readonly */) + err = testAttachWriteReadDetach(volID0, volName, tc0.Instance, tc0.Client, false /* readonly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to attach/write/read/detach on vol1") - err = testAttachWriteReadDetach(volID1, volName, tc1.Instance, tc1.Client, false /* readonly */) + err = testAttachWriteReadDetach(volID1, volName, tc1.Instance, tc1.Client, false /* readonly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to attach/write/read/detach on vol2") // Validate disks can be used in multi-zone mode on both nodes volIDMultiZone := fmt.Sprintf("projects/%s/zones/multi-zone/disks/%s", p, volName) - err = testAttachWriteReadDetach(volIDMultiZone, volName, tc0.Instance, tc0.Client, true /* readonly */) + err = testAttachWriteReadDetach(volIDMultiZone, volName, tc0.Instance, tc0.Client, true /* readonly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to attach/read/detach on vol1") - err = testAttachWriteReadDetach(volIDMultiZone, volName, tc1.Instance, tc1.Client, true /* readonly */) + err = testAttachWriteReadDetach(volIDMultiZone, volName, tc1.Instance, tc1.Client, true /* readonly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to attach/read/detach on vol2") // Validate disks are ROX now @@ -910,7 +910,7 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { if i >= 1 { readOnly = true } - err = testAttachWriteReadDetach(volume.VolumeId, volName, testContext.Instance, testContext.Client, readOnly) + err = testAttachWriteReadDetach(volume.VolumeId, volName, testContext.Instance, testContext.Client, readOnly, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "failed volume lifecycle checks") i = i + 1 } @@ -998,9 +998,10 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { // Attach disk to instance in the first zone. tc0 := zoneToContext[zones[0]] err, detacher, args := testAttachAndMount(volume.VolumeId, volName, tc0.Instance, tc0.Client, attachAndMountArgs{ - readOnly: false, - useBlock: false, - forceAttach: false, + readOnly: false, + useBlock: false, + forceAttach: false, + setupDataCache: false, }) detachers = append(detachers, detacher) Expect(err).To(BeNil(), "failed attach in zone 0") @@ -1018,9 +1019,10 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { // Now force attach to the second instance without detaching. tc1 := zoneToContext[zones[1]] err, detacher, _ = testAttachAndMount(volume.VolumeId, volName, tc1.Instance, tc1.Client, attachAndMountArgs{ - readOnly: false, - useBlock: false, - forceAttach: true, + readOnly: false, + useBlock: false, + forceAttach: true, + setupDataCache: false, }) detachers = append(detachers, detacher) Expect(err).To(BeNil(), "failed force attach in zone 1") @@ -1040,7 +1042,12 @@ func deleteDisk(controllerClient *remote.CsiClient, p, zone, volID, volName stri Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") } -func testAttachWriteReadDetach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool) error { +func testAttachWriteReadDetach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool, detachAndReattach bool, setupDataCache bool) error { + writeFile, verifyReadFile := testWriteAndReadFile(instance, readOnly) + return testLifecycleWithVerify(volID, volName, instance, client, readOnly, false /* fs */, writeFile, verifyReadFile, detachAndReattach, setupDataCache) +} + +func testWriteAndReadFile(instance *remote.InstanceInfo, readOnly bool) (verifyFunc, verifyFunc) { var testFileContents = "test" writeFile := func(a *verifyArgs) error { if !readOnly { @@ -1066,46 +1073,61 @@ func testAttachWriteReadDetach(volID string, volName string, instance *remote.In } return nil } - return testLifecycleWithVerify(volID, volName, instance, client, readOnly, false /* fs */, writeFile, verifyReadFile) + return writeFile, verifyReadFile } type attachAndMountArgs struct { - readOnly bool - useBlock bool - forceAttach bool + readOnly bool + useBlock bool + forceAttach bool + setupDataCache bool } func testAttachAndMount(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, args attachAndMountArgs) (error, func(), *verifyArgs) { + klog.Infof("Starting testAttachAndMount with volume %v node %v \n", volID, instance.GetNodeID()) + err, unstageAndDetach, stageDir := testAttach(volID, volName, instance, client, args) + if err != nil { + return err, nil, nil + } + // Mount Disk + err, unpublish, returnArgs := testMount(volID, volName, instance, client, args, stageDir) + if err != nil { + unstageAndDetach() + return err, nil, nil + } + unpublishUnstageAndDetach := func() { + unpublish() + unstageAndDetach() + } + return nil, unpublishUnstageAndDetach, returnArgs +} + +func testAttach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, args attachAndMountArgs) (error, func(), string) { + klog.Infof("Starting testAttach with volume %v node %v \n", volID, instance.GetNodeID()) // Attach Disk var err error + var stageDir string if args.readOnly { err = client.ControllerPublishVolumeReadOnly(volID, instance.GetNodeID()) } else { err = client.ControllerPublishVolumeReadWrite(volID, instance.GetNodeID(), args.forceAttach) } if err != nil { - return fmt.Errorf("ControllerPublishVolume failed with error for disk %v on node %v: %v", volID, instance.GetNodeID(), err.Error()), nil, nil - } - - detach := func() { - // Detach Disk - err = client.ControllerUnpublishVolume(volID, instance.GetNodeID()) - if err != nil { - klog.Errorf("Failed to detach disk: %v", err) - } + return fmt.Errorf("ControllerPublishVolume failed with error for disk %v on node %v: %v", volID, instance.GetNodeID(), err.Error()), nil, stageDir } // Stage Disk - stageDir := filepath.Join("/tmp/", volName, "stage") + stageDir = filepath.Join("/tmp/", volName, "stage") if args.useBlock { - err = client.NodeStageBlockVolume(volID, stageDir) + err = client.NodeStageBlockVolume(volID, stageDir, args.setupDataCache) + } else { - err = client.NodeStageExt4Volume(volID, stageDir) + err = client.NodeStageExt4Volume(volID, stageDir, args.setupDataCache) } if err != nil { - detach() - return fmt.Errorf("NodeStageExt4Volume failed with error: %w", err), nil, nil + _ = detach(volID, instance, client) + return fmt.Errorf("NodeStageExt4Volume failed with error: %w for node: %v", err, instance.GetNodeID()), nil, stageDir } unstageAndDetach := func() { @@ -1120,9 +1142,23 @@ func testAttachAndMount(volID string, volName string, instance *remote.InstanceI klog.Errorf("Failed to rm file path %s: %v", fp, err) } - detach() + detach(volID, instance, client) + } + return nil, unstageAndDetach, stageDir +} + +func detach(volID string, instance *remote.InstanceInfo, client *remote.CsiClient) error { + // Detach Disk + err := client.ControllerUnpublishVolume(volID, instance.GetNodeID()) + if err != nil { + klog.Errorf("Failed to detach disk %v", err) + return fmt.Errorf("Failed to detach disk: %v", err) } + return nil +} +func testMount(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, args attachAndMountArgs, stageDir string) (error, func(), *verifyArgs) { + var err error // Mount Disk publishDir := filepath.Join("/tmp/", volName, "mount") @@ -1133,7 +1169,6 @@ func testAttachAndMount(volID string, volName string, instance *remote.InstanceI } if err != nil { - unstageAndDetach() return fmt.Errorf("NodePublishVolume failed with error: %v", err.Error()), nil, nil } @@ -1144,14 +1179,10 @@ func testAttachAndMount(volID string, volName string, instance *remote.InstanceI klog.Errorf("Failed to unpublish volume: %v", err) } } - unpublishUnstageAndDetach := func() { - unpublish() - unstageAndDetach() - } err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777", !args.readOnly /* recursive */) if err != nil { - unpublishUnstageAndDetach() + unpublish() return fmt.Errorf("Chmod failed with error: %v", err.Error()), nil, nil } @@ -1160,16 +1191,18 @@ func testAttachAndMount(volID string, volName string, instance *remote.InstanceI stageDir: stageDir, } - return nil, unpublishUnstageAndDetach, returnArgs + return nil, unpublish, returnArgs } -func testLifecycleWithVerify(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly, useBlock bool, firstMountVerify, secondMountVerify verifyFunc) error { +func testLifecycleWithVerify(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly, useBlock bool, firstMountVerify, secondMountVerify verifyFunc, detachAndReattach bool, setupDataCache bool) error { klog.Infof("Starting testAttachWriteReadDetach with volume %v node %v with readonly %v\n", volID, instance.GetNodeID(), readOnly) - err, detacher, args := testAttachAndMount(volID, volName, instance, client, attachAndMountArgs{ - readOnly: readOnly, - useBlock: useBlock, - forceAttach: false, - }) + attachArgs := attachAndMountArgs{ + readOnly: readOnly, + useBlock: useBlock, + forceAttach: false, + setupDataCache: setupDataCache, + } + err, detacher, args := testAttachAndMount(volID, volName, instance, client, attachArgs) if err != nil { return fmt.Errorf("failed to attach and mount: %w", err) } @@ -1186,13 +1219,28 @@ func testLifecycleWithVerify(volID string, volName string, instance *remote.Inst return fmt.Errorf("NodeUnpublishVolume failed with error: %v", err.Error()) } + stageDir := args.stageDir + if detachAndReattach { + // Unstage and detach + err = client.NodeUnstageVolume(volID, stageDir) + if err != nil { + klog.Errorf("Failed to unstage volume: %v", err) + } + detach(volID, instance, client) + // Reattach the volume + err, _, stageDir = testAttach(volID, volName, instance, client, attachArgs) + if err != nil { + return err + } + } + if secondMountVerify != nil { // Mount disk somewhere else secondPublishDir := filepath.Join("/tmp/", volName, "secondmount") if useBlock { - err = client.NodePublishBlockVolume(volID, args.stageDir, secondPublishDir) + err = client.NodePublishBlockVolume(volID, stageDir, secondPublishDir) } else { - err = client.NodePublishVolume(volID, args.stageDir, secondPublishDir) + err = client.NodePublishVolume(volID, stageDir, secondPublishDir) } if err != nil { return fmt.Errorf("NodePublishVolume failed with error: %v", err.Error()) diff --git a/test/e2e/tests/resize_e2e_test.go b/test/e2e/tests/resize_e2e_test.go index fbe8af868..fc4d87558 100644 --- a/test/e2e/tests/resize_e2e_test.go +++ b/test/e2e/tests/resize_e2e_test.go @@ -80,7 +80,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Stage Disk stageDir := filepath.Join("/tmp/", volName, "stage") - err = client.NodeStageExt4Volume(volume.VolumeId, stageDir) + err = client.NodeStageExt4Volume(volume.VolumeId, stageDir, false /* setupDataCache */) Expect(err).To(BeNil(), "Node Stage volume failed") defer func() { @@ -174,7 +174,7 @@ var _ = Describe("GCE PD CSI Driver", func() { }() // Volume should be attached/formatted/mounted/unmounted/detached - err = testAttachWriteReadDetach(volume.VolumeId, volName, instance, client, false /* readOnly */) + err = testAttachWriteReadDetach(volume.VolumeId, volName, instance, client, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle") // Resize controller @@ -203,7 +203,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Stage Disk stageDir := filepath.Join("/tmp/", volName, "stage") - err = client.NodeStageExt4Volume(volume.VolumeId, stageDir) + err = client.NodeStageExt4Volume(volume.VolumeId, stageDir, false /* setupDataCache */) Expect(err).To(BeNil(), "Node Stage volume failed") defer func() { @@ -295,7 +295,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Stage Disk stageDir := filepath.Join("/tmp/", volName, "stage") - err = client.NodeStageBlockVolume(volume.VolumeId, stageDir) + err = client.NodeStageBlockVolume(volume.VolumeId, stageDir, false /* setupDataCache */) Expect(err).To(BeNil(), "Node Stage volume failed") defer func() { diff --git a/test/e2e/tests/setup_e2e_test.go b/test/e2e/tests/setup_e2e_test.go index 7fd12a52d..8641f3346 100644 --- a/test/e2e/tests/setup_e2e_test.go +++ b/test/e2e/tests/setup_e2e_test.go @@ -19,7 +19,9 @@ import ( "flag" "fmt" "math/rand" + "strconv" "strings" + "sync" "testing" "time" @@ -31,27 +33,37 @@ import ( compute "google.golang.org/api/compute/v1" "k8s.io/klog/v2" "k8s.io/utils/strings/slices" + "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils" remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote" ) var ( - project = flag.String("project", "", "Project to run tests in") - serviceAccount = flag.String("service-account", "", "Service account to bring up instance with") - architecture = flag.String("arch", "amd64", "Architecture pd csi driver build on") - zones = flag.String("zones", "us-east4-a,us-east4-c", "Zones to run tests in. If there are multiple zones, separate each by comma") - machineType = flag.String("machine-type", "n2-standard-2", "Type of machine to provision instance on") - imageURL = flag.String("image-url", "projects/debian-cloud/global/images/family/debian-11", "OS image url to get image from") - runInProw = flag.Bool("run-in-prow", false, "If true, use a Boskos loaned project and special CI service accounts and ssh keys") - deleteInstances = flag.Bool("delete-instances", false, "Delete the instances after tests run") - cloudtopHost = flag.Bool("cloudtop-host", false, "The local host is cloudtop, a kind of googler machine with special requirements to access GCP") - extraDriverFlags = flag.String("extra-driver-flags", "", "Extra flags to pass to the driver") - - testContexts = []*remote.TestContext{} - computeService *compute.Service - computeAlphaService *computealpha.Service - computeBetaService *computebeta.Service - kmsClient *cloudkms.KeyManagementClient + project = flag.String("project", "", "Project to run tests in") + serviceAccount = flag.String("service-account", "", "Service account to bring up instance with") + vmNamePrefix = flag.String("vm-name-prefix", "gce-pd-csi-e2e", "VM name prefix") + architecture = flag.String("arch", "amd64", "Architecture pd csi driver build on") + minCpuPlatform = flag.String("min-cpu-platform", "rome", "Minimum CPU architecture") + mwMinCpuPlatform = flag.String("min-cpu-platform-mw", "sapphirerapids", "Minimum CPU architecture for multiwriter tests") + zones = flag.String("zones", "us-east4-a,us-east4-c", "Zones to run tests in. If there are multiple zones, separate each by comma") + machineType = flag.String("machine-type", "n2d-standard-4", "Type of machine to provision instance on") + imageURL = flag.String("image-url", "projects/ubuntu-os-cloud/global/images/family/ubuntu-minimal-2404-lts-amd64", "OS image url to get image from") + runInProw = flag.Bool("run-in-prow", false, "If true, use a Boskos loaned project and special CI service accounts and ssh keys") + deleteInstances = flag.Bool("delete-instances", false, "Delete the instances after tests run") + cloudtopHost = flag.Bool("cloudtop-host", false, "The local host is cloudtop, a kind of googler machine with special requirements to access GCP") + extraDriverFlags = flag.String("extra-driver-flags", "", "Extra flags to pass to the driver") + enableConfidentialCompute = flag.Bool("enable-confidential-compute", false, "Create VMs with confidential compute mode. This uses NVMe devices") + // Multi-writer is only supported on M3, C3, and N4 + // https://cloud.google.com/compute/docs/disks/sharing-disks-between-vms#hd-multi-writer + hdMachineType = flag.String("hyperdisk-machine-type", "c3-standard-4", "Type of machine to provision instance on") + hdMinCpuPlatform = flag.String("hyperdisk-min-cpu-platform", "sapphirerapids", "Minimum CPU architecture") + + testContexts = []*remote.TestContext{} + hyperdiskTestContexts = []*remote.TestContext{} + computeService *compute.Service + computeAlphaService *computealpha.Service + computeBetaService *computebeta.Service + kmsClient *cloudkms.KeyManagementClient ) func init() { @@ -66,10 +78,12 @@ func TestE2E(t *testing.T) { var _ = BeforeSuite(func() { var err error - tcc := make(chan *remote.TestContext) - defer close(tcc) - + numberOfInstancesPerZone := 2 zones := strings.Split(*zones, ",") + tcc := make(chan *remote.TestContext, len(zones)*numberOfInstancesPerZone) + hdtcc := make(chan *remote.TestContext, len(zones)) + defer close(tcc) + defer close(hdtcc) rand.Seed(time.Now().UnixNano()) @@ -95,18 +109,40 @@ var _ = BeforeSuite(func() { klog.Infof("Running in project %v with service account %v", *project, *serviceAccount) - for _, zone := range zones { + setupContext := func(zone string) { + var wg sync.WaitGroup + // Create 2 instances for each zone as we need 2 instances each zone for certain test cases + for j := 0; j < numberOfInstancesPerZone; j++ { + wg.Add(1) + go func(curZone string, randInt int) { + defer GinkgoRecover() + defer wg.Done() + tcc <- NewDefaultTestContext(curZone, strconv.Itoa(randInt)) + }(zone, j) + } go func(curZone string) { + wg.Add(1) defer GinkgoRecover() - tcc <- NewTestContext(curZone) + defer wg.Done() + hdtcc <- NewTestContext(curZone, *hdMinCpuPlatform, *hdMachineType, "0") }(zone) + wg.Wait() } - for i := 0; i < len(zones); i++ { + for _, zone := range zones { + setupContext(zone) + } + + for i := 0; i < len(zones)*numberOfInstancesPerZone; i++ { tc := <-tcc testContexts = append(testContexts, tc) klog.Infof("Added TestContext for node %s", tc.Instance.GetName()) } + for i := 0; i < len(zones); i++ { + tc := <-hdtcc + hyperdiskTestContexts = append(hyperdiskTestContexts, tc) + klog.Infof("Added TestContext for node %s", tc.Instance.GetName()) + } }) var _ = AfterSuite(func() { @@ -117,6 +153,13 @@ var _ = AfterSuite(func() { tc.Instance.DeleteInstance() } } + for _, mwTc := range hyperdiskTestContexts { + err := remote.TeardownDriverAndClient(mwTc) + Expect(err).To(BeNil(), "Multiwriter Teardown Driver and Client failed with error") + if *deleteInstances { + mwTc.Instance.DeleteInstance() + } + } }) func notEmpty(v string) bool { @@ -130,21 +173,34 @@ func getDriverConfig() testutils.DriverConfig { } } -func getRemoteInstanceConfig() *remote.InstanceConfig { - return &remote.InstanceConfig{ - Project: *project, - Architecture: *architecture, - MachineType: *machineType, - ServiceAccount: *serviceAccount, - ImageURL: *imageURL, - CloudtopHost: *cloudtopHost} +func NewDefaultTestContext(zone string, instanceNumber string) *remote.TestContext { + return NewTestContext(zone, *minCpuPlatform, *machineType, instanceNumber) } -func NewTestContext(zone string) *remote.TestContext { - nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", zone) +func NewTestContext(zone, minCpuPlatform, machineType string, instanceNumber string) *remote.TestContext { + nodeID := fmt.Sprintf("%s-%s-%s-%s", *vmNamePrefix, zone, machineType, instanceNumber) klog.Infof("Setting up node %s", nodeID) - i, err := remote.SetupInstance(getRemoteInstanceConfig(), zone, nodeID, computeService) + instanceConfig := remote.InstanceConfig{ + Project: *project, + Architecture: *architecture, + MinCpuPlatform: minCpuPlatform, + Zone: zone, + Name: nodeID, + MachineType: machineType, + ServiceAccount: *serviceAccount, + ImageURL: *imageURL, + CloudtopHost: *cloudtopHost, + EnableConfidentialCompute: *enableConfidentialCompute, + ComputeService: computeService, + LocalSSDCount: common.LocalSSDCountForDataCache, + } + + if machineType == *hdMachineType { + // Machine type is defaulted to c3-standard-2 which doesn't support LSSD and we don't need LSSD for HdHA test context + instanceConfig.LocalSSDCount = 0 + } + i, err := remote.SetupInstance(instanceConfig) if err != nil { klog.Fatalf("Failed to setup instance %v: %v", nodeID, err) } @@ -163,7 +219,16 @@ func NewTestContext(zone string) *remote.TestContext { if err != nil { klog.Fatalf("Failed to copy google_nvme_id to containerized directory: %v", err) } + pkgs := []string{"lvm2", "mdadm", "grep", "coreutils"} + err = testutils.InstallDependencies(i, pkgs) + if err != nil { + klog.Fatalf("Failed to install dependency package on node %v: error : %v", i.GetNodeID(), err) + } + err = testutils.SetupDataCachingConfig(i) + if err != nil { + klog.Fatalf("Failed to setup data cache required config error %v", err) + } klog.Infof("Creating new driver and client for node %s", i.GetName()) tc, err := testutils.GCEClientAndDriverSetup(i, getDriverConfig()) if err != nil { @@ -179,3 +244,9 @@ func getRandomTestContext() *remote.TestContext { rn := rand.Intn(len(testContexts)) return testContexts[rn] } + +func getRandomMwTestContext() *remote.TestContext { + Expect(hyperdiskTestContexts).ToNot(BeEmpty()) + rn := rand.Intn(len(hyperdiskTestContexts)) + return hyperdiskTestContexts[rn] +} diff --git a/test/e2e/tests/single_zone_e2e_test.go b/test/e2e/tests/single_zone_e2e_test.go index 5bb989441..307675036 100644 --- a/test/e2e/tests/single_zone_e2e_test.go +++ b/test/e2e/tests/single_zone_e2e_test.go @@ -97,7 +97,7 @@ var _ = Describe("GCE PD CSI Driver", func() { }() // Attach Disk - err := testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */) + err := testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle") }) @@ -145,7 +145,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Stage Disk stageDir := filepath.Join("/tmp/", volName, "stage") - err = client.NodeStageExt4Volume(volID, stageDir) + err = client.NodeStageExt4Volume(volID, stageDir, false /* setupDataCache */) Expect(err).To(BeNil(), "failed to repair /dev/by-id symlink and stage volume") // Validate that the link is correct @@ -215,7 +215,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Stage Disk stageDir := filepath.Join("/tmp/", volName, "stage") - err = client.NodeStageExt4Volume(volID, stageDir) + err = client.NodeStageExt4Volume(volID, stageDir, false /* setupDataCache */) Expect(err).To(BeNil(), "failed to repair /dev/by-id symlink and stage volume") // Validate that the link is correct @@ -322,7 +322,7 @@ var _ = Describe("GCE PD CSI Driver", func() { }() // Attach Disk - err := testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */) + err := testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache*/) Expect(err).To(BeNil(), "Failed to go through volume lifecycle") }, Entry("on pd-standard", standardDiskType), @@ -579,7 +579,14 @@ var _ = Describe("GCE PD CSI Driver", func() { defer func() { // Delete Disk - err := client.DeleteVolume(volID) + err = wait.Poll(5*time.Second, 1*time.Minute, func() (bool, error) { + err := client.DeleteVolume(volID) + if err == nil { + return true, err + } + return false, err + }) + Expect(err).To(BeNil(), "DeleteVolume failed") // Validate Disk Deleted @@ -666,7 +673,7 @@ var _ = Describe("GCE PD CSI Driver", func() { }() // Test disk works - err = testAttachWriteReadDetach(volume.VolumeId, volName, controllerInstance, controllerClient, false /* readOnly */) + err = testAttachWriteReadDetach(volume.VolumeId, volName, controllerInstance, controllerClient, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle before revoking CMEK key") // Revoke CMEK key @@ -687,7 +694,7 @@ var _ = Describe("GCE PD CSI Driver", func() { } // Make sure attach of PD fails - err = testAttachWriteReadDetach(volume.VolumeId, volName, controllerInstance, controllerClient, false /* readOnly */) + err = testAttachWriteReadDetach(volume.VolumeId, volName, controllerInstance, controllerClient, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).ToNot(BeNil(), "Volume lifecycle should have failed, but succeeded") // Restore CMEK key @@ -708,7 +715,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // The controller publish failure in above step would set a backoff condition on the node. Wait suffcient amount of time for the driver to accept new controller publish requests. time.Sleep(time.Second) // Make sure attach of PD succeeds - err = testAttachWriteReadDetach(volume.VolumeId, volName, controllerInstance, controllerClient, false /* readOnly */) + err = testAttachWriteReadDetach(volume.VolumeId, volName, controllerInstance, controllerClient, false /* readOnly */, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle after restoring CMEK key") }, Entry("on pd-standard", standardDiskType), @@ -851,7 +858,7 @@ var _ = Describe("GCE PD CSI Driver", func() { } // Attach Disk - err := testLifecycleWithVerify(volID, volName, instance, client, false /* readOnly */, true /* block */, verifyVolumeStats, nil) + err := testLifecycleWithVerify(volID, volName, instance, client, false /* readOnly */, true /* block */, verifyVolumeStats, nil, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle") }) @@ -888,7 +895,7 @@ var _ = Describe("GCE PD CSI Driver", func() { } // Attach Disk - err := testLifecycleWithVerify(volID, volName, instance, client, false /* readOnly */, false /* fs */, verifyVolumeStats, nil) + err := testLifecycleWithVerify(volID, volName, instance, client, false /* readOnly */, false /* fs */, verifyVolumeStats, nil, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle") }) @@ -957,7 +964,7 @@ var _ = Describe("GCE PD CSI Driver", func() { } return nil } - err := testLifecycleWithVerify(volID, volName, instance, client, false /* readOnly */, true /* block */, writeFunc, verifyReadFunc) + err := testLifecycleWithVerify(volID, volName, instance, client, false /* readOnly */, true /* block */, writeFunc, verifyReadFunc, false /* detachAndReattach */, false /* setupDataCache */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle") }) @@ -1317,13 +1324,7 @@ var _ = Describe("GCE PD CSI Driver", func() { _, err := getRandomTestContext().Client.ListVolumes() Expect(err).To(BeNil(), "no error expected when passed valid compute url") - zone := "us-central1-c" - nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", zone) - i, err := remote.SetupInstance(getRemoteInstanceConfig(), zone, nodeID, computeService) - - if err != nil { - klog.Fatalf("Failed to setup instance %v: %v", nodeID, err) - } + i := getRandomTestContext().Instance klog.Infof("Creating new driver and client for node %s\n", i.GetName()) @@ -1395,7 +1396,7 @@ var _ = Describe("GCE PD CSI Driver", func() { Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, }, } - err = client.NodeStageVolume(volID, stageDir, volCap) + err = client.NodeStageVolume(volID, stageDir, volCap, false /* setupDataCache */) Expect(err).To(BeNil(), "failed to stage volume: %v", err) // Validate that the link is correct @@ -1432,6 +1433,112 @@ var _ = Describe("GCE PD CSI Driver", func() { } }() }) + It("Should create disks, attach them to instance with local ssd, setup caching between LSSD->detach->reattach to same instance", func() { + Expect(testContexts).ToNot(BeEmpty()) + testContext := getRandomTestContext() + + p, z, _ := testContext.Instance.GetIdentity() + client := testContext.Client + instance := testContext.Instance + volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) + defer deleteVolumeOrError(client, volID) + + // Attach Disk + err := testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */, true /* detachAndReattach */, true /* setupDataCache */) + Expect(err).To(BeNil(), "Failed to go through volume lifecycle") + + }) + It("Should create->attach->setup caching->write->detach->attach to different node->mount->read", func() { + Expect(testContexts).ToNot(BeEmpty()) + zoneToContext := map[string][]*remote.TestContext{} + testZoneContexts := []*remote.TestContext{} + for _, tc := range testContexts { + _, z, _ := tc.Instance.GetIdentity() + // Zone hasn't been seen before + if _, ok := zoneToContext[z]; !ok { + zoneToContext[z] = []*remote.TestContext{tc} + } else { + zoneToContext[z] = append(zoneToContext[z], tc) + } + if len(zoneToContext[z]) >= 2 { + testZoneContexts = zoneToContext[z] + break + } + } + if len(testZoneContexts) < 2 { + klog.Fatalf("No test contexts setup %v", testZoneContexts) + } + testContextForVm1 := testZoneContexts[0] + p, z, _ := testContextForVm1.Instance.GetIdentity() + + client := testContextForVm1.Client + firstInstance := testContextForVm1.Instance + + volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) + defer deleteVolumeOrError(client, volID) + + testContextForVm2 := testZoneContexts[1] + secondClient := testContextForVm2.Client + secondInstance := testContextForVm2.Instance + unmountDisk := func(client *remote.CsiClient, volID string, args *verifyArgs) { + err := client.NodeUnpublishVolume(volID, args.publishDir) + if err != nil { + klog.Errorf("NodeUnpublishVolume failed with error: %v", err) + } + } + attachMountArgs := attachAndMountArgs{ + readOnly: false, + useBlock: false, + forceAttach: false, + setupDataCache: true, + } + // Controller Publish (Attach) - Node Stage - Node Publish(Mount) Volume + err, _, args := testAttachAndMount(volID, volName, firstInstance, client, attachMountArgs) + if err != nil { + klog.Errorf("Failed to attach and mount: %v", err.Error()) + } + // Write file in the volume + firstMountVerify, _ := testWriteAndReadFile(firstInstance, false /* readOnly */) + err = firstMountVerify(args) + if err != nil { + klog.Errorf("failed to verify after first mount to %s: %v", args, err) + } + Expect(err).To(BeNil(), "Failed to write data to volume %s on instance %s", volName, firstInstance.GetName()) + // Unmount Disk + unmountDisk(client, volID, args) + + // Node Unstage + err = client.NodeUnstageVolume(volID, args.stageDir) + if err != nil { + klog.Errorf("Failed to unstage volume: %v", err) + } + detach(volID, firstInstance, client) + + // Attach Disk to secondInstance + err, detacher, stageDir := testAttach(volID, volName, secondInstance, secondClient, attachMountArgs) + if err != nil { + klog.Errorf("Failed to attach disk %v", err) + } + defer func() { + detacher() + deleteVolumeOrError(secondClient, volID) + }() + + // Mount disk + err, _, args = testMount(volID, volName, secondInstance, secondClient, attachMountArgs, stageDir) + if err != nil { + klog.Fatalf("Failed to mount disk %v", err) + } + _, secondMountRead := testWriteAndReadFile(secondInstance, false /* readOnly */) + err = secondMountRead(args) + if err != nil { + klog.Errorf("failed to verify after second mount to %s: %v", args, err) + } + // Unmount disk for cleanup + unmountDisk(secondClient, volID, args) + Expect(err).To(BeNil(), "Failed to read data from volume %s on instance %s", volName, secondInstance.GetName()) + + }) It("Should block unstage if filesystem mounted", func() { testContext := getRandomTestContext() @@ -1467,7 +1574,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Stage Disk stageDir := filepath.Join("/tmp/", volName, "stage") - err = client.NodeStageExt4Volume(volID, stageDir) + err = client.NodeStageExt4Volume(volID, stageDir, false) Expect(err).To(BeNil(), "failed to stage volume: %v", err) // Create private bind mount diff --git a/test/e2e/utils/utils.go b/test/e2e/utils/utils.go index 995b114b0..f558a27af 100644 --- a/test/e2e/utils/utils.go +++ b/test/e2e/utils/utils.go @@ -68,6 +68,11 @@ func GCEClientAndDriverSetup(instance *remote.InstanceInfo, driverConfig DriverC "--use-instance-api-to-list-volumes-published-nodes", fmt.Sprintf("--fallback-requisite-zones=%s", strings.Join(driverConfig.Zones, ",")), } + + if instance.GetLocalSSD() > 0 { + extra_flags = append(extra_flags, "--enable-data-cache") + extra_flags = append(extra_flags, fmt.Sprintf("--node-name=%s", utilcommon.TestNode)) + } extra_flags = append(extra_flags, fmt.Sprintf("--compute-endpoint=%s", driverConfig.ComputeEndpoint)) extra_flags = append(extra_flags, driverConfig.ExtraFlags...) @@ -274,6 +279,33 @@ func CopyFile(instance *remote.InstanceInfo, src, dest string) error { return nil } +func InstallDependencies(instance *remote.InstanceInfo, pkgs []string) error { + _, _ = instance.SSH("apt-get", "update") + for _, pkg := range pkgs { + output, err := instance.SSH("apt-get", "install", "-y", pkg) + if err != nil { + return fmt.Errorf("failed to install package %s. Output: %v, errror: %v", pkg, output, err.Error()) + } + } + return nil +} + +func SetupDataCachingConfig(instance *remote.InstanceInfo) error { + output, err := instance.SSH("/bin/sed", "-i", "-e", "\"s/.*allow_mixed_block_sizes = 0.*/ allow_mixed_block_sizes = 1/\"", "/etc/lvm/lvm.conf") + if err != nil { + return fmt.Errorf("failed to update field allow_mixed_block_sizes, error:%v; output: %v", err, output) + } + output, err = instance.SSH("/bin/sed", "-i", "-e", "\"s/.*udev_sync = 1.*/ udev_sync = 0/\"", "/etc/lvm/lvm.conf") + if err != nil { + return fmt.Errorf("failed to update field udev_sync, error:%v; output: %v", err, output) + } + output, err = instance.SSH("/bin/sed", "-i", "-e", "\"s/.*udev_rules = 1.*/ udev_rules = 0/\"", "/etc/lvm/lvm.conf") + if err != nil { + return fmt.Errorf("failed to update field udev_rules, error:%v; output: %v", err, output) + } + return nil +} + // ValidateLogicalLinkIsDisk takes a symlink location at "link" and finds the // link location - it then finds the backing PD using either scsi_id or // google_nvme_id (depending on the /dev path) and validates that it is the diff --git a/test/k8s-integration/config/data-cache-sc.yaml b/test/k8s-integration/config/data-cache-sc.yaml new file mode 100644 index 000000000..7e911eb36 --- /dev/null +++ b/test/k8s-integration/config/data-cache-sc.yaml @@ -0,0 +1,11 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: csi-gcepd-balanced +provisioner: pd.csi.storage.gke.io +parameters: + type: pd-balanced + data-cache-mode: writeback + data-cache-size: "50Gi" +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true \ No newline at end of file diff --git a/test/remote/client-wrappers.go b/test/remote/client-wrappers.go index 671cbb1df..a7445c0a2 100644 --- a/test/remote/client-wrappers.go +++ b/test/remote/client-wrappers.go @@ -52,6 +52,14 @@ var ( } ) +const ( + // Keys in the volume context. + contextForceAttach = "force-attach" + + defaultLocalSsdCacheSize = "200Gi" + defaultDataCacheMode = common.DataCacheModeWriteThrough +) + type CsiClient struct { conn *grpc.ClientConn idClient csipb.IdentityClient @@ -179,19 +187,25 @@ func (c *CsiClient) ControllerUnpublishVolume(volId, nodeId string) error { return err } -func (c *CsiClient) NodeStageExt4Volume(volId, stageDir string) error { - return c.NodeStageVolume(volId, stageDir, stdVolCap) +func (c *CsiClient) NodeStageExt4Volume(volId, stageDir string, setupDataCache bool) error { + return c.NodeStageVolume(volId, stageDir, stdVolCap, setupDataCache) } -func (c *CsiClient) NodeStageBlockVolume(volId, stageDir string) error { - return c.NodeStageVolume(volId, stageDir, blockVolCap) +func (c *CsiClient) NodeStageBlockVolume(volId, stageDir string, setupDataCache bool) error { + return c.NodeStageVolume(volId, stageDir, blockVolCap, setupDataCache) } -func (c *CsiClient) NodeStageVolume(volId, stageDir string, volumeCap *csipb.VolumeCapability) error { +func (c *CsiClient) NodeStageVolume(volId string, stageDir string, volumeCap *csipb.VolumeCapability, setupDataCache bool) error { + publishContext := map[string]string{} + if setupDataCache { + publishContext[common.ContextDataCacheSize] = defaultLocalSsdCacheSize + publishContext[common.ContextDataCacheMode] = defaultDataCacheMode + } nodeStageReq := &csipb.NodeStageVolumeRequest{ VolumeId: volId, StagingTargetPath: stageDir, VolumeCapability: volumeCap, + PublishContext: publishContext, } _, err := c.nodeClient.NodeStageVolume(context.Background(), nodeStageReq) return err diff --git a/test/remote/instance.go b/test/remote/instance.go index 6fb1b5233..554e7612e 100644 --- a/test/remote/instance.go +++ b/test/remote/instance.go @@ -47,61 +47,72 @@ const ( // InstanceConfig is the common bundle of options used for instance creation. type InstanceConfig struct { - Project string - Architecture string - MachineType string - ServiceAccount string - ImageURL string - CloudtopHost bool + Project string + Architecture string + Zone string + Name string + MachineType string + ServiceAccount string + ImageURL string + CloudtopHost bool + MinCpuPlatform string + ComputeService *compute.Service + EnableConfidentialCompute bool + LocalSSDCount int64 + EnableDataCache bool } type InstanceInfo struct { - project string - architecture string - zone string - name string - machineType string - serviceAccount string - imageURL string - cloudtopHost bool - + cfg InstanceConfig // External IP is filled in after instance creation externalIP string - - computeService *compute.Service } func (i *InstanceInfo) GetIdentity() (string, string, string) { - return i.project, i.zone, i.name + return i.cfg.Project, i.cfg.Zone, i.cfg.Name } func (i *InstanceInfo) GetName() string { - return i.name + return i.cfg.Name } func (i *InstanceInfo) GetNodeID() string { - return common.CreateNodeID(i.project, i.zone, i.name) + return common.CreateNodeID(i.cfg.Project, i.cfg.Zone, i.cfg.Name) +} + +func (i *InstanceInfo) GetLocalSSD() int64 { + return i.cfg.LocalSSDCount } -func CreateInstanceInfo(config *InstanceConfig, zone, name string, cs *compute.Service) (*InstanceInfo, error) { - return &InstanceInfo{ - project: config.Project, - architecture: config.Architecture, - zone: zone, - name: name, - machineType: config.MachineType, - cloudtopHost: config.CloudtopHost, - serviceAccount: config.ServiceAccount, - imageURL: config.ImageURL, - computeService: cs, - }, nil +func machineTypeMismatch(curInst *compute.Instance, newInst *compute.Instance) bool { + if !strings.Contains(curInst.MachineType, newInst.MachineType) { + klog.Infof("Machine type mismatch") + return true + } + // Ideally we could compare to see if the new instance has a greater minCpuPlatfor + // For now we just check it was set and it's different. + if curInst.MinCpuPlatform != "" && curInst.MinCpuPlatform != newInst.MinCpuPlatform { + klog.Infof("CPU Platform mismatch") + return true + } + if (curInst.ConfidentialInstanceConfig != nil && newInst.ConfidentialInstanceConfig == nil) || + (curInst.ConfidentialInstanceConfig == nil && newInst.ConfidentialInstanceConfig != nil) || + (curInst.ConfidentialInstanceConfig != nil && newInst.ConfidentialInstanceConfig != nil && curInst.ConfidentialInstanceConfig.EnableConfidentialCompute != newInst.ConfidentialInstanceConfig.EnableConfidentialCompute) { + klog.Infof("Confidential compute mismatch") + return true + } + if curInst.SourceMachineImage != newInst.SourceMachineImage { + klog.Infof("Source Machine Mismatch") + return true + } + return false } // Provision a gce instance using image -func (i *InstanceInfo) CreateOrGetInstance() error { +func (i *InstanceInfo) CreateOrGetInstance(localSSDCount int) error { var err error var instance *compute.Instance - klog.V(4).Infof("Creating instance: %v", i.name) + klog.V(4).Infof("Creating instance: %v", i.cfg.Name) myuuid := string(uuid.NewUUID()) @@ -111,8 +122,8 @@ func (i *InstanceInfo) CreateOrGetInstance() error { } newInst := &compute.Instance{ - Name: i.name, - MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", i.zone, i.machineType), + Name: i.cfg.Name, + MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", i.cfg.Zone, i.cfg.MachineType), NetworkInterfaces: []*compute.NetworkInterface{ { AccessConfigs: []*compute.AccessConfig{ @@ -129,14 +140,33 @@ func (i *InstanceInfo) CreateOrGetInstance() error { Type: "PERSISTENT", InitializeParams: &compute.AttachedDiskInitializeParams{ DiskName: "my-root-pd-" + myuuid, - SourceImage: i.imageURL, + SourceImage: i.cfg.ImageURL, }, }, }, + MinCpuPlatform: i.cfg.MinCpuPlatform, + } + + if i.cfg.EnableConfidentialCompute { + newInst.ConfidentialInstanceConfig = &compute.ConfidentialInstanceConfig{ + EnableConfidentialCompute: true, + } + } + + localSSDConfig := &compute.AttachedDisk{ + Type: "SCRATCH", + InitializeParams: &compute.AttachedDiskInitializeParams{ + DiskType: fmt.Sprintf("zones/%s/diskTypes/local-ssd", i.cfg.Zone), + }, + AutoDelete: true, + Interface: "NVME", } + for i := 0; i < localSSDCount; i++ { + newInst.Disks = append(newInst.Disks, localSSDConfig) + } saObj := &compute.ServiceAccount{ - Email: i.serviceAccount, + Email: i.cfg.ServiceAccount, Scopes: []string{"https://www.googleapis.com/auth/cloud-platform"}, } newInst.ServiceAccounts = []*compute.ServiceAccount{saObj} @@ -150,19 +180,19 @@ func (i *InstanceInfo) CreateOrGetInstance() error { newInst.Metadata = meta } - // If instance exists but machine-type doesn't match, delete instance - curInst, _ := i.computeService.Instances.Get(i.project, i.zone, newInst.Name).Do() + // If instance exists but settings differ, delete instance + curInst, _ := i.cfg.ComputeService.Instances.Get(i.cfg.Project, i.cfg.Zone, newInst.Name).Do() if curInst != nil { - if !strings.Contains(curInst.MachineType, newInst.MachineType) { + if machineTypeMismatch(curInst, newInst) { klog.V(4).Infof("Instance machine type doesn't match the required one. Delete instance.") - if _, err := i.computeService.Instances.Delete(i.project, i.zone, i.name).Do(); err != nil { + if _, err := i.cfg.ComputeService.Instances.Delete(i.cfg.Project, i.cfg.Zone, i.cfg.Name).Do(); err != nil { return err } start := time.Now() err := wait.Poll(15*time.Second, 5*time.Minute, func() (bool, error) { klog.V(2).Infof("Waiting for instance to be deleted. %v elapsed", time.Since(start)) - if curInst, _ = i.computeService.Instances.Get(i.project, i.zone, i.name).Do(); curInst != nil { + if curInst, _ = i.cfg.ComputeService.Instances.Get(i.cfg.Project, i.cfg.Zone, i.cfg.Name).Do(); curInst != nil { return false, nil } return true, nil @@ -174,16 +204,16 @@ func (i *InstanceInfo) CreateOrGetInstance() error { } if curInst == nil { - op, err := i.computeService.Instances.Insert(i.project, i.zone, newInst).Do() - klog.V(4).Infof("Inserted instance %v in project: %v, zone: %v", newInst.Name, i.project, i.zone) + op, err := i.cfg.ComputeService.Instances.Insert(i.cfg.Project, i.cfg.Zone, newInst).Do() + klog.V(4).Infof("Inserted instance %v in project: %v, zone: %v", newInst.Name, i.cfg.Project, i.cfg.Zone) if err != nil { - ret := fmt.Sprintf("could not create instance %s: API error: %v", i.name, err.Error()) + ret := fmt.Sprintf("could not create instance %s: API error: %v", i.cfg.Name, err.Error()) if op != nil { ret = fmt.Sprintf("%s. op error: %v", ret, op.Error) } return errors.New(ret) } else if op.Error != nil { - return fmt.Errorf("could not create instance %s: %+v", i.name, op.Error) + return fmt.Errorf("could not create instance %s: %+v", i.cfg.Name, op.Error) } } else { klog.V(4).Infof("Compute service GOT instance %v, skipping instance creation", newInst.Name) @@ -191,26 +221,26 @@ func (i *InstanceInfo) CreateOrGetInstance() error { start := time.Now() err = wait.Poll(15*time.Second, 5*time.Minute, func() (bool, error) { - klog.V(2).Infof("Waiting for instance %v to come up. %v elapsed", i.name, time.Since(start)) + klog.V(2).Infof("Waiting for instance %v to come up. %v elapsed", i.cfg.Name, time.Since(start)) - instance, err = i.computeService.Instances.Get(i.project, i.zone, i.name).Do() + instance, err = i.cfg.ComputeService.Instances.Get(i.cfg.Project, i.cfg.Zone, i.cfg.Name).Do() if err != nil { - klog.Errorf("Failed to get instance %q: %v", i.name, err) + klog.Errorf("Failed to get instance %q: %v", i.cfg.Name, err) return false, nil } if strings.ToUpper(instance.Status) != "RUNNING" { - klog.Warningf("instance %s not in state RUNNING, was %s", i.name, instance.Status) + klog.Warningf("instance %s not in state RUNNING, was %s", i.cfg.Name, instance.Status) return false, nil } - if i.cloudtopHost { - output, err := exec.Command("gcloud", "compute", "ssh", i.name, "--zone", i.zone, "--project", i.project, "--", "-o", "ProxyCommand=corp-ssh-helper %h %p", "--", "echo").CombinedOutput() + if i.cfg.CloudtopHost { + output, err := exec.Command("gcloud", "compute", "ssh", i.cfg.Name, "--zone", i.cfg.Zone, "--project", i.cfg.Project, "--", "-o", "ProxyCommand=corp-ssh-helper %h %p", "--", "echo").CombinedOutput() if err != nil { klog.Errorf("Failed to bootstrap ssh (%v): %s", err, string(output)) return false, nil } - klog.V(4).Infof("Bootstrapped cloudtop ssh for instance %v", i.name) + klog.V(4).Infof("Bootstrapped cloudtop ssh for instance %v", i.cfg.Name) } externalIP := getexternalIP(instance) @@ -219,11 +249,11 @@ func (i *InstanceInfo) CreateOrGetInstance() error { } if sshOut, err := i.SSHCheckAlive(); err != nil { - err = fmt.Errorf("Instance %v in state RUNNING but not available by SSH: %v", i.name, err.Error()) + err = fmt.Errorf("Instance %v in state RUNNING but not available by SSH: %v", i.cfg.Name, err.Error()) klog.Warningf("SSH encountered an error: %v, output: %v", err, sshOut) return false, nil } - klog.V(4).Infof("Instance %v in state RUNNING and available by SSH", i.name) + klog.V(4).Infof("Instance %v in state RUNNING and available by SSH", i.cfg.Name) return true, nil }) @@ -233,24 +263,24 @@ func (i *InstanceInfo) CreateOrGetInstance() error { } // Instance reached running state in time, make sure that cloud-init is complete - klog.V(2).Infof("Instance %v has been created successfully", i.name) + klog.V(2).Infof("Instance %v has been created successfully", i.cfg.Name) return nil } func (i *InstanceInfo) DeleteInstance() { - klog.V(4).Infof("Deleting instance %q", i.name) - _, err := i.computeService.Instances.Delete(i.project, i.zone, i.name).Do() + klog.V(4).Infof("Deleting instance %q", i.cfg.Name) + _, err := i.cfg.ComputeService.Instances.Delete(i.cfg.Project, i.cfg.Zone, i.cfg.Name).Do() if err != nil { if isGCEError(err, "notFound") { return } - klog.Errorf("Error deleting instance %q: %v", i.name, err) + klog.Errorf("Error deleting instance %q: %v", i.cfg.Name, err) } } func (i *InstanceInfo) DetachDisk(diskName string) error { klog.V(4).Infof("Detaching disk %q", diskName) - op, err := i.computeService.Instances.DetachDisk(i.project, i.zone, i.name, diskName).Do() + op, err := i.cfg.ComputeService.Instances.DetachDisk(i.cfg.Project, i.cfg.Zone, i.cfg.Name, diskName).Do() if err != nil { if isGCEError(err, "notFound") { return nil @@ -260,9 +290,9 @@ func (i *InstanceInfo) DetachDisk(diskName string) error { start := time.Now() if err := wait.Poll(5*time.Second, 1*time.Minute, func() (bool, error) { - klog.V(2).Infof("Waiting for disk %q to be detached from instance %q. %v elapsed", diskName, i.name, time.Since(start)) + klog.V(2).Infof("Waiting for disk %q to be detached from instance %q. %v elapsed", diskName, i.cfg.Name, time.Since(start)) - op, err = i.computeService.ZoneOperations.Get(i.project, i.zone, op.Name).Do() + op, err = i.cfg.ComputeService.ZoneOperations.Get(i.cfg.Project, i.cfg.Zone, op.Name).Do() if err != nil { return true, fmt.Errorf("Failed to get operation %q, err: %v", op.Name, err) } @@ -271,7 +301,7 @@ func (i *InstanceInfo) DetachDisk(diskName string) error { return err } - klog.V(4).Infof("Disk %q has been successfully detached from instance %q\n%v", diskName, i.name, op.Error) + klog.V(4).Infof("Disk %q has been successfully detached from instance %q\n%v", diskName, i.cfg.Name, op.Error) return nil } @@ -289,7 +319,7 @@ func getexternalIP(instance *compute.Instance) string { } func getTimestamp() string { - return fmt.Sprintf(time.Now().Format(timestampFormat)) + return fmt.Sprintf("%s", time.Now().Format(timestampFormat)) } // Create default SSH filewall rule if it does not exist @@ -297,7 +327,7 @@ func (i *InstanceInfo) createDefaultFirewallRule() error { var err error klog.V(4).Infof("Creating default firewall rule %s...", defaultFirewallRule) - if _, err = i.computeService.Firewalls.Get(i.project, defaultFirewallRule).Do(); err != nil { + if _, err = i.cfg.ComputeService.Firewalls.Get(i.cfg.Project, defaultFirewallRule).Do(); err != nil { klog.V(4).Infof("Default firewall rule %v does not exist, creating", defaultFirewallRule) f := &compute.Firewall{ Name: defaultFirewallRule, @@ -308,7 +338,7 @@ func (i *InstanceInfo) createDefaultFirewallRule() error { }, }, } - _, err = i.computeService.Firewalls.Insert(i.project, f).Do() + _, err = i.cfg.ComputeService.Firewalls.Insert(i.cfg.Project, f).Do() if err != nil { if gce.IsGCEError(err, "alreadyExists") { klog.V(4).Infof("Default firewall rule %v already exists, skipping creation", defaultFirewallRule) diff --git a/test/remote/runner.go b/test/remote/runner.go index 2fb8d8c93..1b2120810 100644 --- a/test/remote/runner.go +++ b/test/remote/runner.go @@ -29,12 +29,12 @@ import ( func (i *InstanceInfo) UploadAndRun(archivePath, remoteWorkspace, driverRunCmd string) (int, error) { // Create the temp staging directory - klog.V(4).Infof("Staging test binaries on %q", i.name) + klog.V(4).Infof("Staging test binaries on %q", i.cfg.Name) // Do not sudo here, so that we can use scp to copy test archive to the directdory. if output, err := i.SSHNoSudo("mkdir", remoteWorkspace); err != nil { // Exit failure with the error - return -1, fmt.Errorf("failed to create remoteWorkspace directory %q on i.name %q: %v output: %q", remoteWorkspace, i.name, err.Error(), output) + return -1, fmt.Errorf("failed to create remoteWorkspace directory %q on i.name %q: %v output: %q", remoteWorkspace, i.cfg.Name, err.Error(), output) } // Copy the archive to the staging directory @@ -49,7 +49,7 @@ func (i *InstanceInfo) UploadAndRun(archivePath, remoteWorkspace, driverRunCmd s fmt.Sprintf("cd %s", remoteWorkspace), fmt.Sprintf("tar -xzvf ./%s", archiveName), ) - klog.V(4).Infof("Extracting tar on %q", i.name) + klog.V(4).Infof("Extracting tar on %q", i.cfg.Name) // Do not use sudo here, because `sudo tar -x` will recover the file ownership inside the tar ball, but // we want the extracted files to be owned by the current user. if output, err := i.SSHNoSudo("sh", "-c", cmd); err != nil { @@ -57,7 +57,7 @@ func (i *InstanceInfo) UploadAndRun(archivePath, remoteWorkspace, driverRunCmd s return -1, fmt.Errorf("failed to extract test archive: %v, output: %q", err.Error(), output) } - klog.V(4).Infof("Starting driver on %q", i.name) + klog.V(4).Infof("Starting driver on %q", i.cfg.Name) // When the process is killed the driver should close the TCP endpoint, then we want to download the logs output, err := i.SSH(driverRunCmd) if err != nil { diff --git a/test/remote/setup-teardown.go b/test/remote/setup-teardown.go index 3db9fcf1e..3026b28b0 100644 --- a/test/remote/setup-teardown.go +++ b/test/remote/setup-teardown.go @@ -20,7 +20,6 @@ import ( "fmt" "os" - compute "google.golang.org/api/compute/v1" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/klog/v2" ) @@ -54,14 +53,13 @@ type processes struct { } // SetupInstance sets up the specified GCE Instance for E2E testing and returns a handle to the instance object for future use. -func SetupInstance(config *InstanceConfig, instanceZone, instanceName string, cs *compute.Service) (*InstanceInfo, error) { +func SetupInstance(cfg InstanceConfig) (*InstanceInfo, error) { // Create the instance in the requisite zone - instance, err := CreateInstanceInfo(config, instanceZone, instanceName, cs) - if err != nil { - return nil, err + instance := &InstanceInfo{ + cfg: cfg, } - err = instance.CreateOrGetInstance() + err := instance.CreateOrGetInstance(int(cfg.LocalSSDCount)) if err != nil { return nil, err } @@ -73,7 +71,7 @@ func SetupInstance(config *InstanceConfig, instanceZone, instanceName string, cs // that the driver is on and the CSI Client object to make CSI calls to the remote driver. func SetupNewDriverAndClient(instance *InstanceInfo, config *ClientConfig) (*TestContext, error) { archiveName := fmt.Sprintf("e2e_driver_binaries_%s.tar.gz", uuid.NewUUID()) - archivePath, err := CreateDriverArchive(archiveName, instance.architecture, config.PkgPath, config.BinPath) + archivePath, err := CreateDriverArchive(archiveName, instance.cfg.Architecture, config.PkgPath, config.BinPath) if err != nil { return nil, err } @@ -84,6 +82,13 @@ func SetupNewDriverAndClient(instance *InstanceInfo, config *ClientConfig) (*Tes } }() + // Copy dependencies + _, _ = instance.SSH("apt-get", "update") + output, err := instance.SSH("apt-get", "install", "-y", "mdadm", "lvm2") + if err != nil { + return nil, fmt.Errorf("failed to install dependencies. Output: %v, errror: %v", output, err.Error()) + } + // Upload archive to instance and run binaries driverPID, err := instance.UploadAndRun(archivePath, config.WorkspaceDir, config.RunDriverCmd) if err != nil { diff --git a/test/run-e2e.sh b/test/run-e2e.sh index 79144ea31..e86931c0d 100755 --- a/test/run-e2e.sh +++ b/test/run-e2e.sh @@ -5,4 +5,9 @@ set -x readonly PKGDIR=sigs.k8s.io/gcp-compute-persistent-disk-csi-driver -go test --timeout 30m --v "${PKGDIR}/test/e2e/tests" --run-in-prow=true --delete-instances=true --logtostderr +TIMEOUT=50m +if [ "$RUN_CONTROLLER_MODIFY_VOLUME_TESTS" = true ]; then + TIMEOUT=45m +fi + +go test --timeout "${TIMEOUT}" --v "${PKGDIR}/test/e2e/tests" --run-in-prow=true --delete-instances=true --logtostderr $@ diff --git a/test/sanity/sanity_test.go b/test/sanity/sanity_test.go index f09c64ef1..5b6251f2d 100644 --- a/test/sanity/sanity_test.go +++ b/test/sanity/sanity_test.go @@ -63,17 +63,21 @@ func TestSanity(t *testing.T) { fallbackRequisiteZones := []string{} enableStoragePools := false + enableDataCache := true multiZoneVolumeHandleConfig := driver.MultiZoneVolumeHandleConfig{} listVolumesConfig := driver.ListVolumesConfig{} mounter := mountmanager.NewFakeSafeMounter() deviceUtils := deviceutils.NewFakeDeviceUtils(true) + args := driver.NodeServerArgs{ + EnableDataCache: true, + } //Initialize GCE Driver identityServer := driver.NewIdentityServer(gceDriver) - controllerServer := driver.NewControllerServer(gceDriver, cloudProvider, 0, 5*time.Minute, fallbackRequisiteZones, enableStoragePools, multiZoneVolumeHandleConfig, listVolumesConfig) + controllerServer := driver.NewControllerServer(gceDriver, cloudProvider, 0, 5*time.Minute, fallbackRequisiteZones, enableStoragePools, enableDataCache, multiZoneVolumeHandleConfig, listVolumesConfig) fakeStatter := mountmanager.NewFakeStatterWithOptions(mounter, mountmanager.FakeStatterOptions{IsBlock: false}) - nodeServer := driver.NewNodeServer(gceDriver, mounter, deviceUtils, metadataservice.NewFakeService(), fakeStatter) + nodeServer := driver.NewNodeServer(gceDriver, mounter, deviceUtils, metadataservice.NewFakeService(), fakeStatter, args) err = gceDriver.SetupGCEDriver(driverName, vendorVersion, extraLabels, nil, identityServer, controllerServer, nodeServer) if err != nil { t.Fatalf("Failed to initialize GCE CSI Driver: %v", err.Error()) diff --git a/vendor/github.com/fsnotify/fsnotify/.cirrus.yml b/vendor/github.com/fsnotify/fsnotify/.cirrus.yml new file mode 100644 index 000000000..f4e7dbf37 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/.cirrus.yml @@ -0,0 +1,14 @@ +freebsd_task: + name: 'FreeBSD' + freebsd_instance: + image_family: freebsd-14-1 + install_script: + - pkg update -f + - pkg install -y go + test_script: + # run tests as user "cirrus" instead of root + - pw useradd cirrus -m + - chown -R cirrus:cirrus . + - FSNOTIFY_BUFFER=4096 sudo --preserve-env=FSNOTIFY_BUFFER -u cirrus go test -parallel 1 -race ./... + - sudo --preserve-env=FSNOTIFY_BUFFER -u cirrus go test -parallel 1 -race ./... + - FSNOTIFY_DEBUG=1 sudo --preserve-env=FSNOTIFY_BUFFER -u cirrus go test -parallel 1 -race -v ./... diff --git a/vendor/github.com/fsnotify/fsnotify/.editorconfig b/vendor/github.com/fsnotify/fsnotify/.editorconfig deleted file mode 100644 index fad895851..000000000 --- a/vendor/github.com/fsnotify/fsnotify/.editorconfig +++ /dev/null @@ -1,12 +0,0 @@ -root = true - -[*.go] -indent_style = tab -indent_size = 4 -insert_final_newline = true - -[*.{yml,yaml}] -indent_style = space -indent_size = 2 -insert_final_newline = true -trim_trailing_whitespace = true diff --git a/vendor/github.com/fsnotify/fsnotify/.gitattributes b/vendor/github.com/fsnotify/fsnotify/.gitattributes deleted file mode 100644 index 32f1001be..000000000 --- a/vendor/github.com/fsnotify/fsnotify/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -go.sum linguist-generated diff --git a/vendor/github.com/fsnotify/fsnotify/.gitignore b/vendor/github.com/fsnotify/fsnotify/.gitignore index 4cd0cbaf4..daea9dd6d 100644 --- a/vendor/github.com/fsnotify/fsnotify/.gitignore +++ b/vendor/github.com/fsnotify/fsnotify/.gitignore @@ -1,6 +1,10 @@ -# Setup a Global .gitignore for OS and editor generated files: -# https://help.github.com/articles/ignoring-files -# git config --global core.excludesfile ~/.gitignore_global +# go test -c output +*.test +*.test.exe -.vagrant -*.sublime-project +# Output of go build ./cmd/fsnotify +/fsnotify +/fsnotify.exe + +/test/kqueue +/test/a.out diff --git a/vendor/github.com/fsnotify/fsnotify/AUTHORS b/vendor/github.com/fsnotify/fsnotify/AUTHORS deleted file mode 100644 index 6cbabe5ef..000000000 --- a/vendor/github.com/fsnotify/fsnotify/AUTHORS +++ /dev/null @@ -1,62 +0,0 @@ -# Names should be added to this file as -# Name or Organization -# The email address is not required for organizations. - -# You can update this list using the following command: -# -# $ (head -n10 AUTHORS && git shortlog -se | sed -E 's/^\s+[0-9]+\t//') | tee AUTHORS - -# Please keep the list sorted. - -Aaron L -Adrien Bustany -Alexey Kazakov -Amit Krishnan -Anmol Sethi -Bjørn Erik Pedersen -Brian Goff -Bruno Bigras -Caleb Spare -Case Nelson -Chris Howey -Christoffer Buchholz -Daniel Wagner-Hall -Dave Cheney -Eric Lin -Evan Phoenix -Francisco Souza -Gautam Dey -Hari haran -Ichinose Shogo -Johannes Ebke -John C Barstow -Kelvin Fo -Ken-ichirou MATSUZAWA -Matt Layher -Matthias Stone -Nathan Youngman -Nickolai Zeldovich -Oliver Bristow -Patrick -Paul Hammond -Pawel Knap -Pieter Droogendijk -Pratik Shinde -Pursuit92 -Riku Voipio -Rob Figueiredo -Rodrigo Chiossi -Slawek Ligus -Soge Zhang -Tiffany Jernigan -Tilak Sharma -Tobias Klauser -Tom Payne -Travis Cline -Tudor Golubenco -Vahe Khachikyan -Yukang -bronze1man -debrando -henrikedwards -铁哥 diff --git a/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md b/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md index cc01c08f5..fa854785d 100644 --- a/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md +++ b/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md @@ -1,11 +1,199 @@ # Changelog -All notable changes to this project will be documented in this file. +1.8.0 2023-10-31 +---------------- -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +### Additions -## [Unreleased] +- all: add `FSNOTIFY_DEBUG` to print debug logs to stderr ([#619]) + +### Changes and fixes + +- windows: fix behaviour of `WatchList()` to be consistent with other platforms ([#610]) + +- kqueue: ignore events with Ident=0 ([#590]) + +- kqueue: set O_CLOEXEC to prevent passing file descriptors to children ([#617]) + +- kqueue: emit events as "/path/dir/file" instead of "path/link/file" when watching a symlink ([#625]) + +- inotify: don't send event for IN_DELETE_SELF when also watching the parent ([#620]) + +- inotify: fix panic when calling Remove() in a goroutine ([#650]) + +- fen: allow watching subdirectories of watched directories ([#621]) + +[#590]: https://github.com/fsnotify/fsnotify/pull/590 +[#610]: https://github.com/fsnotify/fsnotify/pull/610 +[#617]: https://github.com/fsnotify/fsnotify/pull/617 +[#619]: https://github.com/fsnotify/fsnotify/pull/619 +[#620]: https://github.com/fsnotify/fsnotify/pull/620 +[#621]: https://github.com/fsnotify/fsnotify/pull/621 +[#625]: https://github.com/fsnotify/fsnotify/pull/625 +[#650]: https://github.com/fsnotify/fsnotify/pull/650 + +1.7.0 - 2023-10-22 +------------------ +This version of fsnotify needs Go 1.17. + +### Additions + +- illumos: add FEN backend to support illumos and Solaris. ([#371]) + +- all: add `NewBufferedWatcher()` to use a buffered channel, which can be useful + in cases where you can't control the kernel buffer and receive a large number + of events in bursts. ([#550], [#572]) + +- all: add `AddWith()`, which is identical to `Add()` but allows passing + options. ([#521]) + +- windows: allow setting the ReadDirectoryChangesW() buffer size with + `fsnotify.WithBufferSize()`; the default of 64K is the highest value that + works on all platforms and is enough for most purposes, but in some cases a + highest buffer is needed. ([#521]) + +### Changes and fixes + +- inotify: remove watcher if a watched path is renamed ([#518]) + + After a rename the reported name wasn't updated, or even an empty string. + Inotify doesn't provide any good facilities to update it, so just remove the + watcher. This is already how it worked on kqueue and FEN. + + On Windows this does work, and remains working. + +- windows: don't listen for file attribute changes ([#520]) + + File attribute changes are sent as `FILE_ACTION_MODIFIED` by the Windows API, + with no way to see if they're a file write or attribute change, so would show + up as a fsnotify.Write event. This is never useful, and could result in many + spurious Write events. + +- windows: return `ErrEventOverflow` if the buffer is full ([#525]) + + Before it would merely return "short read", making it hard to detect this + error. + +- kqueue: make sure events for all files are delivered properly when removing a + watched directory ([#526]) + + Previously they would get sent with `""` (empty string) or `"."` as the path + name. + +- kqueue: don't emit spurious Create events for symbolic links ([#524]) + + The link would get resolved but kqueue would "forget" it already saw the link + itself, resulting on a Create for every Write event for the directory. + +- all: return `ErrClosed` on `Add()` when the watcher is closed ([#516]) + +- other: add `Watcher.Errors` and `Watcher.Events` to the no-op `Watcher` in + `backend_other.go`, making it easier to use on unsupported platforms such as + WASM, AIX, etc. ([#528]) + +- other: use the `backend_other.go` no-op if the `appengine` build tag is set; + Google AppEngine forbids usage of the unsafe package so the inotify backend + won't compile there. + +[#371]: https://github.com/fsnotify/fsnotify/pull/371 +[#516]: https://github.com/fsnotify/fsnotify/pull/516 +[#518]: https://github.com/fsnotify/fsnotify/pull/518 +[#520]: https://github.com/fsnotify/fsnotify/pull/520 +[#521]: https://github.com/fsnotify/fsnotify/pull/521 +[#524]: https://github.com/fsnotify/fsnotify/pull/524 +[#525]: https://github.com/fsnotify/fsnotify/pull/525 +[#526]: https://github.com/fsnotify/fsnotify/pull/526 +[#528]: https://github.com/fsnotify/fsnotify/pull/528 +[#537]: https://github.com/fsnotify/fsnotify/pull/537 +[#550]: https://github.com/fsnotify/fsnotify/pull/550 +[#572]: https://github.com/fsnotify/fsnotify/pull/572 + +1.6.0 - 2022-10-13 +------------------ +This version of fsnotify needs Go 1.16 (this was already the case since 1.5.1, +but not documented). It also increases the minimum Linux version to 2.6.32. + +### Additions + +- all: add `Event.Has()` and `Op.Has()` ([#477]) + + This makes checking events a lot easier; for example: + + if event.Op&Write == Write && !(event.Op&Remove == Remove) { + } + + Becomes: + + if event.Has(Write) && !event.Has(Remove) { + } + +- all: add cmd/fsnotify ([#463]) + + A command-line utility for testing and some examples. + +### Changes and fixes + +- inotify: don't ignore events for files that don't exist ([#260], [#470]) + + Previously the inotify watcher would call `os.Lstat()` to check if a file + still exists before emitting events. + + This was inconsistent with other platforms and resulted in inconsistent event + reporting (e.g. when a file is quickly removed and re-created), and generally + a source of confusion. It was added in 2013 to fix a memory leak that no + longer exists. + +- all: return `ErrNonExistentWatch` when `Remove()` is called on a path that's + not watched ([#460]) + +- inotify: replace epoll() with non-blocking inotify ([#434]) + + Non-blocking inotify was not generally available at the time this library was + written in 2014, but now it is. As a result, the minimum Linux version is + bumped from 2.6.27 to 2.6.32. This hugely simplifies the code and is faster. + +- kqueue: don't check for events every 100ms ([#480]) + + The watcher would wake up every 100ms, even when there was nothing to do. Now + it waits until there is something to do. + +- macos: retry opening files on EINTR ([#475]) + +- kqueue: skip unreadable files ([#479]) + + kqueue requires a file descriptor for every file in a directory; this would + fail if a file was unreadable by the current user. Now these files are simply + skipped. + +- windows: fix renaming a watched directory if the parent is also watched ([#370]) + +- windows: increase buffer size from 4K to 64K ([#485]) + +- windows: close file handle on Remove() ([#288]) + +- kqueue: put pathname in the error if watching a file fails ([#471]) + +- inotify, windows: calling Close() more than once could race ([#465]) + +- kqueue: improve Close() performance ([#233]) + +- all: various documentation additions and clarifications. + +[#233]: https://github.com/fsnotify/fsnotify/pull/233 +[#260]: https://github.com/fsnotify/fsnotify/pull/260 +[#288]: https://github.com/fsnotify/fsnotify/pull/288 +[#370]: https://github.com/fsnotify/fsnotify/pull/370 +[#434]: https://github.com/fsnotify/fsnotify/pull/434 +[#460]: https://github.com/fsnotify/fsnotify/pull/460 +[#463]: https://github.com/fsnotify/fsnotify/pull/463 +[#465]: https://github.com/fsnotify/fsnotify/pull/465 +[#470]: https://github.com/fsnotify/fsnotify/pull/470 +[#471]: https://github.com/fsnotify/fsnotify/pull/471 +[#475]: https://github.com/fsnotify/fsnotify/pull/475 +[#477]: https://github.com/fsnotify/fsnotify/pull/477 +[#479]: https://github.com/fsnotify/fsnotify/pull/479 +[#480]: https://github.com/fsnotify/fsnotify/pull/480 +[#485]: https://github.com/fsnotify/fsnotify/pull/485 ## [1.5.4] - 2022-04-25 @@ -40,6 +228,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [#385](https://github.com/fsnotify/fsnotify/pull/385) * Go 1.14+: Fix unsafe pointer conversion [#325](https://github.com/fsnotify/fsnotify/pull/325) +## [1.4.9] - 2020-03-11 + +* Move example usage to the readme #329. This may resolve #328. + +## [1.4.8] - 2020-03-10 + +* CI: test more go versions (@nathany 1d13583d846ea9d66dcabbfefbfb9d8e6fb05216) +* Tests: Queued inotify events could have been read by the test before max_queued_events was hit (@matthias-stone #265) +* Tests: t.Fatalf -> t.Errorf in go routines (@gdey #266) +* CI: Less verbosity (@nathany #267) +* Tests: Darwin: Exchangedata is deprecated on 10.13 (@nathany #267) +* Tests: Check if channels are closed in the example (@alexeykazakov #244) +* CI: Only run golint on latest version of go and fix issues (@cpuguy83 #284) +* CI: Add windows to travis matrix (@cpuguy83 #284) +* Docs: Remover appveyor badge (@nathany 11844c0959f6fff69ba325d097fce35bd85a8e93) +* Linux: create epoll and pipe fds with close-on-exec (@JohannesEbke #219) +* Linux: open files with close-on-exec (@linxiulei #273) +* Docs: Plan to support fanotify (@nathany ab058b44498e8b7566a799372a39d150d9ea0119 ) +* Project: Add go.mod (@nathany #309) +* Project: Revise editor config (@nathany #309) +* Project: Update copyright for 2019 (@nathany #309) +* CI: Drop go1.8 from CI matrix (@nathany #309) +* Docs: Updating the FAQ section for supportability with NFS & FUSE filesystems (@Pratik32 4bf2d1fec78374803a39307bfb8d340688f4f28e ) + ## [1.4.7] - 2018-01-09 * BSD/macOS: Fix possible deadlock on closing the watcher on kqueue (thanks @nhooyr and @glycerine) diff --git a/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md b/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md index 8a642563d..e4ac2a2ff 100644 --- a/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md +++ b/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md @@ -1,60 +1,144 @@ -# Contributing +Thank you for your interest in contributing to fsnotify! We try to review and +merge PRs in a reasonable timeframe, but please be aware that: -## Issues +- To avoid "wasted" work, please discuss changes on the issue tracker first. You + can just send PRs, but they may end up being rejected for one reason or the + other. -* Request features and report bugs using the [GitHub Issue Tracker](https://github.com/fsnotify/fsnotify/issues). -* Please indicate the platform you are using fsnotify on. -* A code example to reproduce the problem is appreciated. +- fsnotify is a cross-platform library, and changes must work reasonably well on + all supported platforms. -## Pull Requests +- Changes will need to be compatible; old code should still compile, and the + runtime behaviour can't change in ways that are likely to lead to problems for + users. -### Contributor License Agreement +Testing +------- +Just `go test ./...` runs all the tests; the CI runs this on all supported +platforms. Testing different platforms locally can be done with something like +[goon] or [Vagrant], but this isn't super-easy to set up at the moment. -fsnotify is derived from code in the [golang.org/x/exp](https://godoc.org/golang.org/x/exp) package and it may be included [in the standard library](https://github.com/fsnotify/fsnotify/issues/1) in the future. Therefore fsnotify carries the same [LICENSE](https://github.com/fsnotify/fsnotify/blob/master/LICENSE) as Go. Contributors retain their copyright, so you need to fill out a short form before we can accept your contribution: [Google Individual Contributor License Agreement](https://developers.google.com/open-source/cla/individual). +Use the `-short` flag to make the "stress test" run faster. -Please indicate that you have signed the CLA in your pull request. +Writing new tests +----------------- +Scripts in the testdata directory allow creating test cases in a "shell-like" +syntax. The basic format is: -### How fsnotify is Developed + script -* Development is done on feature branches. -* Tests are run on BSD, Linux, macOS and Windows. -* Pull requests are reviewed and [applied to master][am] using [hub][]. - * Maintainers may modify or squash commits rather than asking contributors to. -* To issue a new release, the maintainers will: - * Update the CHANGELOG - * Tag a version, which will become available through gopkg.in. - -### How to Fork + Output: + desired output -For smooth sailing, always use the original import path. Installing with `go get` makes this easy. +For example: -1. Install from GitHub (`go get -u github.com/fsnotify/fsnotify`) -2. Create your feature branch (`git checkout -b my-new-feature`) -3. Ensure everything works and the tests pass (see below) -4. Commit your changes (`git commit -am 'Add some feature'`) + # Create a new empty file with some data. + watch / + echo data >/file -Contribute upstream: + Output: + create /file + write /file -1. Fork fsnotify on GitHub -2. Add your remote (`git remote add fork git@github.com:mycompany/repo.git`) -3. Push to the branch (`git push fork my-new-feature`) -4. Create a new Pull Request on GitHub +Just create a new file to add a new test; select which tests to run with +`-run TestScript/[path]`. -This workflow is [thoroughly explained by Katrina Owen](https://splice.com/blog/contributing-open-source-git-repositories-go/). +script +------ +The script is a "shell-like" script: -### Testing + cmd arg arg -fsnotify uses build tags to compile different code on Linux, BSD, macOS, and Windows. +Comments are supported with `#`: -Before doing a pull request, please do your best to test your changes on multiple platforms, and list which platforms you were able/unable to test on. + # Comment + cmd arg arg # Comment -### Maintainers +All operations are done in a temp directory; a path like "/foo" is rewritten to +"/tmp/TestFoo/foo". -Help maintaining fsnotify is welcome. To be a maintainer: +Arguments can be quoted with `"` or `'`; there are no escapes and they're +functionally identical right now, but this may change in the future, so best to +assume shell-like rules. -* Submit a pull request and sign the CLA as above. -* You must be able to run the test suite on Mac, Windows, Linux and BSD. + touch "/file with spaces" -All code changes should be internal pull requests. +End-of-line escapes with `\` are not supported. -Releases are tagged using [Semantic Versioning](http://semver.org/). +### Supported commands + + watch path [ops] # Watch the path, reporting events for it. Nothing is + # watched by default. Optionally a list of ops can be + # given, as with AddWith(path, WithOps(...)). + unwatch path # Stop watching the path. + watchlist n # Assert watchlist length. + + stop # Stop running the script; for debugging. + debug [yes/no] # Enable/disable FSNOTIFY_DEBUG (tests are run in + parallel by default, so -parallel=1 is probably a good + idea). + + touch path + mkdir [-p] dir + ln -s target link # Only ln -s supported. + mkfifo path + mknod dev path + mv src dst + rm [-r] path + chmod mode path # Octal only + sleep time-in-ms + + cat path # Read path (does nothing with the data; just reads it). + echo str >>path # Append "str" to "path". + echo str >path # Truncate "path" and write "str". + + require reason # Skip the test if "reason" is true; "skip" and + skip reason # "require" behave identical; it supports both for + # readability. Possible reasons are: + # + # always Always skip this test. + # symlink Symlinks are supported (requires admin + # permissions on Windows). + # mkfifo Platform doesn't support FIFO named sockets. + # mknod Platform doesn't support device nodes. + + +output +------ +After `Output:` the desired output is given; this is indented by convention, but +that's not required. + +The format of that is: + + # Comment + event path # Comment + + system: + event path + system2: + event path + +Every event is one line, and any whitespace between the event and path are +ignored. The path can optionally be surrounded in ". Anything after a "#" is +ignored. + +Platform-specific tests can be added after GOOS; for example: + + watch / + touch /file + + Output: + # Tested if nothing else matches + create /file + + # Windows-specific test. + windows: + write /file + +You can specify multiple platforms with a comma (e.g. "windows, linux:"). +"kqueue" is a shortcut for all kqueue systems (BSD, macOS). + + +[goon]: https://github.com/arp242/goon +[Vagrant]: https://www.vagrantup.com/ +[integration_test.go]: /integration_test.go diff --git a/vendor/github.com/fsnotify/fsnotify/LICENSE b/vendor/github.com/fsnotify/fsnotify/LICENSE index e180c8fb0..fb03ade75 100644 --- a/vendor/github.com/fsnotify/fsnotify/LICENSE +++ b/vendor/github.com/fsnotify/fsnotify/LICENSE @@ -1,28 +1,25 @@ -Copyright (c) 2012 The Go Authors. All rights reserved. -Copyright (c) 2012-2019 fsnotify Authors. All rights reserved. +Copyright © 2012 The Go Authors. All rights reserved. +Copyright © fsnotify Authors. All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. +* Neither the name of Google Inc. nor the names of its contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/fsnotify/fsnotify/README.md b/vendor/github.com/fsnotify/fsnotify/README.md index 0731c5ef8..e480733d1 100644 --- a/vendor/github.com/fsnotify/fsnotify/README.md +++ b/vendor/github.com/fsnotify/fsnotify/README.md @@ -1,120 +1,184 @@ -# File system notifications for Go +fsnotify is a Go library to provide cross-platform filesystem notifications on +Windows, Linux, macOS, BSD, and illumos. -[![Go Reference](https://pkg.go.dev/badge/github.com/fsnotify/fsnotify.svg)](https://pkg.go.dev/github.com/fsnotify/fsnotify) [![Go Report Card](https://goreportcard.com/badge/github.com/fsnotify/fsnotify)](https://goreportcard.com/report/github.com/fsnotify/fsnotify) [![Maintainers Wanted](https://img.shields.io/badge/maintainers-wanted-red.svg)](https://github.com/fsnotify/fsnotify/issues/413) +Go 1.17 or newer is required; the full documentation is at +https://pkg.go.dev/github.com/fsnotify/fsnotify -fsnotify utilizes [`golang.org/x/sys`](https://pkg.go.dev/golang.org/x/sys) rather than [`syscall`](https://pkg.go.dev/syscall) from the standard library. +--- -Cross platform: Windows, Linux, BSD and macOS. +Platform support: -| Adapter | OS | Status | -| --------------------- | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | -| inotify | Linux 2.6.27 or later, Android\* | Supported | -| kqueue | BSD, macOS, iOS\* | Supported | -| ReadDirectoryChangesW | Windows | Supported | -| FSEvents | macOS | [Planned](https://github.com/fsnotify/fsnotify/issues/11) | -| FEN | Solaris 11 | [In Progress](https://github.com/fsnotify/fsnotify/pull/371) | -| fanotify | Linux 2.6.37+ | [Maybe](https://github.com/fsnotify/fsnotify/issues/114) | -| USN Journals | Windows | [Maybe](https://github.com/fsnotify/fsnotify/issues/53) | -| Polling | *All* | [Maybe](https://github.com/fsnotify/fsnotify/issues/9) | +| Backend | OS | Status | +| :-------------------- | :--------- | :------------------------------------------------------------------------ | +| inotify | Linux | Supported | +| kqueue | BSD, macOS | Supported | +| ReadDirectoryChangesW | Windows | Supported | +| FEN | illumos | Supported | +| fanotify | Linux 5.9+ | [Not yet](https://github.com/fsnotify/fsnotify/issues/114) | +| AHAFS | AIX | [aix branch]; experimental due to lack of maintainer and test environment | +| FSEvents | macOS | [Needs support in x/sys/unix][fsevents] | +| USN Journals | Windows | [Needs support in x/sys/windows][usn] | +| Polling | *All* | [Not yet](https://github.com/fsnotify/fsnotify/issues/9) | -\* Android and iOS are untested. +Linux and illumos should include Android and Solaris, but these are currently +untested. -Please see [the documentation](https://pkg.go.dev/github.com/fsnotify/fsnotify) and consult the [FAQ](#faq) for usage information. +[fsevents]: https://github.com/fsnotify/fsnotify/issues/11#issuecomment-1279133120 +[usn]: https://github.com/fsnotify/fsnotify/issues/53#issuecomment-1279829847 +[aix branch]: https://github.com/fsnotify/fsnotify/issues/353#issuecomment-1284590129 -## API stability - -fsnotify is a fork of [howeyc/fsnotify](https://github.com/howeyc/fsnotify) with a new API as of v1.0. The API is based on [this design document](http://goo.gl/MrYxyA). - -All [releases](https://github.com/fsnotify/fsnotify/releases) are tagged based on [Semantic Versioning](http://semver.org/). - -## Usage +Usage +----- +A basic example: ```go package main import ( - "log" + "log" - "github.com/fsnotify/fsnotify" + "github.com/fsnotify/fsnotify" ) func main() { - watcher, err := fsnotify.NewWatcher() - if err != nil { - log.Fatal(err) - } - defer watcher.Close() - - done := make(chan bool) - go func() { - for { - select { - case event, ok := <-watcher.Events: - if !ok { - return - } - log.Println("event:", event) - if event.Op&fsnotify.Write == fsnotify.Write { - log.Println("modified file:", event.Name) - } - case err, ok := <-watcher.Errors: - if !ok { - return - } - log.Println("error:", err) - } - } - }() - - err = watcher.Add("/tmp/foo") - if err != nil { - log.Fatal(err) - } - <-done + // Create new watcher. + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal(err) + } + defer watcher.Close() + + // Start listening for events. + go func() { + for { + select { + case event, ok := <-watcher.Events: + if !ok { + return + } + log.Println("event:", event) + if event.Has(fsnotify.Write) { + log.Println("modified file:", event.Name) + } + case err, ok := <-watcher.Errors: + if !ok { + return + } + log.Println("error:", err) + } + } + }() + + // Add a path. + err = watcher.Add("/tmp") + if err != nil { + log.Fatal(err) + } + + // Block main goroutine forever. + <-make(chan struct{}) } ``` -## Contributing +Some more examples can be found in [cmd/fsnotify](cmd/fsnotify), which can be +run with: + + % go run ./cmd/fsnotify -Please refer to [CONTRIBUTING][] before opening an issue or pull request. +Further detailed documentation can be found in godoc: +https://pkg.go.dev/github.com/fsnotify/fsnotify -## FAQ +FAQ +--- +### Will a file still be watched when it's moved to another directory? +No, not unless you are watching the location it was moved to. -**When a file is moved to another directory is it still being watched?** +### Are subdirectories watched? +No, you must add watches for any directory you want to watch (a recursive +watcher is on the roadmap: [#18]). -No (it shouldn't be, unless you are watching where it was moved to). +[#18]: https://github.com/fsnotify/fsnotify/issues/18 -**When I watch a directory, are all subdirectories watched as well?** +### Do I have to watch the Error and Event channels in a goroutine? +Yes. You can read both channels in the same goroutine using `select` (you don't +need a separate goroutine for both channels; see the example). -No, you must add watches for any directory you want to watch (a recursive watcher is on the roadmap [#18][]). +### Why don't notifications work with NFS, SMB, FUSE, /proc, or /sys? +fsnotify requires support from underlying OS to work. The current NFS and SMB +protocols does not provide network level support for file notifications, and +neither do the /proc and /sys virtual filesystems. -**Do I have to watch the Error and Event channels in a separate goroutine?** +This could be fixed with a polling watcher ([#9]), but it's not yet implemented. -As of now, yes. Looking into making this single-thread friendly (see [howeyc #7][#7]) +[#9]: https://github.com/fsnotify/fsnotify/issues/9 -**Why am I receiving multiple events for the same file on OS X?** +### Why do I get many Chmod events? +Some programs may generate a lot of attribute changes; for example Spotlight on +macOS, anti-virus programs, backup applications, and some others are known to do +this. As a rule, it's typically best to ignore Chmod events. They're often not +useful, and tend to cause problems. -Spotlight indexing on OS X can result in multiple events (see [howeyc #62][#62]). A temporary workaround is to add your folder(s) to the *Spotlight Privacy settings* until we have a native FSEvents implementation (see [#11][]). +Spotlight indexing on macOS can result in multiple events (see [#15]). A +temporary workaround is to add your folder(s) to the *Spotlight Privacy +settings* until we have a native FSEvents implementation (see [#11]). -**How many files can be watched at once?** +[#11]: https://github.com/fsnotify/fsnotify/issues/11 +[#15]: https://github.com/fsnotify/fsnotify/issues/15 -There are OS-specific limits as to how many watches can be created: -* Linux: /proc/sys/fs/inotify/max_user_watches contains the limit, reaching this limit results in a "no space left on device" error. -* BSD / OSX: sysctl variables "kern.maxfiles" and "kern.maxfilesperproc", reaching these limits results in a "too many open files" error. +### Watching a file doesn't work well +Watching individual files (rather than directories) is generally not recommended +as many programs (especially editors) update files atomically: it will write to +a temporary file which is then moved to to destination, overwriting the original +(or some variant thereof). The watcher on the original file is now lost, as that +no longer exists. -**Why don't notifications work with NFS filesystems or filesystem in userspace (FUSE)?** +The upshot of this is that a power failure or crash won't leave a half-written +file. -fsnotify requires support from underlying OS to work. The current NFS protocol does not provide network level support for file notifications. +Watch the parent directory and use `Event.Name` to filter out files you're not +interested in. There is an example of this in `cmd/fsnotify/file.go`. -[#62]: https://github.com/howeyc/fsnotify/issues/62 -[#18]: https://github.com/fsnotify/fsnotify/issues/18 -[#11]: https://github.com/fsnotify/fsnotify/issues/11 -[#7]: https://github.com/howeyc/fsnotify/issues/7 +Platform-specific notes +----------------------- +### Linux +When a file is removed a REMOVE event won't be emitted until all file +descriptors are closed; it will emit a CHMOD instead: + + fp := os.Open("file") + os.Remove("file") // CHMOD + fp.Close() // REMOVE + +This is the event that inotify sends, so not much can be changed about this. + +The `fs.inotify.max_user_watches` sysctl variable specifies the upper limit for +the number of watches per user, and `fs.inotify.max_user_instances` specifies +the maximum number of inotify instances per user. Every Watcher you create is an +"instance", and every path you add is a "watch". + +These are also exposed in `/proc` as `/proc/sys/fs/inotify/max_user_watches` and +`/proc/sys/fs/inotify/max_user_instances` + +To increase them you can use `sysctl` or write the value to proc file: + + # The default values on Linux 5.18 + sysctl fs.inotify.max_user_watches=124983 + sysctl fs.inotify.max_user_instances=128 + +To make the changes persist on reboot edit `/etc/sysctl.conf` or +`/usr/lib/sysctl.d/50-default.conf` (details differ per Linux distro; check your +distro's documentation): -[contributing]: https://github.com/fsnotify/fsnotify/blob/master/CONTRIBUTING.md + fs.inotify.max_user_watches=124983 + fs.inotify.max_user_instances=128 -## Related Projects +Reaching the limit will result in a "no space left on device" or "too many open +files" error. -* [notify](https://github.com/rjeczalik/notify) -* [fsevents](https://github.com/fsnotify/fsevents) +### kqueue (macOS, all BSD systems) +kqueue requires opening a file descriptor for every file that's being watched; +so if you're watching a directory with five files then that's six file +descriptors. You will run in to your system's "max open files" limit faster on +these platforms. +The sysctl variables `kern.maxfiles` and `kern.maxfilesperproc` can be used to +control the maximum number of open files. diff --git a/vendor/github.com/fsnotify/fsnotify/backend_fen.go b/vendor/github.com/fsnotify/fsnotify/backend_fen.go new file mode 100644 index 000000000..c349c326c --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/backend_fen.go @@ -0,0 +1,484 @@ +//go:build solaris + +// FEN backend for illumos (supported) and Solaris (untested, but should work). +// +// See port_create(3c) etc. for docs. https://www.illumos.org/man/3C/port_create + +package fsnotify + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/fsnotify/fsnotify/internal" + "golang.org/x/sys/unix" +) + +type fen struct { + Events chan Event + Errors chan error + + mu sync.Mutex + port *unix.EventPort + done chan struct{} // Channel for sending a "quit message" to the reader goroutine + dirs map[string]Op // Explicitly watched directories + watches map[string]Op // Explicitly watched non-directories +} + +func newBackend(ev chan Event, errs chan error) (backend, error) { + return newBufferedBackend(0, ev, errs) +} + +func newBufferedBackend(sz uint, ev chan Event, errs chan error) (backend, error) { + w := &fen{ + Events: ev, + Errors: errs, + dirs: make(map[string]Op), + watches: make(map[string]Op), + done: make(chan struct{}), + } + + var err error + w.port, err = unix.NewEventPort() + if err != nil { + return nil, fmt.Errorf("fsnotify.NewWatcher: %w", err) + } + + go w.readEvents() + return w, nil +} + +// sendEvent attempts to send an event to the user, returning true if the event +// was put in the channel successfully and false if the watcher has been closed. +func (w *fen) sendEvent(name string, op Op) (sent bool) { + select { + case <-w.done: + return false + case w.Events <- Event{Name: name, Op: op}: + return true + } +} + +// sendError attempts to send an error to the user, returning true if the error +// was put in the channel successfully and false if the watcher has been closed. +func (w *fen) sendError(err error) (sent bool) { + if err == nil { + return true + } + select { + case <-w.done: + return false + case w.Errors <- err: + return true + } +} + +func (w *fen) isClosed() bool { + select { + case <-w.done: + return true + default: + return false + } +} + +func (w *fen) Close() error { + // Take the lock used by associateFile to prevent lingering events from + // being processed after the close + w.mu.Lock() + defer w.mu.Unlock() + if w.isClosed() { + return nil + } + close(w.done) + return w.port.Close() +} + +func (w *fen) Add(name string) error { return w.AddWith(name) } + +func (w *fen) AddWith(name string, opts ...addOpt) error { + if w.isClosed() { + return ErrClosed + } + if debug { + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s AddWith(%q)\n", + time.Now().Format("15:04:05.000000000"), name) + } + + with := getOptions(opts...) + if !w.xSupports(with.op) { + return fmt.Errorf("%w: %s", xErrUnsupported, with.op) + } + + // Currently we resolve symlinks that were explicitly requested to be + // watched. Otherwise we would use LStat here. + stat, err := os.Stat(name) + if err != nil { + return err + } + + // Associate all files in the directory. + if stat.IsDir() { + err := w.handleDirectory(name, stat, true, w.associateFile) + if err != nil { + return err + } + + w.mu.Lock() + w.dirs[name] = with.op + w.mu.Unlock() + return nil + } + + err = w.associateFile(name, stat, true) + if err != nil { + return err + } + + w.mu.Lock() + w.watches[name] = with.op + w.mu.Unlock() + return nil +} + +func (w *fen) Remove(name string) error { + if w.isClosed() { + return nil + } + if !w.port.PathIsWatched(name) { + return fmt.Errorf("%w: %s", ErrNonExistentWatch, name) + } + if debug { + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s Remove(%q)\n", + time.Now().Format("15:04:05.000000000"), name) + } + + // The user has expressed an intent. Immediately remove this name from + // whichever watch list it might be in. If it's not in there the delete + // doesn't cause harm. + w.mu.Lock() + delete(w.watches, name) + delete(w.dirs, name) + w.mu.Unlock() + + stat, err := os.Stat(name) + if err != nil { + return err + } + + // Remove associations for every file in the directory. + if stat.IsDir() { + err := w.handleDirectory(name, stat, false, w.dissociateFile) + if err != nil { + return err + } + return nil + } + + err = w.port.DissociatePath(name) + if err != nil { + return err + } + + return nil +} + +// readEvents contains the main loop that runs in a goroutine watching for events. +func (w *fen) readEvents() { + // If this function returns, the watcher has been closed and we can close + // these channels + defer func() { + close(w.Errors) + close(w.Events) + }() + + pevents := make([]unix.PortEvent, 8) + for { + count, err := w.port.Get(pevents, 1, nil) + if err != nil && err != unix.ETIME { + // Interrupted system call (count should be 0) ignore and continue + if errors.Is(err, unix.EINTR) && count == 0 { + continue + } + // Get failed because we called w.Close() + if errors.Is(err, unix.EBADF) && w.isClosed() { + return + } + // There was an error not caused by calling w.Close() + if !w.sendError(err) { + return + } + } + + p := pevents[:count] + for _, pevent := range p { + if pevent.Source != unix.PORT_SOURCE_FILE { + // Event from unexpected source received; should never happen. + if !w.sendError(errors.New("Event from unexpected source received")) { + return + } + continue + } + + if debug { + internal.Debug(pevent.Path, pevent.Events) + } + + err = w.handleEvent(&pevent) + if !w.sendError(err) { + return + } + } + } +} + +func (w *fen) handleDirectory(path string, stat os.FileInfo, follow bool, handler func(string, os.FileInfo, bool) error) error { + files, err := os.ReadDir(path) + if err != nil { + return err + } + + // Handle all children of the directory. + for _, entry := range files { + finfo, err := entry.Info() + if err != nil { + return err + } + err = handler(filepath.Join(path, finfo.Name()), finfo, false) + if err != nil { + return err + } + } + + // And finally handle the directory itself. + return handler(path, stat, follow) +} + +// handleEvent might need to emit more than one fsnotify event if the events +// bitmap matches more than one event type (e.g. the file was both modified and +// had the attributes changed between when the association was created and the +// when event was returned) +func (w *fen) handleEvent(event *unix.PortEvent) error { + var ( + events = event.Events + path = event.Path + fmode = event.Cookie.(os.FileMode) + reRegister = true + ) + + w.mu.Lock() + _, watchedDir := w.dirs[path] + _, watchedPath := w.watches[path] + w.mu.Unlock() + isWatched := watchedDir || watchedPath + + if events&unix.FILE_DELETE != 0 { + if !w.sendEvent(path, Remove) { + return nil + } + reRegister = false + } + if events&unix.FILE_RENAME_FROM != 0 { + if !w.sendEvent(path, Rename) { + return nil + } + // Don't keep watching the new file name + reRegister = false + } + if events&unix.FILE_RENAME_TO != 0 { + // We don't report a Rename event for this case, because Rename events + // are interpreted as referring to the _old_ name of the file, and in + // this case the event would refer to the new name of the file. This + // type of rename event is not supported by fsnotify. + + // inotify reports a Remove event in this case, so we simulate this + // here. + if !w.sendEvent(path, Remove) { + return nil + } + // Don't keep watching the file that was removed + reRegister = false + } + + // The file is gone, nothing left to do. + if !reRegister { + if watchedDir { + w.mu.Lock() + delete(w.dirs, path) + w.mu.Unlock() + } + if watchedPath { + w.mu.Lock() + delete(w.watches, path) + w.mu.Unlock() + } + return nil + } + + // If we didn't get a deletion the file still exists and we're going to have + // to watch it again. Let's Stat it now so that we can compare permissions + // and have what we need to continue watching the file + + stat, err := os.Lstat(path) + if err != nil { + // This is unexpected, but we should still emit an event. This happens + // most often on "rm -r" of a subdirectory inside a watched directory We + // get a modify event of something happening inside, but by the time we + // get here, the sudirectory is already gone. Clearly we were watching + // this path but now it is gone. Let's tell the user that it was + // removed. + if !w.sendEvent(path, Remove) { + return nil + } + // Suppress extra write events on removed directories; they are not + // informative and can be confusing. + return nil + } + + // resolve symlinks that were explicitly watched as we would have at Add() + // time. this helps suppress spurious Chmod events on watched symlinks + if isWatched { + stat, err = os.Stat(path) + if err != nil { + // The symlink still exists, but the target is gone. Report the + // Remove similar to above. + if !w.sendEvent(path, Remove) { + return nil + } + // Don't return the error + } + } + + if events&unix.FILE_MODIFIED != 0 { + if fmode.IsDir() && watchedDir { + if err := w.updateDirectory(path); err != nil { + return err + } + } else { + if !w.sendEvent(path, Write) { + return nil + } + } + } + if events&unix.FILE_ATTRIB != 0 && stat != nil { + // Only send Chmod if perms changed + if stat.Mode().Perm() != fmode.Perm() { + if !w.sendEvent(path, Chmod) { + return nil + } + } + } + + if stat != nil { + // If we get here, it means we've hit an event above that requires us to + // continue watching the file or directory + return w.associateFile(path, stat, isWatched) + } + return nil +} + +func (w *fen) updateDirectory(path string) error { + // The directory was modified, so we must find unwatched entities and watch + // them. If something was removed from the directory, nothing will happen, + // as everything else should still be watched. + files, err := os.ReadDir(path) + if err != nil { + return err + } + + for _, entry := range files { + path := filepath.Join(path, entry.Name()) + if w.port.PathIsWatched(path) { + continue + } + + finfo, err := entry.Info() + if err != nil { + return err + } + err = w.associateFile(path, finfo, false) + if !w.sendError(err) { + return nil + } + if !w.sendEvent(path, Create) { + return nil + } + } + return nil +} + +func (w *fen) associateFile(path string, stat os.FileInfo, follow bool) error { + if w.isClosed() { + return ErrClosed + } + // This is primarily protecting the call to AssociatePath but it is + // important and intentional that the call to PathIsWatched is also + // protected by this mutex. Without this mutex, AssociatePath has been seen + // to error out that the path is already associated. + w.mu.Lock() + defer w.mu.Unlock() + + if w.port.PathIsWatched(path) { + // Remove the old association in favor of this one If we get ENOENT, + // then while the x/sys/unix wrapper still thought that this path was + // associated, the underlying event port did not. This call will have + // cleared up that discrepancy. The most likely cause is that the event + // has fired but we haven't processed it yet. + err := w.port.DissociatePath(path) + if err != nil && !errors.Is(err, unix.ENOENT) { + return err + } + } + + var events int + if !follow { + // Watch symlinks themselves rather than their targets unless this entry + // is explicitly watched. + events |= unix.FILE_NOFOLLOW + } + if true { // TODO: implement withOps() + events |= unix.FILE_MODIFIED + } + if true { + events |= unix.FILE_ATTRIB + } + return w.port.AssociatePath(path, stat, events, stat.Mode()) +} + +func (w *fen) dissociateFile(path string, stat os.FileInfo, unused bool) error { + if !w.port.PathIsWatched(path) { + return nil + } + return w.port.DissociatePath(path) +} + +func (w *fen) WatchList() []string { + if w.isClosed() { + return nil + } + + w.mu.Lock() + defer w.mu.Unlock() + + entries := make([]string, 0, len(w.watches)+len(w.dirs)) + for pathname := range w.dirs { + entries = append(entries, pathname) + } + for pathname := range w.watches { + entries = append(entries, pathname) + } + + return entries +} + +func (w *fen) xSupports(op Op) bool { + if op.Has(xUnportableOpen) || op.Has(xUnportableRead) || + op.Has(xUnportableCloseWrite) || op.Has(xUnportableCloseRead) { + return false + } + return true +} diff --git a/vendor/github.com/fsnotify/fsnotify/backend_inotify.go b/vendor/github.com/fsnotify/fsnotify/backend_inotify.go new file mode 100644 index 000000000..36c311694 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/backend_inotify.go @@ -0,0 +1,658 @@ +//go:build linux && !appengine + +package fsnotify + +import ( + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "sync" + "time" + "unsafe" + + "github.com/fsnotify/fsnotify/internal" + "golang.org/x/sys/unix" +) + +type inotify struct { + Events chan Event + Errors chan error + + // Store fd here as os.File.Read() will no longer return on close after + // calling Fd(). See: https://github.com/golang/go/issues/26439 + fd int + inotifyFile *os.File + watches *watches + done chan struct{} // Channel for sending a "quit message" to the reader goroutine + doneMu sync.Mutex + doneResp chan struct{} // Channel to respond to Close + + // Store rename cookies in an array, with the index wrapping to 0. Almost + // all of the time what we get is a MOVED_FROM to set the cookie and the + // next event inotify sends will be MOVED_TO to read it. However, this is + // not guaranteed – as described in inotify(7) – and we may get other events + // between the two MOVED_* events (including other MOVED_* ones). + // + // A second issue is that moving a file outside the watched directory will + // trigger a MOVED_FROM to set the cookie, but we never see the MOVED_TO to + // read and delete it. So just storing it in a map would slowly leak memory. + // + // Doing it like this gives us a simple fast LRU-cache that won't allocate. + // Ten items should be more than enough for our purpose, and a loop over + // such a short array is faster than a map access anyway (not that it hugely + // matters since we're talking about hundreds of ns at the most, but still). + cookies [10]koekje + cookieIndex uint8 + cookiesMu sync.Mutex +} + +type ( + watches struct { + mu sync.RWMutex + wd map[uint32]*watch // wd → watch + path map[string]uint32 // pathname → wd + } + watch struct { + wd uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall) + flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags) + path string // Watch path. + recurse bool // Recursion with ./...? + } + koekje struct { + cookie uint32 + path string + } +) + +func newWatches() *watches { + return &watches{ + wd: make(map[uint32]*watch), + path: make(map[string]uint32), + } +} + +func (w *watches) len() int { + w.mu.RLock() + defer w.mu.RUnlock() + return len(w.wd) +} + +func (w *watches) add(ww *watch) { + w.mu.Lock() + defer w.mu.Unlock() + w.wd[ww.wd] = ww + w.path[ww.path] = ww.wd +} + +func (w *watches) remove(wd uint32) { + w.mu.Lock() + defer w.mu.Unlock() + watch := w.wd[wd] // Could have had Remove() called. See #616. + if watch == nil { + return + } + delete(w.path, watch.path) + delete(w.wd, wd) +} + +func (w *watches) removePath(path string) ([]uint32, error) { + w.mu.Lock() + defer w.mu.Unlock() + + path, recurse := recursivePath(path) + wd, ok := w.path[path] + if !ok { + return nil, fmt.Errorf("%w: %s", ErrNonExistentWatch, path) + } + + watch := w.wd[wd] + if recurse && !watch.recurse { + return nil, fmt.Errorf("can't use /... with non-recursive watch %q", path) + } + + delete(w.path, path) + delete(w.wd, wd) + if !watch.recurse { + return []uint32{wd}, nil + } + + wds := make([]uint32, 0, 8) + wds = append(wds, wd) + for p, rwd := range w.path { + if filepath.HasPrefix(p, path) { + delete(w.path, p) + delete(w.wd, rwd) + wds = append(wds, rwd) + } + } + return wds, nil +} + +func (w *watches) byPath(path string) *watch { + w.mu.RLock() + defer w.mu.RUnlock() + return w.wd[w.path[path]] +} + +func (w *watches) byWd(wd uint32) *watch { + w.mu.RLock() + defer w.mu.RUnlock() + return w.wd[wd] +} + +func (w *watches) updatePath(path string, f func(*watch) (*watch, error)) error { + w.mu.Lock() + defer w.mu.Unlock() + + var existing *watch + wd, ok := w.path[path] + if ok { + existing = w.wd[wd] + } + + upd, err := f(existing) + if err != nil { + return err + } + if upd != nil { + w.wd[upd.wd] = upd + w.path[upd.path] = upd.wd + + if upd.wd != wd { + delete(w.wd, wd) + } + } + + return nil +} + +func newBackend(ev chan Event, errs chan error) (backend, error) { + return newBufferedBackend(0, ev, errs) +} + +func newBufferedBackend(sz uint, ev chan Event, errs chan error) (backend, error) { + // Need to set nonblocking mode for SetDeadline to work, otherwise blocking + // I/O operations won't terminate on close. + fd, errno := unix.InotifyInit1(unix.IN_CLOEXEC | unix.IN_NONBLOCK) + if fd == -1 { + return nil, errno + } + + w := &inotify{ + Events: ev, + Errors: errs, + fd: fd, + inotifyFile: os.NewFile(uintptr(fd), ""), + watches: newWatches(), + done: make(chan struct{}), + doneResp: make(chan struct{}), + } + + go w.readEvents() + return w, nil +} + +// Returns true if the event was sent, or false if watcher is closed. +func (w *inotify) sendEvent(e Event) bool { + select { + case <-w.done: + return false + case w.Events <- e: + return true + } +} + +// Returns true if the error was sent, or false if watcher is closed. +func (w *inotify) sendError(err error) bool { + if err == nil { + return true + } + select { + case <-w.done: + return false + case w.Errors <- err: + return true + } +} + +func (w *inotify) isClosed() bool { + select { + case <-w.done: + return true + default: + return false + } +} + +func (w *inotify) Close() error { + w.doneMu.Lock() + if w.isClosed() { + w.doneMu.Unlock() + return nil + } + close(w.done) + w.doneMu.Unlock() + + // Causes any blocking reads to return with an error, provided the file + // still supports deadline operations. + err := w.inotifyFile.Close() + if err != nil { + return err + } + + // Wait for goroutine to close + <-w.doneResp + + return nil +} + +func (w *inotify) Add(name string) error { return w.AddWith(name) } + +func (w *inotify) AddWith(path string, opts ...addOpt) error { + if w.isClosed() { + return ErrClosed + } + if debug { + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s AddWith(%q)\n", + time.Now().Format("15:04:05.000000000"), path) + } + + with := getOptions(opts...) + if !w.xSupports(with.op) { + return fmt.Errorf("%w: %s", xErrUnsupported, with.op) + } + + path, recurse := recursivePath(path) + if recurse { + return filepath.WalkDir(path, func(root string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + if root == path { + return fmt.Errorf("fsnotify: not a directory: %q", path) + } + return nil + } + + // Send a Create event when adding new directory from a recursive + // watch; this is for "mkdir -p one/two/three". Usually all those + // directories will be created before we can set up watchers on the + // subdirectories, so only "one" would be sent as a Create event and + // not "one/two" and "one/two/three" (inotifywait -r has the same + // problem). + if with.sendCreate && root != path { + w.sendEvent(Event{Name: root, Op: Create}) + } + + return w.add(root, with, true) + }) + } + + return w.add(path, with, false) +} + +func (w *inotify) add(path string, with withOpts, recurse bool) error { + var flags uint32 + if with.noFollow { + flags |= unix.IN_DONT_FOLLOW + } + if with.op.Has(Create) { + flags |= unix.IN_CREATE + } + if with.op.Has(Write) { + flags |= unix.IN_MODIFY + } + if with.op.Has(Remove) { + flags |= unix.IN_DELETE | unix.IN_DELETE_SELF + } + if with.op.Has(Rename) { + flags |= unix.IN_MOVED_TO | unix.IN_MOVED_FROM | unix.IN_MOVE_SELF + } + if with.op.Has(Chmod) { + flags |= unix.IN_ATTRIB + } + if with.op.Has(xUnportableOpen) { + flags |= unix.IN_OPEN + } + if with.op.Has(xUnportableRead) { + flags |= unix.IN_ACCESS + } + if with.op.Has(xUnportableCloseWrite) { + flags |= unix.IN_CLOSE_WRITE + } + if with.op.Has(xUnportableCloseRead) { + flags |= unix.IN_CLOSE_NOWRITE + } + return w.register(path, flags, recurse) +} + +func (w *inotify) register(path string, flags uint32, recurse bool) error { + return w.watches.updatePath(path, func(existing *watch) (*watch, error) { + if existing != nil { + flags |= existing.flags | unix.IN_MASK_ADD + } + + wd, err := unix.InotifyAddWatch(w.fd, path, flags) + if wd == -1 { + return nil, err + } + + if existing == nil { + return &watch{ + wd: uint32(wd), + path: path, + flags: flags, + recurse: recurse, + }, nil + } + + existing.wd = uint32(wd) + existing.flags = flags + return existing, nil + }) +} + +func (w *inotify) Remove(name string) error { + if w.isClosed() { + return nil + } + if debug { + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s Remove(%q)\n", + time.Now().Format("15:04:05.000000000"), name) + } + return w.remove(filepath.Clean(name)) +} + +func (w *inotify) remove(name string) error { + wds, err := w.watches.removePath(name) + if err != nil { + return err + } + + for _, wd := range wds { + _, err := unix.InotifyRmWatch(w.fd, wd) + if err != nil { + // TODO: Perhaps it's not helpful to return an error here in every + // case; the only two possible errors are: + // + // EBADF, which happens when w.fd is not a valid file descriptor of + // any kind. + // + // EINVAL, which is when fd is not an inotify descriptor or wd is + // not a valid watch descriptor. Watch descriptors are invalidated + // when they are removed explicitly or implicitly; explicitly by + // inotify_rm_watch, implicitly when the file they are watching is + // deleted. + return err + } + } + return nil +} + +func (w *inotify) WatchList() []string { + if w.isClosed() { + return nil + } + + entries := make([]string, 0, w.watches.len()) + w.watches.mu.RLock() + for pathname := range w.watches.path { + entries = append(entries, pathname) + } + w.watches.mu.RUnlock() + + return entries +} + +// readEvents reads from the inotify file descriptor, converts the +// received events into Event objects and sends them via the Events channel +func (w *inotify) readEvents() { + defer func() { + close(w.doneResp) + close(w.Errors) + close(w.Events) + }() + + var ( + buf [unix.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events + errno error // Syscall errno + ) + for { + // See if we have been closed. + if w.isClosed() { + return + } + + n, err := w.inotifyFile.Read(buf[:]) + switch { + case errors.Unwrap(err) == os.ErrClosed: + return + case err != nil: + if !w.sendError(err) { + return + } + continue + } + + if n < unix.SizeofInotifyEvent { + var err error + if n == 0 { + err = io.EOF // If EOF is received. This should really never happen. + } else if n < 0 { + err = errno // If an error occurred while reading. + } else { + err = errors.New("notify: short read in readEvents()") // Read was too short. + } + if !w.sendError(err) { + return + } + continue + } + + // We don't know how many events we just read into the buffer + // While the offset points to at least one whole event... + var offset uint32 + for offset <= uint32(n-unix.SizeofInotifyEvent) { + var ( + // Point "raw" to the event in the buffer + raw = (*unix.InotifyEvent)(unsafe.Pointer(&buf[offset])) + mask = uint32(raw.Mask) + nameLen = uint32(raw.Len) + // Move to the next event in the buffer + next = func() { offset += unix.SizeofInotifyEvent + nameLen } + ) + + if mask&unix.IN_Q_OVERFLOW != 0 { + if !w.sendError(ErrEventOverflow) { + return + } + } + + /// If the event happened to the watched directory or the watched + /// file, the kernel doesn't append the filename to the event, but + /// we would like to always fill the the "Name" field with a valid + /// filename. We retrieve the path of the watch from the "paths" + /// map. + watch := w.watches.byWd(uint32(raw.Wd)) + /// Can be nil if Remove() was called in another goroutine for this + /// path inbetween reading the events from the kernel and reading + /// the internal state. Not much we can do about it, so just skip. + /// See #616. + if watch == nil { + next() + continue + } + + name := watch.path + if nameLen > 0 { + /// Point "bytes" at the first byte of the filename + bytes := (*[unix.PathMax]byte)(unsafe.Pointer(&buf[offset+unix.SizeofInotifyEvent]))[:nameLen:nameLen] + /// The filename is padded with NULL bytes. TrimRight() gets rid of those. + name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000") + } + + if debug { + internal.Debug(name, raw.Mask, raw.Cookie) + } + + if mask&unix.IN_IGNORED != 0 { //&& event.Op != 0 + next() + continue + } + + // inotify will automatically remove the watch on deletes; just need + // to clean our state here. + if mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF { + w.watches.remove(watch.wd) + } + + // We can't really update the state when a watched path is moved; + // only IN_MOVE_SELF is sent and not IN_MOVED_{FROM,TO}. So remove + // the watch. + if mask&unix.IN_MOVE_SELF == unix.IN_MOVE_SELF { + if watch.recurse { + next() // Do nothing + continue + } + + err := w.remove(watch.path) + if err != nil && !errors.Is(err, ErrNonExistentWatch) { + if !w.sendError(err) { + return + } + } + } + + /// Skip if we're watching both this path and the parent; the parent + /// will already send a delete so no need to do it twice. + if mask&unix.IN_DELETE_SELF != 0 { + if _, ok := w.watches.path[filepath.Dir(watch.path)]; ok { + next() + continue + } + } + + ev := w.newEvent(name, mask, raw.Cookie) + // Need to update watch path for recurse. + if watch.recurse { + isDir := mask&unix.IN_ISDIR == unix.IN_ISDIR + /// New directory created: set up watch on it. + if isDir && ev.Has(Create) { + err := w.register(ev.Name, watch.flags, true) + if !w.sendError(err) { + return + } + + // This was a directory rename, so we need to update all + // the children. + // + // TODO: this is of course pretty slow; we should use a + // better data structure for storing all of this, e.g. store + // children in the watch. I have some code for this in my + // kqueue refactor we can use in the future. For now I'm + // okay with this as it's not publicly available. + // Correctness first, performance second. + if ev.renamedFrom != "" { + w.watches.mu.Lock() + for k, ww := range w.watches.wd { + if k == watch.wd || ww.path == ev.Name { + continue + } + if strings.HasPrefix(ww.path, ev.renamedFrom) { + ww.path = strings.Replace(ww.path, ev.renamedFrom, ev.Name, 1) + w.watches.wd[k] = ww + } + } + w.watches.mu.Unlock() + } + } + } + + /// Send the events that are not ignored on the events channel + if !w.sendEvent(ev) { + return + } + next() + } + } +} + +func (w *inotify) isRecursive(path string) bool { + ww := w.watches.byPath(path) + if ww == nil { // path could be a file, so also check the Dir. + ww = w.watches.byPath(filepath.Dir(path)) + } + return ww != nil && ww.recurse +} + +func (w *inotify) newEvent(name string, mask, cookie uint32) Event { + e := Event{Name: name} + if mask&unix.IN_CREATE == unix.IN_CREATE || mask&unix.IN_MOVED_TO == unix.IN_MOVED_TO { + e.Op |= Create + } + if mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF || mask&unix.IN_DELETE == unix.IN_DELETE { + e.Op |= Remove + } + if mask&unix.IN_MODIFY == unix.IN_MODIFY { + e.Op |= Write + } + if mask&unix.IN_OPEN == unix.IN_OPEN { + e.Op |= xUnportableOpen + } + if mask&unix.IN_ACCESS == unix.IN_ACCESS { + e.Op |= xUnportableRead + } + if mask&unix.IN_CLOSE_WRITE == unix.IN_CLOSE_WRITE { + e.Op |= xUnportableCloseWrite + } + if mask&unix.IN_CLOSE_NOWRITE == unix.IN_CLOSE_NOWRITE { + e.Op |= xUnportableCloseRead + } + if mask&unix.IN_MOVE_SELF == unix.IN_MOVE_SELF || mask&unix.IN_MOVED_FROM == unix.IN_MOVED_FROM { + e.Op |= Rename + } + if mask&unix.IN_ATTRIB == unix.IN_ATTRIB { + e.Op |= Chmod + } + + if cookie != 0 { + if mask&unix.IN_MOVED_FROM == unix.IN_MOVED_FROM { + w.cookiesMu.Lock() + w.cookies[w.cookieIndex] = koekje{cookie: cookie, path: e.Name} + w.cookieIndex++ + if w.cookieIndex > 9 { + w.cookieIndex = 0 + } + w.cookiesMu.Unlock() + } else if mask&unix.IN_MOVED_TO == unix.IN_MOVED_TO { + w.cookiesMu.Lock() + var prev string + for _, c := range w.cookies { + if c.cookie == cookie { + prev = c.path + break + } + } + w.cookiesMu.Unlock() + e.renamedFrom = prev + } + } + return e +} + +func (w *inotify) xSupports(op Op) bool { + return true // Supports everything. +} + +func (w *inotify) state() { + w.watches.mu.Lock() + defer w.watches.mu.Unlock() + for wd, ww := range w.watches.wd { + fmt.Fprintf(os.Stderr, "%4d: recurse=%t %q\n", wd, ww.recurse, ww.path) + } +} diff --git a/vendor/github.com/fsnotify/fsnotify/backend_kqueue.go b/vendor/github.com/fsnotify/fsnotify/backend_kqueue.go new file mode 100644 index 000000000..d8de5ab76 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/backend_kqueue.go @@ -0,0 +1,733 @@ +//go:build freebsd || openbsd || netbsd || dragonfly || darwin + +package fsnotify + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "sync" + "time" + + "github.com/fsnotify/fsnotify/internal" + "golang.org/x/sys/unix" +) + +type kqueue struct { + Events chan Event + Errors chan error + + kq int // File descriptor (as returned by the kqueue() syscall). + closepipe [2]int // Pipe used for closing kq. + watches *watches + done chan struct{} + doneMu sync.Mutex +} + +type ( + watches struct { + mu sync.RWMutex + wd map[int]watch // wd → watch + path map[string]int // pathname → wd + byDir map[string]map[int]struct{} // dirname(path) → wd + seen map[string]struct{} // Keep track of if we know this file exists. + byUser map[string]struct{} // Watches added with Watcher.Add() + } + watch struct { + wd int + name string + linkName string // In case of links; name is the target, and this is the link. + isDir bool + dirFlags uint32 + } +) + +func newWatches() *watches { + return &watches{ + wd: make(map[int]watch), + path: make(map[string]int), + byDir: make(map[string]map[int]struct{}), + seen: make(map[string]struct{}), + byUser: make(map[string]struct{}), + } +} + +func (w *watches) listPaths(userOnly bool) []string { + w.mu.RLock() + defer w.mu.RUnlock() + + if userOnly { + l := make([]string, 0, len(w.byUser)) + for p := range w.byUser { + l = append(l, p) + } + return l + } + + l := make([]string, 0, len(w.path)) + for p := range w.path { + l = append(l, p) + } + return l +} + +func (w *watches) watchesInDir(path string) []string { + w.mu.RLock() + defer w.mu.RUnlock() + + l := make([]string, 0, 4) + for fd := range w.byDir[path] { + info := w.wd[fd] + if _, ok := w.byUser[info.name]; !ok { + l = append(l, info.name) + } + } + return l +} + +// Mark path as added by the user. +func (w *watches) addUserWatch(path string) { + w.mu.Lock() + defer w.mu.Unlock() + w.byUser[path] = struct{}{} +} + +func (w *watches) addLink(path string, fd int) { + w.mu.Lock() + defer w.mu.Unlock() + + w.path[path] = fd + w.seen[path] = struct{}{} +} + +func (w *watches) add(path, linkPath string, fd int, isDir bool) { + w.mu.Lock() + defer w.mu.Unlock() + + w.path[path] = fd + w.wd[fd] = watch{wd: fd, name: path, linkName: linkPath, isDir: isDir} + + parent := filepath.Dir(path) + byDir, ok := w.byDir[parent] + if !ok { + byDir = make(map[int]struct{}, 1) + w.byDir[parent] = byDir + } + byDir[fd] = struct{}{} +} + +func (w *watches) byWd(fd int) (watch, bool) { + w.mu.RLock() + defer w.mu.RUnlock() + info, ok := w.wd[fd] + return info, ok +} + +func (w *watches) byPath(path string) (watch, bool) { + w.mu.RLock() + defer w.mu.RUnlock() + info, ok := w.wd[w.path[path]] + return info, ok +} + +func (w *watches) updateDirFlags(path string, flags uint32) { + w.mu.Lock() + defer w.mu.Unlock() + + fd := w.path[path] + info := w.wd[fd] + info.dirFlags = flags + w.wd[fd] = info +} + +func (w *watches) remove(fd int, path string) bool { + w.mu.Lock() + defer w.mu.Unlock() + + isDir := w.wd[fd].isDir + delete(w.path, path) + delete(w.byUser, path) + + parent := filepath.Dir(path) + delete(w.byDir[parent], fd) + + if len(w.byDir[parent]) == 0 { + delete(w.byDir, parent) + } + + delete(w.wd, fd) + delete(w.seen, path) + return isDir +} + +func (w *watches) markSeen(path string, exists bool) { + w.mu.Lock() + defer w.mu.Unlock() + if exists { + w.seen[path] = struct{}{} + } else { + delete(w.seen, path) + } +} + +func (w *watches) seenBefore(path string) bool { + w.mu.RLock() + defer w.mu.RUnlock() + _, ok := w.seen[path] + return ok +} + +func newBackend(ev chan Event, errs chan error) (backend, error) { + return newBufferedBackend(0, ev, errs) +} + +func newBufferedBackend(sz uint, ev chan Event, errs chan error) (backend, error) { + kq, closepipe, err := newKqueue() + if err != nil { + return nil, err + } + + w := &kqueue{ + Events: ev, + Errors: errs, + kq: kq, + closepipe: closepipe, + done: make(chan struct{}), + watches: newWatches(), + } + + go w.readEvents() + return w, nil +} + +// newKqueue creates a new kernel event queue and returns a descriptor. +// +// This registers a new event on closepipe, which will trigger an event when +// it's closed. This way we can use kevent() without timeout/polling; without +// the closepipe, it would block forever and we wouldn't be able to stop it at +// all. +func newKqueue() (kq int, closepipe [2]int, err error) { + kq, err = unix.Kqueue() + if kq == -1 { + return kq, closepipe, err + } + + // Register the close pipe. + err = unix.Pipe(closepipe[:]) + if err != nil { + unix.Close(kq) + return kq, closepipe, err + } + unix.CloseOnExec(closepipe[0]) + unix.CloseOnExec(closepipe[1]) + + // Register changes to listen on the closepipe. + changes := make([]unix.Kevent_t, 1) + // SetKevent converts int to the platform-specific types. + unix.SetKevent(&changes[0], closepipe[0], unix.EVFILT_READ, + unix.EV_ADD|unix.EV_ENABLE|unix.EV_ONESHOT) + + ok, err := unix.Kevent(kq, changes, nil, nil) + if ok == -1 { + unix.Close(kq) + unix.Close(closepipe[0]) + unix.Close(closepipe[1]) + return kq, closepipe, err + } + return kq, closepipe, nil +} + +// Returns true if the event was sent, or false if watcher is closed. +func (w *kqueue) sendEvent(e Event) bool { + select { + case <-w.done: + return false + case w.Events <- e: + return true + } +} + +// Returns true if the error was sent, or false if watcher is closed. +func (w *kqueue) sendError(err error) bool { + if err == nil { + return true + } + select { + case <-w.done: + return false + case w.Errors <- err: + return true + } +} + +func (w *kqueue) isClosed() bool { + select { + case <-w.done: + return true + default: + return false + } +} + +func (w *kqueue) Close() error { + w.doneMu.Lock() + if w.isClosed() { + w.doneMu.Unlock() + return nil + } + close(w.done) + w.doneMu.Unlock() + + pathsToRemove := w.watches.listPaths(false) + for _, name := range pathsToRemove { + w.Remove(name) + } + + // Send "quit" message to the reader goroutine. + unix.Close(w.closepipe[1]) + return nil +} + +func (w *kqueue) Add(name string) error { return w.AddWith(name) } + +func (w *kqueue) AddWith(name string, opts ...addOpt) error { + if debug { + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s AddWith(%q)\n", + time.Now().Format("15:04:05.000000000"), name) + } + + with := getOptions(opts...) + if !w.xSupports(with.op) { + return fmt.Errorf("%w: %s", xErrUnsupported, with.op) + } + + _, err := w.addWatch(name, noteAllEvents) + if err != nil { + return err + } + w.watches.addUserWatch(name) + return nil +} + +func (w *kqueue) Remove(name string) error { + if debug { + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s Remove(%q)\n", + time.Now().Format("15:04:05.000000000"), name) + } + return w.remove(name, true) +} + +func (w *kqueue) remove(name string, unwatchFiles bool) error { + if w.isClosed() { + return nil + } + + name = filepath.Clean(name) + info, ok := w.watches.byPath(name) + if !ok { + return fmt.Errorf("%w: %s", ErrNonExistentWatch, name) + } + + err := w.register([]int{info.wd}, unix.EV_DELETE, 0) + if err != nil { + return err + } + + unix.Close(info.wd) + + isDir := w.watches.remove(info.wd, name) + + // Find all watched paths that are in this directory that are not external. + if unwatchFiles && isDir { + pathsToRemove := w.watches.watchesInDir(name) + for _, name := range pathsToRemove { + // Since these are internal, not much sense in propagating error to + // the user, as that will just confuse them with an error about a + // path they did not explicitly watch themselves. + w.Remove(name) + } + } + return nil +} + +func (w *kqueue) WatchList() []string { + if w.isClosed() { + return nil + } + return w.watches.listPaths(true) +} + +// Watch all events (except NOTE_EXTEND, NOTE_LINK, NOTE_REVOKE) +const noteAllEvents = unix.NOTE_DELETE | unix.NOTE_WRITE | unix.NOTE_ATTRIB | unix.NOTE_RENAME + +// addWatch adds name to the watched file set; the flags are interpreted as +// described in kevent(2). +// +// Returns the real path to the file which was added, with symlinks resolved. +func (w *kqueue) addWatch(name string, flags uint32) (string, error) { + if w.isClosed() { + return "", ErrClosed + } + + name = filepath.Clean(name) + + info, alreadyWatching := w.watches.byPath(name) + if !alreadyWatching { + fi, err := os.Lstat(name) + if err != nil { + return "", err + } + + // Don't watch sockets or named pipes. + if (fi.Mode()&os.ModeSocket == os.ModeSocket) || (fi.Mode()&os.ModeNamedPipe == os.ModeNamedPipe) { + return "", nil + } + + // Follow symlinks. + if fi.Mode()&os.ModeSymlink == os.ModeSymlink { + link, err := os.Readlink(name) + if err != nil { + // Return nil because Linux can add unresolvable symlinks to the + // watch list without problems, so maintain consistency with + // that. There will be no file events for broken symlinks. + // TODO: more specific check; returns os.PathError; ENOENT? + return "", nil + } + + _, alreadyWatching = w.watches.byPath(link) + if alreadyWatching { + // Add to watches so we don't get spurious Create events later + // on when we diff the directories. + w.watches.addLink(name, 0) + return link, nil + } + + info.linkName = name + name = link + fi, err = os.Lstat(name) + if err != nil { + return "", nil + } + } + + // Retry on EINTR; open() can return EINTR in practice on macOS. + // See #354, and Go issues 11180 and 39237. + for { + info.wd, err = unix.Open(name, openMode, 0) + if err == nil { + break + } + if errors.Is(err, unix.EINTR) { + continue + } + + return "", err + } + + info.isDir = fi.IsDir() + } + + err := w.register([]int{info.wd}, unix.EV_ADD|unix.EV_CLEAR|unix.EV_ENABLE, flags) + if err != nil { + unix.Close(info.wd) + return "", err + } + + if !alreadyWatching { + w.watches.add(name, info.linkName, info.wd, info.isDir) + } + + // Watch the directory if it has not been watched before, or if it was + // watched before, but perhaps only a NOTE_DELETE (watchDirectoryFiles) + if info.isDir { + watchDir := (flags&unix.NOTE_WRITE) == unix.NOTE_WRITE && + (!alreadyWatching || (info.dirFlags&unix.NOTE_WRITE) != unix.NOTE_WRITE) + w.watches.updateDirFlags(name, flags) + + if watchDir { + if err := w.watchDirectoryFiles(name); err != nil { + return "", err + } + } + } + return name, nil +} + +// readEvents reads from kqueue and converts the received kevents into +// Event values that it sends down the Events channel. +func (w *kqueue) readEvents() { + defer func() { + close(w.Events) + close(w.Errors) + _ = unix.Close(w.kq) + unix.Close(w.closepipe[0]) + }() + + eventBuffer := make([]unix.Kevent_t, 10) + for { + kevents, err := w.read(eventBuffer) + // EINTR is okay, the syscall was interrupted before timeout expired. + if err != nil && err != unix.EINTR { + if !w.sendError(fmt.Errorf("fsnotify.readEvents: %w", err)) { + return + } + } + + for _, kevent := range kevents { + var ( + wd = int(kevent.Ident) + mask = uint32(kevent.Fflags) + ) + + // Shut down the loop when the pipe is closed, but only after all + // other events have been processed. + if wd == w.closepipe[0] { + return + } + + path, ok := w.watches.byWd(wd) + if debug { + internal.Debug(path.name, &kevent) + } + + // On macOS it seems that sometimes an event with Ident=0 is + // delivered, and no other flags/information beyond that, even + // though we never saw such a file descriptor. For example in + // TestWatchSymlink/277 (usually at the end, but sometimes sooner): + // + // fmt.Printf("READ: %2d %#v\n", kevent.Ident, kevent) + // unix.Kevent_t{Ident:0x2a, Filter:-4, Flags:0x25, Fflags:0x2, Data:0, Udata:(*uint8)(nil)} + // unix.Kevent_t{Ident:0x0, Filter:-4, Flags:0x25, Fflags:0x2, Data:0, Udata:(*uint8)(nil)} + // + // The first is a normal event, the second with Ident 0. No error + // flag, no data, no ... nothing. + // + // I read a bit through bsd/kern_event.c from the xnu source, but I + // don't really see an obvious location where this is triggered – + // this doesn't seem intentional, but idk... + // + // Technically fd 0 is a valid descriptor, so only skip it if + // there's no path, and if we're on macOS. + if !ok && kevent.Ident == 0 && runtime.GOOS == "darwin" { + continue + } + + event := w.newEvent(path.name, path.linkName, mask) + + if event.Has(Rename) || event.Has(Remove) { + w.remove(event.Name, false) + w.watches.markSeen(event.Name, false) + } + + if path.isDir && event.Has(Write) && !event.Has(Remove) { + w.dirChange(event.Name) + } else if !w.sendEvent(event) { + return + } + + if event.Has(Remove) { + // Look for a file that may have overwritten this; for example, + // mv f1 f2 will delete f2, then create f2. + if path.isDir { + fileDir := filepath.Clean(event.Name) + _, found := w.watches.byPath(fileDir) + if found { + // TODO: this branch is never triggered in any test. + // Added in d6220df (2012). + // isDir check added in 8611c35 (2016): https://github.com/fsnotify/fsnotify/pull/111 + // + // I don't really get how this can be triggered either. + // And it wasn't triggered in the patch that added it, + // either. + // + // Original also had a comment: + // make sure the directory exists before we watch for + // changes. When we do a recursive watch and perform + // rm -rf, the parent directory might have gone + // missing, ignore the missing directory and let the + // upcoming delete event remove the watch from the + // parent directory. + err := w.dirChange(fileDir) + if !w.sendError(err) { + return + } + } + } else { + path := filepath.Clean(event.Name) + if fi, err := os.Lstat(path); err == nil { + err := w.sendCreateIfNew(path, fi) + if !w.sendError(err) { + return + } + } + } + } + } + } +} + +// newEvent returns an platform-independent Event based on kqueue Fflags. +func (w *kqueue) newEvent(name, linkName string, mask uint32) Event { + e := Event{Name: name} + if linkName != "" { + // If the user watched "/path/link" then emit events as "/path/link" + // rather than "/path/target". + e.Name = linkName + } + + if mask&unix.NOTE_DELETE == unix.NOTE_DELETE { + e.Op |= Remove + } + if mask&unix.NOTE_WRITE == unix.NOTE_WRITE { + e.Op |= Write + } + if mask&unix.NOTE_RENAME == unix.NOTE_RENAME { + e.Op |= Rename + } + if mask&unix.NOTE_ATTRIB == unix.NOTE_ATTRIB { + e.Op |= Chmod + } + // No point sending a write and delete event at the same time: if it's gone, + // then it's gone. + if e.Op.Has(Write) && e.Op.Has(Remove) { + e.Op &^= Write + } + return e +} + +// watchDirectoryFiles to mimic inotify when adding a watch on a directory +func (w *kqueue) watchDirectoryFiles(dirPath string) error { + files, err := os.ReadDir(dirPath) + if err != nil { + return err + } + + for _, f := range files { + path := filepath.Join(dirPath, f.Name()) + + fi, err := f.Info() + if err != nil { + return fmt.Errorf("%q: %w", path, err) + } + + cleanPath, err := w.internalWatch(path, fi) + if err != nil { + // No permission to read the file; that's not a problem: just skip. + // But do add it to w.fileExists to prevent it from being picked up + // as a "new" file later (it still shows up in the directory + // listing). + switch { + case errors.Is(err, unix.EACCES) || errors.Is(err, unix.EPERM): + cleanPath = filepath.Clean(path) + default: + return fmt.Errorf("%q: %w", path, err) + } + } + + w.watches.markSeen(cleanPath, true) + } + + return nil +} + +// Search the directory for new files and send an event for them. +// +// This functionality is to have the BSD watcher match the inotify, which sends +// a create event for files created in a watched directory. +func (w *kqueue) dirChange(dir string) error { + files, err := os.ReadDir(dir) + if err != nil { + // Directory no longer exists: we can ignore this safely. kqueue will + // still give us the correct events. + if errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("fsnotify.dirChange: %w", err) + } + + for _, f := range files { + fi, err := f.Info() + if err != nil { + return fmt.Errorf("fsnotify.dirChange: %w", err) + } + + err = w.sendCreateIfNew(filepath.Join(dir, fi.Name()), fi) + if err != nil { + // Don't need to send an error if this file isn't readable. + if errors.Is(err, unix.EACCES) || errors.Is(err, unix.EPERM) { + return nil + } + return fmt.Errorf("fsnotify.dirChange: %w", err) + } + } + return nil +} + +// Send a create event if the file isn't already being tracked, and start +// watching this file. +func (w *kqueue) sendCreateIfNew(path string, fi os.FileInfo) error { + if !w.watches.seenBefore(path) { + if !w.sendEvent(Event{Name: path, Op: Create}) { + return nil + } + } + + // Like watchDirectoryFiles, but without doing another ReadDir. + path, err := w.internalWatch(path, fi) + if err != nil { + return err + } + w.watches.markSeen(path, true) + return nil +} + +func (w *kqueue) internalWatch(name string, fi os.FileInfo) (string, error) { + if fi.IsDir() { + // mimic Linux providing delete events for subdirectories, but preserve + // the flags used if currently watching subdirectory + info, _ := w.watches.byPath(name) + return w.addWatch(name, info.dirFlags|unix.NOTE_DELETE|unix.NOTE_RENAME) + } + + // watch file to mimic Linux inotify + return w.addWatch(name, noteAllEvents) +} + +// Register events with the queue. +func (w *kqueue) register(fds []int, flags int, fflags uint32) error { + changes := make([]unix.Kevent_t, len(fds)) + for i, fd := range fds { + // SetKevent converts int to the platform-specific types. + unix.SetKevent(&changes[i], fd, unix.EVFILT_VNODE, flags) + changes[i].Fflags = fflags + } + + // Register the events. + success, err := unix.Kevent(w.kq, changes, nil, nil) + if success == -1 { + return err + } + return nil +} + +// read retrieves pending events, or waits until an event occurs. +func (w *kqueue) read(events []unix.Kevent_t) ([]unix.Kevent_t, error) { + n, err := unix.Kevent(w.kq, nil, events, nil) + if err != nil { + return nil, err + } + return events[0:n], nil +} + +func (w *kqueue) xSupports(op Op) bool { + if runtime.GOOS == "freebsd" { + //return true // Supports everything. + } + if op.Has(xUnportableOpen) || op.Has(xUnportableRead) || + op.Has(xUnportableCloseWrite) || op.Has(xUnportableCloseRead) { + return false + } + return true +} diff --git a/vendor/github.com/fsnotify/fsnotify/backend_other.go b/vendor/github.com/fsnotify/fsnotify/backend_other.go new file mode 100644 index 000000000..5eb5dbc66 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/backend_other.go @@ -0,0 +1,23 @@ +//go:build appengine || (!darwin && !dragonfly && !freebsd && !openbsd && !linux && !netbsd && !solaris && !windows) + +package fsnotify + +import "errors" + +type other struct { + Events chan Event + Errors chan error +} + +func newBackend(ev chan Event, errs chan error) (backend, error) { + return nil, errors.New("fsnotify not supported on the current platform") +} +func newBufferedBackend(sz uint, ev chan Event, errs chan error) (backend, error) { + return newBackend(ev, errs) +} +func (w *other) Close() error { return nil } +func (w *other) WatchList() []string { return nil } +func (w *other) Add(name string) error { return nil } +func (w *other) AddWith(name string, opts ...addOpt) error { return nil } +func (w *other) Remove(name string) error { return nil } +func (w *other) xSupports(op Op) bool { return false } diff --git a/vendor/github.com/fsnotify/fsnotify/backend_windows.go b/vendor/github.com/fsnotify/fsnotify/backend_windows.go new file mode 100644 index 000000000..c54a63083 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/backend_windows.go @@ -0,0 +1,682 @@ +//go:build windows + +// Windows backend based on ReadDirectoryChangesW() +// +// https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-readdirectorychangesw + +package fsnotify + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "sync" + "time" + "unsafe" + + "github.com/fsnotify/fsnotify/internal" + "golang.org/x/sys/windows" +) + +type readDirChangesW struct { + Events chan Event + Errors chan error + + port windows.Handle // Handle to completion port + input chan *input // Inputs to the reader are sent on this channel + quit chan chan<- error + + mu sync.Mutex // Protects access to watches, closed + watches watchMap // Map of watches (key: i-number) + closed bool // Set to true when Close() is first called +} + +func newBackend(ev chan Event, errs chan error) (backend, error) { + return newBufferedBackend(50, ev, errs) +} + +func newBufferedBackend(sz uint, ev chan Event, errs chan error) (backend, error) { + port, err := windows.CreateIoCompletionPort(windows.InvalidHandle, 0, 0, 0) + if err != nil { + return nil, os.NewSyscallError("CreateIoCompletionPort", err) + } + w := &readDirChangesW{ + Events: ev, + Errors: errs, + port: port, + watches: make(watchMap), + input: make(chan *input, 1), + quit: make(chan chan<- error, 1), + } + go w.readEvents() + return w, nil +} + +func (w *readDirChangesW) isClosed() bool { + w.mu.Lock() + defer w.mu.Unlock() + return w.closed +} + +func (w *readDirChangesW) sendEvent(name, renamedFrom string, mask uint64) bool { + if mask == 0 { + return false + } + + event := w.newEvent(name, uint32(mask)) + event.renamedFrom = renamedFrom + select { + case ch := <-w.quit: + w.quit <- ch + case w.Events <- event: + } + return true +} + +// Returns true if the error was sent, or false if watcher is closed. +func (w *readDirChangesW) sendError(err error) bool { + if err == nil { + return true + } + select { + case w.Errors <- err: + return true + case <-w.quit: + return false + } +} + +func (w *readDirChangesW) Close() error { + if w.isClosed() { + return nil + } + + w.mu.Lock() + w.closed = true + w.mu.Unlock() + + // Send "quit" message to the reader goroutine + ch := make(chan error) + w.quit <- ch + if err := w.wakeupReader(); err != nil { + return err + } + return <-ch +} + +func (w *readDirChangesW) Add(name string) error { return w.AddWith(name) } + +func (w *readDirChangesW) AddWith(name string, opts ...addOpt) error { + if w.isClosed() { + return ErrClosed + } + if debug { + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s AddWith(%q)\n", + time.Now().Format("15:04:05.000000000"), filepath.ToSlash(name)) + } + + with := getOptions(opts...) + if !w.xSupports(with.op) { + return fmt.Errorf("%w: %s", xErrUnsupported, with.op) + } + if with.bufsize < 4096 { + return fmt.Errorf("fsnotify.WithBufferSize: buffer size cannot be smaller than 4096 bytes") + } + + in := &input{ + op: opAddWatch, + path: filepath.Clean(name), + flags: sysFSALLEVENTS, + reply: make(chan error), + bufsize: with.bufsize, + } + w.input <- in + if err := w.wakeupReader(); err != nil { + return err + } + return <-in.reply +} + +func (w *readDirChangesW) Remove(name string) error { + if w.isClosed() { + return nil + } + if debug { + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s Remove(%q)\n", + time.Now().Format("15:04:05.000000000"), filepath.ToSlash(name)) + } + + in := &input{ + op: opRemoveWatch, + path: filepath.Clean(name), + reply: make(chan error), + } + w.input <- in + if err := w.wakeupReader(); err != nil { + return err + } + return <-in.reply +} + +func (w *readDirChangesW) WatchList() []string { + if w.isClosed() { + return nil + } + + w.mu.Lock() + defer w.mu.Unlock() + + entries := make([]string, 0, len(w.watches)) + for _, entry := range w.watches { + for _, watchEntry := range entry { + for name := range watchEntry.names { + entries = append(entries, filepath.Join(watchEntry.path, name)) + } + // the directory itself is being watched + if watchEntry.mask != 0 { + entries = append(entries, watchEntry.path) + } + } + } + + return entries +} + +// These options are from the old golang.org/x/exp/winfsnotify, where you could +// add various options to the watch. This has long since been removed. +// +// The "sys" in the name is misleading as they're not part of any "system". +// +// This should all be removed at some point, and just use windows.FILE_NOTIFY_* +const ( + sysFSALLEVENTS = 0xfff + sysFSCREATE = 0x100 + sysFSDELETE = 0x200 + sysFSDELETESELF = 0x400 + sysFSMODIFY = 0x2 + sysFSMOVE = 0xc0 + sysFSMOVEDFROM = 0x40 + sysFSMOVEDTO = 0x80 + sysFSMOVESELF = 0x800 + sysFSIGNORED = 0x8000 +) + +func (w *readDirChangesW) newEvent(name string, mask uint32) Event { + e := Event{Name: name} + if mask&sysFSCREATE == sysFSCREATE || mask&sysFSMOVEDTO == sysFSMOVEDTO { + e.Op |= Create + } + if mask&sysFSDELETE == sysFSDELETE || mask&sysFSDELETESELF == sysFSDELETESELF { + e.Op |= Remove + } + if mask&sysFSMODIFY == sysFSMODIFY { + e.Op |= Write + } + if mask&sysFSMOVE == sysFSMOVE || mask&sysFSMOVESELF == sysFSMOVESELF || mask&sysFSMOVEDFROM == sysFSMOVEDFROM { + e.Op |= Rename + } + return e +} + +const ( + opAddWatch = iota + opRemoveWatch +) + +const ( + provisional uint64 = 1 << (32 + iota) +) + +type input struct { + op int + path string + flags uint32 + bufsize int + reply chan error +} + +type inode struct { + handle windows.Handle + volume uint32 + index uint64 +} + +type watch struct { + ov windows.Overlapped + ino *inode // i-number + recurse bool // Recursive watch? + path string // Directory path + mask uint64 // Directory itself is being watched with these notify flags + names map[string]uint64 // Map of names being watched and their notify flags + rename string // Remembers the old name while renaming a file + buf []byte // buffer, allocated later +} + +type ( + indexMap map[uint64]*watch + watchMap map[uint32]indexMap +) + +func (w *readDirChangesW) wakeupReader() error { + err := windows.PostQueuedCompletionStatus(w.port, 0, 0, nil) + if err != nil { + return os.NewSyscallError("PostQueuedCompletionStatus", err) + } + return nil +} + +func (w *readDirChangesW) getDir(pathname string) (dir string, err error) { + attr, err := windows.GetFileAttributes(windows.StringToUTF16Ptr(pathname)) + if err != nil { + return "", os.NewSyscallError("GetFileAttributes", err) + } + if attr&windows.FILE_ATTRIBUTE_DIRECTORY != 0 { + dir = pathname + } else { + dir, _ = filepath.Split(pathname) + dir = filepath.Clean(dir) + } + return +} + +func (w *readDirChangesW) getIno(path string) (ino *inode, err error) { + h, err := windows.CreateFile(windows.StringToUTF16Ptr(path), + windows.FILE_LIST_DIRECTORY, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, + nil, windows.OPEN_EXISTING, + windows.FILE_FLAG_BACKUP_SEMANTICS|windows.FILE_FLAG_OVERLAPPED, 0) + if err != nil { + return nil, os.NewSyscallError("CreateFile", err) + } + + var fi windows.ByHandleFileInformation + err = windows.GetFileInformationByHandle(h, &fi) + if err != nil { + windows.CloseHandle(h) + return nil, os.NewSyscallError("GetFileInformationByHandle", err) + } + ino = &inode{ + handle: h, + volume: fi.VolumeSerialNumber, + index: uint64(fi.FileIndexHigh)<<32 | uint64(fi.FileIndexLow), + } + return ino, nil +} + +// Must run within the I/O thread. +func (m watchMap) get(ino *inode) *watch { + if i := m[ino.volume]; i != nil { + return i[ino.index] + } + return nil +} + +// Must run within the I/O thread. +func (m watchMap) set(ino *inode, watch *watch) { + i := m[ino.volume] + if i == nil { + i = make(indexMap) + m[ino.volume] = i + } + i[ino.index] = watch +} + +// Must run within the I/O thread. +func (w *readDirChangesW) addWatch(pathname string, flags uint64, bufsize int) error { + pathname, recurse := recursivePath(pathname) + + dir, err := w.getDir(pathname) + if err != nil { + return err + } + + ino, err := w.getIno(dir) + if err != nil { + return err + } + w.mu.Lock() + watchEntry := w.watches.get(ino) + w.mu.Unlock() + if watchEntry == nil { + _, err := windows.CreateIoCompletionPort(ino.handle, w.port, 0, 0) + if err != nil { + windows.CloseHandle(ino.handle) + return os.NewSyscallError("CreateIoCompletionPort", err) + } + watchEntry = &watch{ + ino: ino, + path: dir, + names: make(map[string]uint64), + recurse: recurse, + buf: make([]byte, bufsize), + } + w.mu.Lock() + w.watches.set(ino, watchEntry) + w.mu.Unlock() + flags |= provisional + } else { + windows.CloseHandle(ino.handle) + } + if pathname == dir { + watchEntry.mask |= flags + } else { + watchEntry.names[filepath.Base(pathname)] |= flags + } + + err = w.startRead(watchEntry) + if err != nil { + return err + } + + if pathname == dir { + watchEntry.mask &= ^provisional + } else { + watchEntry.names[filepath.Base(pathname)] &= ^provisional + } + return nil +} + +// Must run within the I/O thread. +func (w *readDirChangesW) remWatch(pathname string) error { + pathname, recurse := recursivePath(pathname) + + dir, err := w.getDir(pathname) + if err != nil { + return err + } + ino, err := w.getIno(dir) + if err != nil { + return err + } + + w.mu.Lock() + watch := w.watches.get(ino) + w.mu.Unlock() + + if recurse && !watch.recurse { + return fmt.Errorf("can't use \\... with non-recursive watch %q", pathname) + } + + err = windows.CloseHandle(ino.handle) + if err != nil { + w.sendError(os.NewSyscallError("CloseHandle", err)) + } + if watch == nil { + return fmt.Errorf("%w: %s", ErrNonExistentWatch, pathname) + } + if pathname == dir { + w.sendEvent(watch.path, "", watch.mask&sysFSIGNORED) + watch.mask = 0 + } else { + name := filepath.Base(pathname) + w.sendEvent(filepath.Join(watch.path, name), "", watch.names[name]&sysFSIGNORED) + delete(watch.names, name) + } + + return w.startRead(watch) +} + +// Must run within the I/O thread. +func (w *readDirChangesW) deleteWatch(watch *watch) { + for name, mask := range watch.names { + if mask&provisional == 0 { + w.sendEvent(filepath.Join(watch.path, name), "", mask&sysFSIGNORED) + } + delete(watch.names, name) + } + if watch.mask != 0 { + if watch.mask&provisional == 0 { + w.sendEvent(watch.path, "", watch.mask&sysFSIGNORED) + } + watch.mask = 0 + } +} + +// Must run within the I/O thread. +func (w *readDirChangesW) startRead(watch *watch) error { + err := windows.CancelIo(watch.ino.handle) + if err != nil { + w.sendError(os.NewSyscallError("CancelIo", err)) + w.deleteWatch(watch) + } + mask := w.toWindowsFlags(watch.mask) + for _, m := range watch.names { + mask |= w.toWindowsFlags(m) + } + if mask == 0 { + err := windows.CloseHandle(watch.ino.handle) + if err != nil { + w.sendError(os.NewSyscallError("CloseHandle", err)) + } + w.mu.Lock() + delete(w.watches[watch.ino.volume], watch.ino.index) + w.mu.Unlock() + return nil + } + + // We need to pass the array, rather than the slice. + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&watch.buf)) + rdErr := windows.ReadDirectoryChanges(watch.ino.handle, + (*byte)(unsafe.Pointer(hdr.Data)), uint32(hdr.Len), + watch.recurse, mask, nil, &watch.ov, 0) + if rdErr != nil { + err := os.NewSyscallError("ReadDirectoryChanges", rdErr) + if rdErr == windows.ERROR_ACCESS_DENIED && watch.mask&provisional == 0 { + // Watched directory was probably removed + w.sendEvent(watch.path, "", watch.mask&sysFSDELETESELF) + err = nil + } + w.deleteWatch(watch) + w.startRead(watch) + return err + } + return nil +} + +// readEvents reads from the I/O completion port, converts the +// received events into Event objects and sends them via the Events channel. +// Entry point to the I/O thread. +func (w *readDirChangesW) readEvents() { + var ( + n uint32 + key uintptr + ov *windows.Overlapped + ) + runtime.LockOSThread() + + for { + // This error is handled after the watch == nil check below. + qErr := windows.GetQueuedCompletionStatus(w.port, &n, &key, &ov, windows.INFINITE) + + watch := (*watch)(unsafe.Pointer(ov)) + if watch == nil { + select { + case ch := <-w.quit: + w.mu.Lock() + var indexes []indexMap + for _, index := range w.watches { + indexes = append(indexes, index) + } + w.mu.Unlock() + for _, index := range indexes { + for _, watch := range index { + w.deleteWatch(watch) + w.startRead(watch) + } + } + + err := windows.CloseHandle(w.port) + if err != nil { + err = os.NewSyscallError("CloseHandle", err) + } + close(w.Events) + close(w.Errors) + ch <- err + return + case in := <-w.input: + switch in.op { + case opAddWatch: + in.reply <- w.addWatch(in.path, uint64(in.flags), in.bufsize) + case opRemoveWatch: + in.reply <- w.remWatch(in.path) + } + default: + } + continue + } + + switch qErr { + case nil: + // No error + case windows.ERROR_MORE_DATA: + if watch == nil { + w.sendError(errors.New("ERROR_MORE_DATA has unexpectedly null lpOverlapped buffer")) + } else { + // The i/o succeeded but the buffer is full. + // In theory we should be building up a full packet. + // In practice we can get away with just carrying on. + n = uint32(unsafe.Sizeof(watch.buf)) + } + case windows.ERROR_ACCESS_DENIED: + // Watched directory was probably removed + w.sendEvent(watch.path, "", watch.mask&sysFSDELETESELF) + w.deleteWatch(watch) + w.startRead(watch) + continue + case windows.ERROR_OPERATION_ABORTED: + // CancelIo was called on this handle + continue + default: + w.sendError(os.NewSyscallError("GetQueuedCompletionPort", qErr)) + continue + } + + var offset uint32 + for { + if n == 0 { + w.sendError(ErrEventOverflow) + break + } + + // Point "raw" to the event in the buffer + raw := (*windows.FileNotifyInformation)(unsafe.Pointer(&watch.buf[offset])) + + // Create a buf that is the size of the path name + size := int(raw.FileNameLength / 2) + var buf []uint16 + // TODO: Use unsafe.Slice in Go 1.17; https://stackoverflow.com/questions/51187973 + sh := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) + sh.Data = uintptr(unsafe.Pointer(&raw.FileName)) + sh.Len = size + sh.Cap = size + name := windows.UTF16ToString(buf) + fullname := filepath.Join(watch.path, name) + + if debug { + internal.Debug(fullname, raw.Action) + } + + var mask uint64 + switch raw.Action { + case windows.FILE_ACTION_REMOVED: + mask = sysFSDELETESELF + case windows.FILE_ACTION_MODIFIED: + mask = sysFSMODIFY + case windows.FILE_ACTION_RENAMED_OLD_NAME: + watch.rename = name + case windows.FILE_ACTION_RENAMED_NEW_NAME: + // Update saved path of all sub-watches. + old := filepath.Join(watch.path, watch.rename) + w.mu.Lock() + for _, watchMap := range w.watches { + for _, ww := range watchMap { + if strings.HasPrefix(ww.path, old) { + ww.path = filepath.Join(fullname, strings.TrimPrefix(ww.path, old)) + } + } + } + w.mu.Unlock() + + if watch.names[watch.rename] != 0 { + watch.names[name] |= watch.names[watch.rename] + delete(watch.names, watch.rename) + mask = sysFSMOVESELF + } + } + + if raw.Action != windows.FILE_ACTION_RENAMED_NEW_NAME { + w.sendEvent(fullname, "", watch.names[name]&mask) + } + if raw.Action == windows.FILE_ACTION_REMOVED { + w.sendEvent(fullname, "", watch.names[name]&sysFSIGNORED) + delete(watch.names, name) + } + + if watch.rename != "" && raw.Action == windows.FILE_ACTION_RENAMED_NEW_NAME { + w.sendEvent(fullname, filepath.Join(watch.path, watch.rename), watch.mask&w.toFSnotifyFlags(raw.Action)) + } else { + w.sendEvent(fullname, "", watch.mask&w.toFSnotifyFlags(raw.Action)) + } + + if raw.Action == windows.FILE_ACTION_RENAMED_NEW_NAME { + w.sendEvent(filepath.Join(watch.path, watch.rename), "", watch.names[name]&mask) + } + + // Move to the next event in the buffer + if raw.NextEntryOffset == 0 { + break + } + offset += raw.NextEntryOffset + + // Error! + if offset >= n { + //lint:ignore ST1005 Windows should be capitalized + w.sendError(errors.New("Windows system assumed buffer larger than it is, events have likely been missed")) + break + } + } + + if err := w.startRead(watch); err != nil { + w.sendError(err) + } + } +} + +func (w *readDirChangesW) toWindowsFlags(mask uint64) uint32 { + var m uint32 + if mask&sysFSMODIFY != 0 { + m |= windows.FILE_NOTIFY_CHANGE_LAST_WRITE + } + if mask&(sysFSMOVE|sysFSCREATE|sysFSDELETE) != 0 { + m |= windows.FILE_NOTIFY_CHANGE_FILE_NAME | windows.FILE_NOTIFY_CHANGE_DIR_NAME + } + return m +} + +func (w *readDirChangesW) toFSnotifyFlags(action uint32) uint64 { + switch action { + case windows.FILE_ACTION_ADDED: + return sysFSCREATE + case windows.FILE_ACTION_REMOVED: + return sysFSDELETE + case windows.FILE_ACTION_MODIFIED: + return sysFSMODIFY + case windows.FILE_ACTION_RENAMED_OLD_NAME: + return sysFSMOVEDFROM + case windows.FILE_ACTION_RENAMED_NEW_NAME: + return sysFSMOVEDTO + } + return 0 +} + +func (w *readDirChangesW) xSupports(op Op) bool { + if op.Has(xUnportableOpen) || op.Has(xUnportableRead) || + op.Has(xUnportableCloseWrite) || op.Has(xUnportableCloseRead) { + return false + } + return true +} diff --git a/vendor/github.com/fsnotify/fsnotify/fen.go b/vendor/github.com/fsnotify/fsnotify/fen.go deleted file mode 100644 index b3ac3d8f5..000000000 --- a/vendor/github.com/fsnotify/fsnotify/fen.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build solaris -// +build solaris - -package fsnotify - -import ( - "errors" -) - -// Watcher watches a set of files, delivering events to a channel. -type Watcher struct { - Events chan Event - Errors chan error -} - -// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. -func NewWatcher() (*Watcher, error) { - return nil, errors.New("FEN based watcher not yet supported for fsnotify\n") -} - -// Close removes all watches and closes the events channel. -func (w *Watcher) Close() error { - return nil -} - -// Add starts watching the named file or directory (non-recursively). -func (w *Watcher) Add(name string) error { - return nil -} - -// Remove stops watching the the named file or directory (non-recursively). -func (w *Watcher) Remove(name string) error { - return nil -} diff --git a/vendor/github.com/fsnotify/fsnotify/fsnotify.go b/vendor/github.com/fsnotify/fsnotify/fsnotify.go index 0f4ee52e8..0760efe91 100644 --- a/vendor/github.com/fsnotify/fsnotify/fsnotify.go +++ b/vendor/github.com/fsnotify/fsnotify/fsnotify.go @@ -1,69 +1,494 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !plan9 -// +build !plan9 - -// Package fsnotify provides a platform-independent interface for file system notifications. +// Package fsnotify provides a cross-platform interface for file system +// notifications. +// +// Currently supported systems: +// +// - Linux via inotify +// - BSD, macOS via kqueue +// - Windows via ReadDirectoryChangesW +// - illumos via FEN +// +// # FSNOTIFY_DEBUG +// +// Set the FSNOTIFY_DEBUG environment variable to "1" to print debug messages to +// stderr. This can be useful to track down some problems, especially in cases +// where fsnotify is used as an indirect dependency. +// +// Every event will be printed as soon as there's something useful to print, +// with as little processing from fsnotify. +// +// Example output: +// +// FSNOTIFY_DEBUG: 11:34:23.633087586 256:IN_CREATE → "/tmp/file-1" +// FSNOTIFY_DEBUG: 11:34:23.633202319 4:IN_ATTRIB → "/tmp/file-1" +// FSNOTIFY_DEBUG: 11:34:28.989728764 512:IN_DELETE → "/tmp/file-1" package fsnotify import ( - "bytes" "errors" "fmt" + "os" + "path/filepath" + "strings" ) -// Event represents a single file system notification. +// Watcher watches a set of paths, delivering events on a channel. +// +// A watcher should not be copied (e.g. pass it by pointer, rather than by +// value). +// +// # Linux notes +// +// When a file is removed a Remove event won't be emitted until all file +// descriptors are closed, and deletes will always emit a Chmod. For example: +// +// fp := os.Open("file") +// os.Remove("file") // Triggers Chmod +// fp.Close() // Triggers Remove +// +// This is the event that inotify sends, so not much can be changed about this. +// +// The fs.inotify.max_user_watches sysctl variable specifies the upper limit +// for the number of watches per user, and fs.inotify.max_user_instances +// specifies the maximum number of inotify instances per user. Every Watcher you +// create is an "instance", and every path you add is a "watch". +// +// These are also exposed in /proc as /proc/sys/fs/inotify/max_user_watches and +// /proc/sys/fs/inotify/max_user_instances +// +// To increase them you can use sysctl or write the value to the /proc file: +// +// # Default values on Linux 5.18 +// sysctl fs.inotify.max_user_watches=124983 +// sysctl fs.inotify.max_user_instances=128 +// +// To make the changes persist on reboot edit /etc/sysctl.conf or +// /usr/lib/sysctl.d/50-default.conf (details differ per Linux distro; check +// your distro's documentation): +// +// fs.inotify.max_user_watches=124983 +// fs.inotify.max_user_instances=128 +// +// Reaching the limit will result in a "no space left on device" or "too many open +// files" error. +// +// # kqueue notes (macOS, BSD) +// +// kqueue requires opening a file descriptor for every file that's being watched; +// so if you're watching a directory with five files then that's six file +// descriptors. You will run in to your system's "max open files" limit faster on +// these platforms. +// +// The sysctl variables kern.maxfiles and kern.maxfilesperproc can be used to +// control the maximum number of open files, as well as /etc/login.conf on BSD +// systems. +// +// # Windows notes +// +// Paths can be added as "C:\\path\\to\\dir", but forward slashes +// ("C:/path/to/dir") will also work. +// +// When a watched directory is removed it will always send an event for the +// directory itself, but may not send events for all files in that directory. +// Sometimes it will send events for all files, sometimes it will send no +// events, and often only for some files. +// +// The default ReadDirectoryChangesW() buffer size is 64K, which is the largest +// value that is guaranteed to work with SMB filesystems. If you have many +// events in quick succession this may not be enough, and you will have to use +// [WithBufferSize] to increase the value. +type Watcher struct { + b backend + + // Events sends the filesystem change events. + // + // fsnotify can send the following events; a "path" here can refer to a + // file, directory, symbolic link, or special file like a FIFO. + // + // fsnotify.Create A new path was created; this may be followed by one + // or more Write events if data also gets written to a + // file. + // + // fsnotify.Remove A path was removed. + // + // fsnotify.Rename A path was renamed. A rename is always sent with the + // old path as Event.Name, and a Create event will be + // sent with the new name. Renames are only sent for + // paths that are currently watched; e.g. moving an + // unmonitored file into a monitored directory will + // show up as just a Create. Similarly, renaming a file + // to outside a monitored directory will show up as + // only a Rename. + // + // fsnotify.Write A file or named pipe was written to. A Truncate will + // also trigger a Write. A single "write action" + // initiated by the user may show up as one or multiple + // writes, depending on when the system syncs things to + // disk. For example when compiling a large Go program + // you may get hundreds of Write events, and you may + // want to wait until you've stopped receiving them + // (see the dedup example in cmd/fsnotify). + // + // Some systems may send Write event for directories + // when the directory content changes. + // + // fsnotify.Chmod Attributes were changed. On Linux this is also sent + // when a file is removed (or more accurately, when a + // link to an inode is removed). On kqueue it's sent + // when a file is truncated. On Windows it's never + // sent. + Events chan Event + + // Errors sends any errors. + Errors chan error +} + +// Event represents a file system notification. type Event struct { - Name string // Relative path to the file or directory. - Op Op // File operation that triggered the event. + // Path to the file or directory. + // + // Paths are relative to the input; for example with Add("dir") the Name + // will be set to "dir/file" if you create that file, but if you use + // Add("/path/to/dir") it will be "/path/to/dir/file". + Name string + + // File operation that triggered the event. + // + // This is a bitmask and some systems may send multiple operations at once. + // Use the Event.Has() method instead of comparing with ==. + Op Op + + // Create events will have this set to the old path if it's a rename. This + // only works when both the source and destination are watched. It's not + // reliable when watching individual files, only directories. + // + // For example "mv /tmp/file /tmp/rename" will emit: + // + // Event{Op: Rename, Name: "/tmp/file"} + // Event{Op: Create, Name: "/tmp/rename", RenamedFrom: "/tmp/file"} + renamedFrom string } // Op describes a set of file operations. type Op uint32 -// These are the generalized file operations that can trigger a notification. +// The operations fsnotify can trigger; see the documentation on [Watcher] for a +// full description, and check them with [Event.Has]. const ( + // A new pathname was created. Create Op = 1 << iota + + // The pathname was written to; this does *not* mean the write has finished, + // and a write can be followed by more writes. Write + + // The path was removed; any watches on it will be removed. Some "remove" + // operations may trigger a Rename if the file is actually moved (for + // example "remove to trash" is often a rename). Remove + + // The path was renamed to something else; any watches on it will be + // removed. Rename + + // File attributes were changed. + // + // It's generally not recommended to take action on this event, as it may + // get triggered very frequently by some software. For example, Spotlight + // indexing on macOS, anti-virus software, backup software, etc. Chmod + + // File descriptor was opened. + // + // Only works on Linux and FreeBSD. + xUnportableOpen + + // File was read from. + // + // Only works on Linux and FreeBSD. + xUnportableRead + + // File opened for writing was closed. + // + // Only works on Linux and FreeBSD. + // + // The advantage of using this over Write is that it's more reliable than + // waiting for Write events to stop. It's also faster (if you're not + // listening to Write events): copying a file of a few GB can easily + // generate tens of thousands of Write events in a short span of time. + xUnportableCloseWrite + + // File opened for reading was closed. + // + // Only works on Linux and FreeBSD. + xUnportableCloseRead ) -func (op Op) String() string { - // Use a buffer for efficient string concatenation - var buffer bytes.Buffer +var ( + // ErrNonExistentWatch is used when Remove() is called on a path that's not + // added. + ErrNonExistentWatch = errors.New("fsnotify: can't remove non-existent watch") + + // ErrClosed is used when trying to operate on a closed Watcher. + ErrClosed = errors.New("fsnotify: watcher already closed") + + // ErrEventOverflow is reported from the Errors channel when there are too + // many events: + // + // - inotify: inotify returns IN_Q_OVERFLOW – because there are too + // many queued events (the fs.inotify.max_queued_events + // sysctl can be used to increase this). + // - windows: The buffer size is too small; WithBufferSize() can be used to increase it. + // - kqueue, fen: Not used. + ErrEventOverflow = errors.New("fsnotify: queue or buffer overflow") + + // ErrUnsupported is returned by AddWith() when WithOps() specified an + // Unportable event that's not supported on this platform. + xErrUnsupported = errors.New("fsnotify: not supported with this backend") +) + +// NewWatcher creates a new Watcher. +func NewWatcher() (*Watcher, error) { + ev, errs := make(chan Event), make(chan error) + b, err := newBackend(ev, errs) + if err != nil { + return nil, err + } + return &Watcher{b: b, Events: ev, Errors: errs}, nil +} + +// NewBufferedWatcher creates a new Watcher with a buffered Watcher.Events +// channel. +// +// The main use case for this is situations with a very large number of events +// where the kernel buffer size can't be increased (e.g. due to lack of +// permissions). An unbuffered Watcher will perform better for almost all use +// cases, and whenever possible you will be better off increasing the kernel +// buffers instead of adding a large userspace buffer. +func NewBufferedWatcher(sz uint) (*Watcher, error) { + ev, errs := make(chan Event), make(chan error) + b, err := newBufferedBackend(sz, ev, errs) + if err != nil { + return nil, err + } + return &Watcher{b: b, Events: ev, Errors: errs}, nil +} + +// Add starts monitoring the path for changes. +// +// A path can only be watched once; watching it more than once is a no-op and will +// not return an error. Paths that do not yet exist on the filesystem cannot be +// watched. +// +// A watch will be automatically removed if the watched path is deleted or +// renamed. The exception is the Windows backend, which doesn't remove the +// watcher on renames. +// +// Notifications on network filesystems (NFS, SMB, FUSE, etc.) or special +// filesystems (/proc, /sys, etc.) generally don't work. +// +// Returns [ErrClosed] if [Watcher.Close] was called. +// +// See [Watcher.AddWith] for a version that allows adding options. +// +// # Watching directories +// +// All files in a directory are monitored, including new files that are created +// after the watcher is started. Subdirectories are not watched (i.e. it's +// non-recursive). +// +// # Watching files +// +// Watching individual files (rather than directories) is generally not +// recommended as many programs (especially editors) update files atomically: it +// will write to a temporary file which is then moved to destination, +// overwriting the original (or some variant thereof). The watcher on the +// original file is now lost, as that no longer exists. +// +// The upshot of this is that a power failure or crash won't leave a +// half-written file. +// +// Watch the parent directory and use Event.Name to filter out files you're not +// interested in. There is an example of this in cmd/fsnotify/file.go. +func (w *Watcher) Add(path string) error { return w.b.Add(path) } + +// AddWith is like [Watcher.Add], but allows adding options. When using Add() +// the defaults described below are used. +// +// Possible options are: +// +// - [WithBufferSize] sets the buffer size for the Windows backend; no-op on +// other platforms. The default is 64K (65536 bytes). +func (w *Watcher) AddWith(path string, opts ...addOpt) error { return w.b.AddWith(path, opts...) } + +// Remove stops monitoring the path for changes. +// +// Directories are always removed non-recursively. For example, if you added +// /tmp/dir and /tmp/dir/subdir then you will need to remove both. +// +// Removing a path that has not yet been added returns [ErrNonExistentWatch]. +// +// Returns nil if [Watcher.Close] was called. +func (w *Watcher) Remove(path string) error { return w.b.Remove(path) } + +// Close removes all watches and closes the Events channel. +func (w *Watcher) Close() error { return w.b.Close() } - if op&Create == Create { - buffer.WriteString("|CREATE") +// WatchList returns all paths explicitly added with [Watcher.Add] (and are not +// yet removed). +// +// Returns nil if [Watcher.Close] was called. +func (w *Watcher) WatchList() []string { return w.b.WatchList() } + +// Supports reports if all the listed operations are supported by this platform. +// +// Create, Write, Remove, Rename, and Chmod are always supported. It can only +// return false for an Op starting with Unportable. +func (w *Watcher) xSupports(op Op) bool { return w.b.xSupports(op) } + +func (o Op) String() string { + var b strings.Builder + if o.Has(Create) { + b.WriteString("|CREATE") + } + if o.Has(Remove) { + b.WriteString("|REMOVE") + } + if o.Has(Write) { + b.WriteString("|WRITE") } - if op&Remove == Remove { - buffer.WriteString("|REMOVE") + if o.Has(xUnportableOpen) { + b.WriteString("|OPEN") } - if op&Write == Write { - buffer.WriteString("|WRITE") + if o.Has(xUnportableRead) { + b.WriteString("|READ") } - if op&Rename == Rename { - buffer.WriteString("|RENAME") + if o.Has(xUnportableCloseWrite) { + b.WriteString("|CLOSE_WRITE") } - if op&Chmod == Chmod { - buffer.WriteString("|CHMOD") + if o.Has(xUnportableCloseRead) { + b.WriteString("|CLOSE_READ") } - if buffer.Len() == 0 { - return "" + if o.Has(Rename) { + b.WriteString("|RENAME") } - return buffer.String()[1:] // Strip leading pipe + if o.Has(Chmod) { + b.WriteString("|CHMOD") + } + if b.Len() == 0 { + return "[no events]" + } + return b.String()[1:] } -// String returns a string representation of the event in the form -// "file: REMOVE|WRITE|..." +// Has reports if this operation has the given operation. +func (o Op) Has(h Op) bool { return o&h != 0 } + +// Has reports if this event has the given operation. +func (e Event) Has(op Op) bool { return e.Op.Has(op) } + +// String returns a string representation of the event with their path. func (e Event) String() string { - return fmt.Sprintf("%q: %s", e.Name, e.Op.String()) + if e.renamedFrom != "" { + return fmt.Sprintf("%-13s %q ← %q", e.Op.String(), e.Name, e.renamedFrom) + } + return fmt.Sprintf("%-13s %q", e.Op.String(), e.Name) } -// Common errors that can be reported by a watcher -var ( - ErrEventOverflow = errors.New("fsnotify queue overflow") +type ( + backend interface { + Add(string) error + AddWith(string, ...addOpt) error + Remove(string) error + WatchList() []string + Close() error + xSupports(Op) bool + } + addOpt func(opt *withOpts) + withOpts struct { + bufsize int + op Op + noFollow bool + sendCreate bool + } ) + +var debug = func() bool { + // Check for exactly "1" (rather than mere existence) so we can add + // options/flags in the future. I don't know if we ever want that, but it's + // nice to leave the option open. + return os.Getenv("FSNOTIFY_DEBUG") == "1" +}() + +var defaultOpts = withOpts{ + bufsize: 65536, // 64K + op: Create | Write | Remove | Rename | Chmod, +} + +func getOptions(opts ...addOpt) withOpts { + with := defaultOpts + for _, o := range opts { + if o != nil { + o(&with) + } + } + return with +} + +// WithBufferSize sets the [ReadDirectoryChangesW] buffer size. +// +// This only has effect on Windows systems, and is a no-op for other backends. +// +// The default value is 64K (65536 bytes) which is the highest value that works +// on all filesystems and should be enough for most applications, but if you +// have a large burst of events it may not be enough. You can increase it if +// you're hitting "queue or buffer overflow" errors ([ErrEventOverflow]). +// +// [ReadDirectoryChangesW]: https://learn.microsoft.com/en-gb/windows/win32/api/winbase/nf-winbase-readdirectorychangesw +func WithBufferSize(bytes int) addOpt { + return func(opt *withOpts) { opt.bufsize = bytes } +} + +// WithOps sets which operations to listen for. The default is [Create], +// [Write], [Remove], [Rename], and [Chmod]. +// +// Excluding operations you're not interested in can save quite a bit of CPU +// time; in some use cases there may be hundreds of thousands of useless Write +// or Chmod operations per second. +// +// This can also be used to add unportable operations not supported by all +// platforms; unportable operations all start with "Unportable": +// [UnportableOpen], [UnportableRead], [UnportableCloseWrite], and +// [UnportableCloseRead]. +// +// AddWith returns an error when using an unportable operation that's not +// supported. Use [Watcher.Support] to check for support. +func withOps(op Op) addOpt { + return func(opt *withOpts) { opt.op = op } +} + +// WithNoFollow disables following symlinks, so the symlinks themselves are +// watched. +func withNoFollow() addOpt { + return func(opt *withOpts) { opt.noFollow = true } +} + +// "Internal" option for recursive watches on inotify. +func withCreate() addOpt { + return func(opt *withOpts) { opt.sendCreate = true } +} + +var enableRecurse = false + +// Check if this path is recursive (ends with "/..." or "\..."), and return the +// path with the /... stripped. +func recursivePath(path string) (string, bool) { + path = filepath.Clean(path) + if !enableRecurse { // Only enabled in tests for now. + return path, false + } + if filepath.Base(path) == "..." { + return filepath.Dir(path), true + } + return path, false +} diff --git a/vendor/github.com/fsnotify/fsnotify/fsnotify_unsupported.go b/vendor/github.com/fsnotify/fsnotify/fsnotify_unsupported.go deleted file mode 100644 index 596885598..000000000 --- a/vendor/github.com/fsnotify/fsnotify/fsnotify_unsupported.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !darwin && !dragonfly && !freebsd && !openbsd && !linux && !netbsd && !solaris && !windows -// +build !darwin,!dragonfly,!freebsd,!openbsd,!linux,!netbsd,!solaris,!windows - -package fsnotify - -import ( - "fmt" - "runtime" -) - -// Watcher watches a set of files, delivering events to a channel. -type Watcher struct{} - -// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. -func NewWatcher() (*Watcher, error) { - return nil, fmt.Errorf("fsnotify not supported on %s", runtime.GOOS) -} - -// Close removes all watches and closes the events channel. -func (w *Watcher) Close() error { - return nil -} - -// Add starts watching the named file or directory (non-recursively). -func (w *Watcher) Add(name string) error { - return nil -} - -// Remove stops watching the the named file or directory (non-recursively). -func (w *Watcher) Remove(name string) error { - return nil -} diff --git a/vendor/github.com/fsnotify/fsnotify/inotify.go b/vendor/github.com/fsnotify/fsnotify/inotify.go deleted file mode 100644 index a6d0e0ec8..000000000 --- a/vendor/github.com/fsnotify/fsnotify/inotify.go +++ /dev/null @@ -1,351 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build linux -// +build linux - -package fsnotify - -import ( - "errors" - "fmt" - "io" - "os" - "path/filepath" - "strings" - "sync" - "unsafe" - - "golang.org/x/sys/unix" -) - -// Watcher watches a set of files, delivering events to a channel. -type Watcher struct { - Events chan Event - Errors chan error - mu sync.Mutex // Map access - fd int - poller *fdPoller - watches map[string]*watch // Map of inotify watches (key: path) - paths map[int]string // Map of watched paths (key: watch descriptor) - done chan struct{} // Channel for sending a "quit message" to the reader goroutine - doneResp chan struct{} // Channel to respond to Close -} - -// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. -func NewWatcher() (*Watcher, error) { - // Create inotify fd - fd, errno := unix.InotifyInit1(unix.IN_CLOEXEC) - if fd == -1 { - return nil, errno - } - // Create epoll - poller, err := newFdPoller(fd) - if err != nil { - unix.Close(fd) - return nil, err - } - w := &Watcher{ - fd: fd, - poller: poller, - watches: make(map[string]*watch), - paths: make(map[int]string), - Events: make(chan Event), - Errors: make(chan error), - done: make(chan struct{}), - doneResp: make(chan struct{}), - } - - go w.readEvents() - return w, nil -} - -func (w *Watcher) isClosed() bool { - select { - case <-w.done: - return true - default: - return false - } -} - -// Close removes all watches and closes the events channel. -func (w *Watcher) Close() error { - if w.isClosed() { - return nil - } - - // Send 'close' signal to goroutine, and set the Watcher to closed. - close(w.done) - - // Wake up goroutine - w.poller.wake() - - // Wait for goroutine to close - <-w.doneResp - - return nil -} - -// Add starts watching the named file or directory (non-recursively). -func (w *Watcher) Add(name string) error { - name = filepath.Clean(name) - if w.isClosed() { - return errors.New("inotify instance already closed") - } - - const agnosticEvents = unix.IN_MOVED_TO | unix.IN_MOVED_FROM | - unix.IN_CREATE | unix.IN_ATTRIB | unix.IN_MODIFY | - unix.IN_MOVE_SELF | unix.IN_DELETE | unix.IN_DELETE_SELF - - var flags uint32 = agnosticEvents - - w.mu.Lock() - defer w.mu.Unlock() - watchEntry := w.watches[name] - if watchEntry != nil { - flags |= watchEntry.flags | unix.IN_MASK_ADD - } - wd, errno := unix.InotifyAddWatch(w.fd, name, flags) - if wd == -1 { - return errno - } - - if watchEntry == nil { - w.watches[name] = &watch{wd: uint32(wd), flags: flags} - w.paths[wd] = name - } else { - watchEntry.wd = uint32(wd) - watchEntry.flags = flags - } - - return nil -} - -// Remove stops watching the named file or directory (non-recursively). -func (w *Watcher) Remove(name string) error { - name = filepath.Clean(name) - - // Fetch the watch. - w.mu.Lock() - defer w.mu.Unlock() - watch, ok := w.watches[name] - - // Remove it from inotify. - if !ok { - return fmt.Errorf("can't remove non-existent inotify watch for: %s", name) - } - - // We successfully removed the watch if InotifyRmWatch doesn't return an - // error, we need to clean up our internal state to ensure it matches - // inotify's kernel state. - delete(w.paths, int(watch.wd)) - delete(w.watches, name) - - // inotify_rm_watch will return EINVAL if the file has been deleted; - // the inotify will already have been removed. - // watches and pathes are deleted in ignoreLinux() implicitly and asynchronously - // by calling inotify_rm_watch() below. e.g. readEvents() goroutine receives IN_IGNORE - // so that EINVAL means that the wd is being rm_watch()ed or its file removed - // by another thread and we have not received IN_IGNORE event. - success, errno := unix.InotifyRmWatch(w.fd, watch.wd) - if success == -1 { - // TODO: Perhaps it's not helpful to return an error here in every case. - // the only two possible errors are: - // EBADF, which happens when w.fd is not a valid file descriptor of any kind. - // EINVAL, which is when fd is not an inotify descriptor or wd is not a valid watch descriptor. - // Watch descriptors are invalidated when they are removed explicitly or implicitly; - // explicitly by inotify_rm_watch, implicitly when the file they are watching is deleted. - return errno - } - - return nil -} - -// WatchList returns the directories and files that are being monitered. -func (w *Watcher) WatchList() []string { - w.mu.Lock() - defer w.mu.Unlock() - - entries := make([]string, 0, len(w.watches)) - for pathname := range w.watches { - entries = append(entries, pathname) - } - - return entries -} - -type watch struct { - wd uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall) - flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags) -} - -// readEvents reads from the inotify file descriptor, converts the -// received events into Event objects and sends them via the Events channel -func (w *Watcher) readEvents() { - var ( - buf [unix.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events - n int // Number of bytes read with read() - errno error // Syscall errno - ok bool // For poller.wait - ) - - defer close(w.doneResp) - defer close(w.Errors) - defer close(w.Events) - defer unix.Close(w.fd) - defer w.poller.close() - - for { - // See if we have been closed. - if w.isClosed() { - return - } - - ok, errno = w.poller.wait() - if errno != nil { - select { - case w.Errors <- errno: - case <-w.done: - return - } - continue - } - - if !ok { - continue - } - - n, errno = unix.Read(w.fd, buf[:]) - // If a signal interrupted execution, see if we've been asked to close, and try again. - // http://man7.org/linux/man-pages/man7/signal.7.html : - // "Before Linux 3.8, reads from an inotify(7) file descriptor were not restartable" - if errno == unix.EINTR { - continue - } - - // unix.Read might have been woken up by Close. If so, we're done. - if w.isClosed() { - return - } - - if n < unix.SizeofInotifyEvent { - var err error - if n == 0 { - // If EOF is received. This should really never happen. - err = io.EOF - } else if n < 0 { - // If an error occurred while reading. - err = errno - } else { - // Read was too short. - err = errors.New("notify: short read in readEvents()") - } - select { - case w.Errors <- err: - case <-w.done: - return - } - continue - } - - var offset uint32 - // We don't know how many events we just read into the buffer - // While the offset points to at least one whole event... - for offset <= uint32(n-unix.SizeofInotifyEvent) { - // Point "raw" to the event in the buffer - raw := (*unix.InotifyEvent)(unsafe.Pointer(&buf[offset])) - - mask := uint32(raw.Mask) - nameLen := uint32(raw.Len) - - if mask&unix.IN_Q_OVERFLOW != 0 { - select { - case w.Errors <- ErrEventOverflow: - case <-w.done: - return - } - } - - // If the event happened to the watched directory or the watched file, the kernel - // doesn't append the filename to the event, but we would like to always fill the - // the "Name" field with a valid filename. We retrieve the path of the watch from - // the "paths" map. - w.mu.Lock() - name, ok := w.paths[int(raw.Wd)] - // IN_DELETE_SELF occurs when the file/directory being watched is removed. - // This is a sign to clean up the maps, otherwise we are no longer in sync - // with the inotify kernel state which has already deleted the watch - // automatically. - if ok && mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF { - delete(w.paths, int(raw.Wd)) - delete(w.watches, name) - } - w.mu.Unlock() - - if nameLen > 0 { - // Point "bytes" at the first byte of the filename - bytes := (*[unix.PathMax]byte)(unsafe.Pointer(&buf[offset+unix.SizeofInotifyEvent]))[:nameLen:nameLen] - // The filename is padded with NULL bytes. TrimRight() gets rid of those. - name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000") - } - - event := newEvent(name, mask) - - // Send the events that are not ignored on the events channel - if !event.ignoreLinux(mask) { - select { - case w.Events <- event: - case <-w.done: - return - } - } - - // Move to the next event in the buffer - offset += unix.SizeofInotifyEvent + nameLen - } - } -} - -// Certain types of events can be "ignored" and not sent over the Events -// channel. Such as events marked ignore by the kernel, or MODIFY events -// against files that do not exist. -func (e *Event) ignoreLinux(mask uint32) bool { - // Ignore anything the inotify API says to ignore - if mask&unix.IN_IGNORED == unix.IN_IGNORED { - return true - } - - // If the event is not a DELETE or RENAME, the file must exist. - // Otherwise the event is ignored. - // *Note*: this was put in place because it was seen that a MODIFY - // event was sent after the DELETE. This ignores that MODIFY and - // assumes a DELETE will come or has come if the file doesn't exist. - if !(e.Op&Remove == Remove || e.Op&Rename == Rename) { - _, statErr := os.Lstat(e.Name) - return os.IsNotExist(statErr) - } - return false -} - -// newEvent returns an platform-independent Event based on an inotify mask. -func newEvent(name string, mask uint32) Event { - e := Event{Name: name} - if mask&unix.IN_CREATE == unix.IN_CREATE || mask&unix.IN_MOVED_TO == unix.IN_MOVED_TO { - e.Op |= Create - } - if mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF || mask&unix.IN_DELETE == unix.IN_DELETE { - e.Op |= Remove - } - if mask&unix.IN_MODIFY == unix.IN_MODIFY { - e.Op |= Write - } - if mask&unix.IN_MOVE_SELF == unix.IN_MOVE_SELF || mask&unix.IN_MOVED_FROM == unix.IN_MOVED_FROM { - e.Op |= Rename - } - if mask&unix.IN_ATTRIB == unix.IN_ATTRIB { - e.Op |= Chmod - } - return e -} diff --git a/vendor/github.com/fsnotify/fsnotify/inotify_poller.go b/vendor/github.com/fsnotify/fsnotify/inotify_poller.go deleted file mode 100644 index b572a37c3..000000000 --- a/vendor/github.com/fsnotify/fsnotify/inotify_poller.go +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build linux -// +build linux - -package fsnotify - -import ( - "errors" - - "golang.org/x/sys/unix" -) - -type fdPoller struct { - fd int // File descriptor (as returned by the inotify_init() syscall) - epfd int // Epoll file descriptor - pipe [2]int // Pipe for waking up -} - -func emptyPoller(fd int) *fdPoller { - poller := new(fdPoller) - poller.fd = fd - poller.epfd = -1 - poller.pipe[0] = -1 - poller.pipe[1] = -1 - return poller -} - -// Create a new inotify poller. -// This creates an inotify handler, and an epoll handler. -func newFdPoller(fd int) (*fdPoller, error) { - var errno error - poller := emptyPoller(fd) - defer func() { - if errno != nil { - poller.close() - } - }() - - // Create epoll fd - poller.epfd, errno = unix.EpollCreate1(unix.EPOLL_CLOEXEC) - if poller.epfd == -1 { - return nil, errno - } - // Create pipe; pipe[0] is the read end, pipe[1] the write end. - errno = unix.Pipe2(poller.pipe[:], unix.O_NONBLOCK|unix.O_CLOEXEC) - if errno != nil { - return nil, errno - } - - // Register inotify fd with epoll - event := unix.EpollEvent{ - Fd: int32(poller.fd), - Events: unix.EPOLLIN, - } - errno = unix.EpollCtl(poller.epfd, unix.EPOLL_CTL_ADD, poller.fd, &event) - if errno != nil { - return nil, errno - } - - // Register pipe fd with epoll - event = unix.EpollEvent{ - Fd: int32(poller.pipe[0]), - Events: unix.EPOLLIN, - } - errno = unix.EpollCtl(poller.epfd, unix.EPOLL_CTL_ADD, poller.pipe[0], &event) - if errno != nil { - return nil, errno - } - - return poller, nil -} - -// Wait using epoll. -// Returns true if something is ready to be read, -// false if there is not. -func (poller *fdPoller) wait() (bool, error) { - // 3 possible events per fd, and 2 fds, makes a maximum of 6 events. - // I don't know whether epoll_wait returns the number of events returned, - // or the total number of events ready. - // I decided to catch both by making the buffer one larger than the maximum. - events := make([]unix.EpollEvent, 7) - for { - n, errno := unix.EpollWait(poller.epfd, events, -1) - if n == -1 { - if errno == unix.EINTR { - continue - } - return false, errno - } - if n == 0 { - // If there are no events, try again. - continue - } - if n > 6 { - // This should never happen. More events were returned than should be possible. - return false, errors.New("epoll_wait returned more events than I know what to do with") - } - ready := events[:n] - epollhup := false - epollerr := false - epollin := false - for _, event := range ready { - if event.Fd == int32(poller.fd) { - if event.Events&unix.EPOLLHUP != 0 { - // This should not happen, but if it does, treat it as a wakeup. - epollhup = true - } - if event.Events&unix.EPOLLERR != 0 { - // If an error is waiting on the file descriptor, we should pretend - // something is ready to read, and let unix.Read pick up the error. - epollerr = true - } - if event.Events&unix.EPOLLIN != 0 { - // There is data to read. - epollin = true - } - } - if event.Fd == int32(poller.pipe[0]) { - if event.Events&unix.EPOLLHUP != 0 { - // Write pipe descriptor was closed, by us. This means we're closing down the - // watcher, and we should wake up. - } - if event.Events&unix.EPOLLERR != 0 { - // If an error is waiting on the pipe file descriptor. - // This is an absolute mystery, and should never ever happen. - return false, errors.New("Error on the pipe descriptor.") - } - if event.Events&unix.EPOLLIN != 0 { - // This is a regular wakeup, so we have to clear the buffer. - err := poller.clearWake() - if err != nil { - return false, err - } - } - } - } - - if epollhup || epollerr || epollin { - return true, nil - } - return false, nil - } -} - -// Close the write end of the poller. -func (poller *fdPoller) wake() error { - buf := make([]byte, 1) - n, errno := unix.Write(poller.pipe[1], buf) - if n == -1 { - if errno == unix.EAGAIN { - // Buffer is full, poller will wake. - return nil - } - return errno - } - return nil -} - -func (poller *fdPoller) clearWake() error { - // You have to be woken up a LOT in order to get to 100! - buf := make([]byte, 100) - n, errno := unix.Read(poller.pipe[0], buf) - if n == -1 { - if errno == unix.EAGAIN { - // Buffer is empty, someone else cleared our wake. - return nil - } - return errno - } - return nil -} - -// Close all poller file descriptors, but not the one passed to it. -func (poller *fdPoller) close() { - if poller.pipe[1] != -1 { - unix.Close(poller.pipe[1]) - } - if poller.pipe[0] != -1 { - unix.Close(poller.pipe[0]) - } - if poller.epfd != -1 { - unix.Close(poller.epfd) - } -} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/darwin.go b/vendor/github.com/fsnotify/fsnotify/internal/darwin.go new file mode 100644 index 000000000..b0eab1009 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/darwin.go @@ -0,0 +1,39 @@ +//go:build darwin + +package internal + +import ( + "syscall" + + "golang.org/x/sys/unix" +) + +var ( + SyscallEACCES = syscall.EACCES + UnixEACCES = unix.EACCES +) + +var maxfiles uint64 + +// Go 1.19 will do this automatically: https://go-review.googlesource.com/c/go/+/393354/ +func SetRlimit() { + var l syscall.Rlimit + err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &l) + if err == nil && l.Cur != l.Max { + l.Cur = l.Max + syscall.Setrlimit(syscall.RLIMIT_NOFILE, &l) + } + maxfiles = l.Cur + + if n, err := syscall.SysctlUint32("kern.maxfiles"); err == nil && uint64(n) < maxfiles { + maxfiles = uint64(n) + } + + if n, err := syscall.SysctlUint32("kern.maxfilesperproc"); err == nil && uint64(n) < maxfiles { + maxfiles = uint64(n) + } +} + +func Maxfiles() uint64 { return maxfiles } +func Mkfifo(path string, mode uint32) error { return unix.Mkfifo(path, mode) } +func Mknod(path string, mode uint32, dev int) error { return unix.Mknod(path, mode, dev) } diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go new file mode 100644 index 000000000..928319fb0 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_darwin.go @@ -0,0 +1,57 @@ +package internal + +import "golang.org/x/sys/unix" + +var names = []struct { + n string + m uint32 +}{ + {"NOTE_ABSOLUTE", unix.NOTE_ABSOLUTE}, + {"NOTE_ATTRIB", unix.NOTE_ATTRIB}, + {"NOTE_BACKGROUND", unix.NOTE_BACKGROUND}, + {"NOTE_CHILD", unix.NOTE_CHILD}, + {"NOTE_CRITICAL", unix.NOTE_CRITICAL}, + {"NOTE_DELETE", unix.NOTE_DELETE}, + {"NOTE_EXEC", unix.NOTE_EXEC}, + {"NOTE_EXIT", unix.NOTE_EXIT}, + {"NOTE_EXITSTATUS", unix.NOTE_EXITSTATUS}, + {"NOTE_EXIT_CSERROR", unix.NOTE_EXIT_CSERROR}, + {"NOTE_EXIT_DECRYPTFAIL", unix.NOTE_EXIT_DECRYPTFAIL}, + {"NOTE_EXIT_DETAIL", unix.NOTE_EXIT_DETAIL}, + {"NOTE_EXIT_DETAIL_MASK", unix.NOTE_EXIT_DETAIL_MASK}, + {"NOTE_EXIT_MEMORY", unix.NOTE_EXIT_MEMORY}, + {"NOTE_EXIT_REPARENTED", unix.NOTE_EXIT_REPARENTED}, + {"NOTE_EXTEND", unix.NOTE_EXTEND}, + {"NOTE_FFAND", unix.NOTE_FFAND}, + {"NOTE_FFCOPY", unix.NOTE_FFCOPY}, + {"NOTE_FFCTRLMASK", unix.NOTE_FFCTRLMASK}, + {"NOTE_FFLAGSMASK", unix.NOTE_FFLAGSMASK}, + {"NOTE_FFNOP", unix.NOTE_FFNOP}, + {"NOTE_FFOR", unix.NOTE_FFOR}, + {"NOTE_FORK", unix.NOTE_FORK}, + {"NOTE_FUNLOCK", unix.NOTE_FUNLOCK}, + {"NOTE_LEEWAY", unix.NOTE_LEEWAY}, + {"NOTE_LINK", unix.NOTE_LINK}, + {"NOTE_LOWAT", unix.NOTE_LOWAT}, + {"NOTE_MACHTIME", unix.NOTE_MACHTIME}, + {"NOTE_MACH_CONTINUOUS_TIME", unix.NOTE_MACH_CONTINUOUS_TIME}, + {"NOTE_NONE", unix.NOTE_NONE}, + {"NOTE_NSECONDS", unix.NOTE_NSECONDS}, + {"NOTE_OOB", unix.NOTE_OOB}, + //{"NOTE_PCTRLMASK", unix.NOTE_PCTRLMASK}, -0x100000 (?!) + {"NOTE_PDATAMASK", unix.NOTE_PDATAMASK}, + {"NOTE_REAP", unix.NOTE_REAP}, + {"NOTE_RENAME", unix.NOTE_RENAME}, + {"NOTE_REVOKE", unix.NOTE_REVOKE}, + {"NOTE_SECONDS", unix.NOTE_SECONDS}, + {"NOTE_SIGNAL", unix.NOTE_SIGNAL}, + {"NOTE_TRACK", unix.NOTE_TRACK}, + {"NOTE_TRACKERR", unix.NOTE_TRACKERR}, + {"NOTE_TRIGGER", unix.NOTE_TRIGGER}, + {"NOTE_USECONDS", unix.NOTE_USECONDS}, + {"NOTE_VM_ERROR", unix.NOTE_VM_ERROR}, + {"NOTE_VM_PRESSURE", unix.NOTE_VM_PRESSURE}, + {"NOTE_VM_PRESSURE_SUDDEN_TERMINATE", unix.NOTE_VM_PRESSURE_SUDDEN_TERMINATE}, + {"NOTE_VM_PRESSURE_TERMINATE", unix.NOTE_VM_PRESSURE_TERMINATE}, + {"NOTE_WRITE", unix.NOTE_WRITE}, +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go new file mode 100644 index 000000000..3186b0c34 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_dragonfly.go @@ -0,0 +1,33 @@ +package internal + +import "golang.org/x/sys/unix" + +var names = []struct { + n string + m uint32 +}{ + {"NOTE_ATTRIB", unix.NOTE_ATTRIB}, + {"NOTE_CHILD", unix.NOTE_CHILD}, + {"NOTE_DELETE", unix.NOTE_DELETE}, + {"NOTE_EXEC", unix.NOTE_EXEC}, + {"NOTE_EXIT", unix.NOTE_EXIT}, + {"NOTE_EXTEND", unix.NOTE_EXTEND}, + {"NOTE_FFAND", unix.NOTE_FFAND}, + {"NOTE_FFCOPY", unix.NOTE_FFCOPY}, + {"NOTE_FFCTRLMASK", unix.NOTE_FFCTRLMASK}, + {"NOTE_FFLAGSMASK", unix.NOTE_FFLAGSMASK}, + {"NOTE_FFNOP", unix.NOTE_FFNOP}, + {"NOTE_FFOR", unix.NOTE_FFOR}, + {"NOTE_FORK", unix.NOTE_FORK}, + {"NOTE_LINK", unix.NOTE_LINK}, + {"NOTE_LOWAT", unix.NOTE_LOWAT}, + {"NOTE_OOB", unix.NOTE_OOB}, + {"NOTE_PCTRLMASK", unix.NOTE_PCTRLMASK}, + {"NOTE_PDATAMASK", unix.NOTE_PDATAMASK}, + {"NOTE_RENAME", unix.NOTE_RENAME}, + {"NOTE_REVOKE", unix.NOTE_REVOKE}, + {"NOTE_TRACK", unix.NOTE_TRACK}, + {"NOTE_TRACKERR", unix.NOTE_TRACKERR}, + {"NOTE_TRIGGER", unix.NOTE_TRIGGER}, + {"NOTE_WRITE", unix.NOTE_WRITE}, +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go new file mode 100644 index 000000000..f69fdb930 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_freebsd.go @@ -0,0 +1,42 @@ +package internal + +import "golang.org/x/sys/unix" + +var names = []struct { + n string + m uint32 +}{ + {"NOTE_ABSTIME", unix.NOTE_ABSTIME}, + {"NOTE_ATTRIB", unix.NOTE_ATTRIB}, + {"NOTE_CHILD", unix.NOTE_CHILD}, + {"NOTE_CLOSE", unix.NOTE_CLOSE}, + {"NOTE_CLOSE_WRITE", unix.NOTE_CLOSE_WRITE}, + {"NOTE_DELETE", unix.NOTE_DELETE}, + {"NOTE_EXEC", unix.NOTE_EXEC}, + {"NOTE_EXIT", unix.NOTE_EXIT}, + {"NOTE_EXTEND", unix.NOTE_EXTEND}, + {"NOTE_FFAND", unix.NOTE_FFAND}, + {"NOTE_FFCOPY", unix.NOTE_FFCOPY}, + {"NOTE_FFCTRLMASK", unix.NOTE_FFCTRLMASK}, + {"NOTE_FFLAGSMASK", unix.NOTE_FFLAGSMASK}, + {"NOTE_FFNOP", unix.NOTE_FFNOP}, + {"NOTE_FFOR", unix.NOTE_FFOR}, + {"NOTE_FILE_POLL", unix.NOTE_FILE_POLL}, + {"NOTE_FORK", unix.NOTE_FORK}, + {"NOTE_LINK", unix.NOTE_LINK}, + {"NOTE_LOWAT", unix.NOTE_LOWAT}, + {"NOTE_MSECONDS", unix.NOTE_MSECONDS}, + {"NOTE_NSECONDS", unix.NOTE_NSECONDS}, + {"NOTE_OPEN", unix.NOTE_OPEN}, + {"NOTE_PCTRLMASK", unix.NOTE_PCTRLMASK}, + {"NOTE_PDATAMASK", unix.NOTE_PDATAMASK}, + {"NOTE_READ", unix.NOTE_READ}, + {"NOTE_RENAME", unix.NOTE_RENAME}, + {"NOTE_REVOKE", unix.NOTE_REVOKE}, + {"NOTE_SECONDS", unix.NOTE_SECONDS}, + {"NOTE_TRACK", unix.NOTE_TRACK}, + {"NOTE_TRACKERR", unix.NOTE_TRACKERR}, + {"NOTE_TRIGGER", unix.NOTE_TRIGGER}, + {"NOTE_USECONDS", unix.NOTE_USECONDS}, + {"NOTE_WRITE", unix.NOTE_WRITE}, +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go new file mode 100644 index 000000000..607e683bd --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_kqueue.go @@ -0,0 +1,32 @@ +//go:build freebsd || openbsd || netbsd || dragonfly || darwin + +package internal + +import ( + "fmt" + "os" + "strings" + "time" + + "golang.org/x/sys/unix" +) + +func Debug(name string, kevent *unix.Kevent_t) { + mask := uint32(kevent.Fflags) + + var ( + l []string + unknown = mask + ) + for _, n := range names { + if mask&n.m == n.m { + l = append(l, n.n) + unknown ^= n.m + } + } + if unknown > 0 { + l = append(l, fmt.Sprintf("0x%x", unknown)) + } + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s %10d:%-60s → %q\n", + time.Now().Format("15:04:05.000000000"), mask, strings.Join(l, " | "), name) +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_linux.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_linux.go new file mode 100644 index 000000000..35c734be4 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_linux.go @@ -0,0 +1,56 @@ +package internal + +import ( + "fmt" + "os" + "strings" + "time" + + "golang.org/x/sys/unix" +) + +func Debug(name string, mask, cookie uint32) { + names := []struct { + n string + m uint32 + }{ + {"IN_ACCESS", unix.IN_ACCESS}, + {"IN_ATTRIB", unix.IN_ATTRIB}, + {"IN_CLOSE", unix.IN_CLOSE}, + {"IN_CLOSE_NOWRITE", unix.IN_CLOSE_NOWRITE}, + {"IN_CLOSE_WRITE", unix.IN_CLOSE_WRITE}, + {"IN_CREATE", unix.IN_CREATE}, + {"IN_DELETE", unix.IN_DELETE}, + {"IN_DELETE_SELF", unix.IN_DELETE_SELF}, + {"IN_IGNORED", unix.IN_IGNORED}, + {"IN_ISDIR", unix.IN_ISDIR}, + {"IN_MODIFY", unix.IN_MODIFY}, + {"IN_MOVE", unix.IN_MOVE}, + {"IN_MOVED_FROM", unix.IN_MOVED_FROM}, + {"IN_MOVED_TO", unix.IN_MOVED_TO}, + {"IN_MOVE_SELF", unix.IN_MOVE_SELF}, + {"IN_OPEN", unix.IN_OPEN}, + {"IN_Q_OVERFLOW", unix.IN_Q_OVERFLOW}, + {"IN_UNMOUNT", unix.IN_UNMOUNT}, + } + + var ( + l []string + unknown = mask + ) + for _, n := range names { + if mask&n.m == n.m { + l = append(l, n.n) + unknown ^= n.m + } + } + if unknown > 0 { + l = append(l, fmt.Sprintf("0x%x", unknown)) + } + var c string + if cookie > 0 { + c = fmt.Sprintf("(cookie: %d) ", cookie) + } + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s %-30s → %s%q\n", + time.Now().Format("15:04:05.000000000"), strings.Join(l, "|"), c, name) +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go new file mode 100644 index 000000000..e5b3b6f69 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_netbsd.go @@ -0,0 +1,25 @@ +package internal + +import "golang.org/x/sys/unix" + +var names = []struct { + n string + m uint32 +}{ + {"NOTE_ATTRIB", unix.NOTE_ATTRIB}, + {"NOTE_CHILD", unix.NOTE_CHILD}, + {"NOTE_DELETE", unix.NOTE_DELETE}, + {"NOTE_EXEC", unix.NOTE_EXEC}, + {"NOTE_EXIT", unix.NOTE_EXIT}, + {"NOTE_EXTEND", unix.NOTE_EXTEND}, + {"NOTE_FORK", unix.NOTE_FORK}, + {"NOTE_LINK", unix.NOTE_LINK}, + {"NOTE_LOWAT", unix.NOTE_LOWAT}, + {"NOTE_PCTRLMASK", unix.NOTE_PCTRLMASK}, + {"NOTE_PDATAMASK", unix.NOTE_PDATAMASK}, + {"NOTE_RENAME", unix.NOTE_RENAME}, + {"NOTE_REVOKE", unix.NOTE_REVOKE}, + {"NOTE_TRACK", unix.NOTE_TRACK}, + {"NOTE_TRACKERR", unix.NOTE_TRACKERR}, + {"NOTE_WRITE", unix.NOTE_WRITE}, +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go new file mode 100644 index 000000000..1dd455bc5 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_openbsd.go @@ -0,0 +1,28 @@ +package internal + +import "golang.org/x/sys/unix" + +var names = []struct { + n string + m uint32 +}{ + {"NOTE_ATTRIB", unix.NOTE_ATTRIB}, + // {"NOTE_CHANGE", unix.NOTE_CHANGE}, // Not on 386? + {"NOTE_CHILD", unix.NOTE_CHILD}, + {"NOTE_DELETE", unix.NOTE_DELETE}, + {"NOTE_EOF", unix.NOTE_EOF}, + {"NOTE_EXEC", unix.NOTE_EXEC}, + {"NOTE_EXIT", unix.NOTE_EXIT}, + {"NOTE_EXTEND", unix.NOTE_EXTEND}, + {"NOTE_FORK", unix.NOTE_FORK}, + {"NOTE_LINK", unix.NOTE_LINK}, + {"NOTE_LOWAT", unix.NOTE_LOWAT}, + {"NOTE_PCTRLMASK", unix.NOTE_PCTRLMASK}, + {"NOTE_PDATAMASK", unix.NOTE_PDATAMASK}, + {"NOTE_RENAME", unix.NOTE_RENAME}, + {"NOTE_REVOKE", unix.NOTE_REVOKE}, + {"NOTE_TRACK", unix.NOTE_TRACK}, + {"NOTE_TRACKERR", unix.NOTE_TRACKERR}, + {"NOTE_TRUNCATE", unix.NOTE_TRUNCATE}, + {"NOTE_WRITE", unix.NOTE_WRITE}, +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go new file mode 100644 index 000000000..f1b2e73bd --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_solaris.go @@ -0,0 +1,45 @@ +package internal + +import ( + "fmt" + "os" + "strings" + "time" + + "golang.org/x/sys/unix" +) + +func Debug(name string, mask int32) { + names := []struct { + n string + m int32 + }{ + {"FILE_ACCESS", unix.FILE_ACCESS}, + {"FILE_MODIFIED", unix.FILE_MODIFIED}, + {"FILE_ATTRIB", unix.FILE_ATTRIB}, + {"FILE_TRUNC", unix.FILE_TRUNC}, + {"FILE_NOFOLLOW", unix.FILE_NOFOLLOW}, + {"FILE_DELETE", unix.FILE_DELETE}, + {"FILE_RENAME_TO", unix.FILE_RENAME_TO}, + {"FILE_RENAME_FROM", unix.FILE_RENAME_FROM}, + {"UNMOUNTED", unix.UNMOUNTED}, + {"MOUNTEDOVER", unix.MOUNTEDOVER}, + {"FILE_EXCEPTION", unix.FILE_EXCEPTION}, + } + + var ( + l []string + unknown = mask + ) + for _, n := range names { + if mask&n.m == n.m { + l = append(l, n.n) + unknown ^= n.m + } + } + if unknown > 0 { + l = append(l, fmt.Sprintf("0x%x", unknown)) + } + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s %10d:%-30s → %q\n", + time.Now().Format("15:04:05.000000000"), mask, strings.Join(l, " | "), name) +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/debug_windows.go b/vendor/github.com/fsnotify/fsnotify/internal/debug_windows.go new file mode 100644 index 000000000..52bf4ce53 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/debug_windows.go @@ -0,0 +1,40 @@ +package internal + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "golang.org/x/sys/windows" +) + +func Debug(name string, mask uint32) { + names := []struct { + n string + m uint32 + }{ + {"FILE_ACTION_ADDED", windows.FILE_ACTION_ADDED}, + {"FILE_ACTION_REMOVED", windows.FILE_ACTION_REMOVED}, + {"FILE_ACTION_MODIFIED", windows.FILE_ACTION_MODIFIED}, + {"FILE_ACTION_RENAMED_OLD_NAME", windows.FILE_ACTION_RENAMED_OLD_NAME}, + {"FILE_ACTION_RENAMED_NEW_NAME", windows.FILE_ACTION_RENAMED_NEW_NAME}, + } + + var ( + l []string + unknown = mask + ) + for _, n := range names { + if mask&n.m == n.m { + l = append(l, n.n) + unknown ^= n.m + } + } + if unknown > 0 { + l = append(l, fmt.Sprintf("0x%x", unknown)) + } + fmt.Fprintf(os.Stderr, "FSNOTIFY_DEBUG: %s %-65s → %q\n", + time.Now().Format("15:04:05.000000000"), strings.Join(l, " | "), filepath.ToSlash(name)) +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/freebsd.go b/vendor/github.com/fsnotify/fsnotify/internal/freebsd.go new file mode 100644 index 000000000..547df1df8 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/freebsd.go @@ -0,0 +1,31 @@ +//go:build freebsd + +package internal + +import ( + "syscall" + + "golang.org/x/sys/unix" +) + +var ( + SyscallEACCES = syscall.EACCES + UnixEACCES = unix.EACCES +) + +var maxfiles uint64 + +func SetRlimit() { + // Go 1.19 will do this automatically: https://go-review.googlesource.com/c/go/+/393354/ + var l syscall.Rlimit + err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &l) + if err == nil && l.Cur != l.Max { + l.Cur = l.Max + syscall.Setrlimit(syscall.RLIMIT_NOFILE, &l) + } + maxfiles = uint64(l.Cur) +} + +func Maxfiles() uint64 { return maxfiles } +func Mkfifo(path string, mode uint32) error { return unix.Mkfifo(path, mode) } +func Mknod(path string, mode uint32, dev int) error { return unix.Mknod(path, mode, uint64(dev)) } diff --git a/vendor/github.com/fsnotify/fsnotify/internal/internal.go b/vendor/github.com/fsnotify/fsnotify/internal/internal.go new file mode 100644 index 000000000..7daa45e19 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/internal.go @@ -0,0 +1,2 @@ +// Package internal contains some helpers. +package internal diff --git a/vendor/github.com/fsnotify/fsnotify/internal/unix.go b/vendor/github.com/fsnotify/fsnotify/internal/unix.go new file mode 100644 index 000000000..30976ce97 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/unix.go @@ -0,0 +1,31 @@ +//go:build !windows && !darwin && !freebsd + +package internal + +import ( + "syscall" + + "golang.org/x/sys/unix" +) + +var ( + SyscallEACCES = syscall.EACCES + UnixEACCES = unix.EACCES +) + +var maxfiles uint64 + +func SetRlimit() { + // Go 1.19 will do this automatically: https://go-review.googlesource.com/c/go/+/393354/ + var l syscall.Rlimit + err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &l) + if err == nil && l.Cur != l.Max { + l.Cur = l.Max + syscall.Setrlimit(syscall.RLIMIT_NOFILE, &l) + } + maxfiles = uint64(l.Cur) +} + +func Maxfiles() uint64 { return maxfiles } +func Mkfifo(path string, mode uint32) error { return unix.Mkfifo(path, mode) } +func Mknod(path string, mode uint32, dev int) error { return unix.Mknod(path, mode, dev) } diff --git a/vendor/github.com/fsnotify/fsnotify/internal/unix2.go b/vendor/github.com/fsnotify/fsnotify/internal/unix2.go new file mode 100644 index 000000000..37dfeddc2 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/unix2.go @@ -0,0 +1,7 @@ +//go:build !windows + +package internal + +func HasPrivilegesForSymlink() bool { + return true +} diff --git a/vendor/github.com/fsnotify/fsnotify/internal/windows.go b/vendor/github.com/fsnotify/fsnotify/internal/windows.go new file mode 100644 index 000000000..a72c64954 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/internal/windows.go @@ -0,0 +1,41 @@ +//go:build windows + +package internal + +import ( + "errors" + + "golang.org/x/sys/windows" +) + +// Just a dummy. +var ( + SyscallEACCES = errors.New("dummy") + UnixEACCES = errors.New("dummy") +) + +func SetRlimit() {} +func Maxfiles() uint64 { return 1<<64 - 1 } +func Mkfifo(path string, mode uint32) error { return errors.New("no FIFOs on Windows") } +func Mknod(path string, mode uint32, dev int) error { return errors.New("no device nodes on Windows") } + +func HasPrivilegesForSymlink() bool { + var sid *windows.SID + err := windows.AllocateAndInitializeSid( + &windows.SECURITY_NT_AUTHORITY, + 2, + windows.SECURITY_BUILTIN_DOMAIN_RID, + windows.DOMAIN_ALIAS_RID_ADMINS, + 0, 0, 0, 0, 0, 0, + &sid) + if err != nil { + return false + } + defer windows.FreeSid(sid) + token := windows.Token(0) + member, err := token.IsMember(sid) + if err != nil { + return false + } + return member || token.IsElevated() +} diff --git a/vendor/github.com/fsnotify/fsnotify/kqueue.go b/vendor/github.com/fsnotify/fsnotify/kqueue.go deleted file mode 100644 index 6fb8d8532..000000000 --- a/vendor/github.com/fsnotify/fsnotify/kqueue.go +++ /dev/null @@ -1,535 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build freebsd || openbsd || netbsd || dragonfly || darwin -// +build freebsd openbsd netbsd dragonfly darwin - -package fsnotify - -import ( - "errors" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "sync" - "time" - - "golang.org/x/sys/unix" -) - -// Watcher watches a set of files, delivering events to a channel. -type Watcher struct { - Events chan Event - Errors chan error - done chan struct{} // Channel for sending a "quit message" to the reader goroutine - - kq int // File descriptor (as returned by the kqueue() syscall). - - mu sync.Mutex // Protects access to watcher data - watches map[string]int // Map of watched file descriptors (key: path). - externalWatches map[string]bool // Map of watches added by user of the library. - dirFlags map[string]uint32 // Map of watched directories to fflags used in kqueue. - paths map[int]pathInfo // Map file descriptors to path names for processing kqueue events. - fileExists map[string]bool // Keep track of if we know this file exists (to stop duplicate create events). - isClosed bool // Set to true when Close() is first called -} - -type pathInfo struct { - name string - isDir bool -} - -// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. -func NewWatcher() (*Watcher, error) { - kq, err := kqueue() - if err != nil { - return nil, err - } - - w := &Watcher{ - kq: kq, - watches: make(map[string]int), - dirFlags: make(map[string]uint32), - paths: make(map[int]pathInfo), - fileExists: make(map[string]bool), - externalWatches: make(map[string]bool), - Events: make(chan Event), - Errors: make(chan error), - done: make(chan struct{}), - } - - go w.readEvents() - return w, nil -} - -// Close removes all watches and closes the events channel. -func (w *Watcher) Close() error { - w.mu.Lock() - if w.isClosed { - w.mu.Unlock() - return nil - } - w.isClosed = true - - // copy paths to remove while locked - var pathsToRemove = make([]string, 0, len(w.watches)) - for name := range w.watches { - pathsToRemove = append(pathsToRemove, name) - } - w.mu.Unlock() - // unlock before calling Remove, which also locks - - for _, name := range pathsToRemove { - w.Remove(name) - } - - // send a "quit" message to the reader goroutine - close(w.done) - - return nil -} - -// Add starts watching the named file or directory (non-recursively). -func (w *Watcher) Add(name string) error { - w.mu.Lock() - w.externalWatches[name] = true - w.mu.Unlock() - _, err := w.addWatch(name, noteAllEvents) - return err -} - -// Remove stops watching the the named file or directory (non-recursively). -func (w *Watcher) Remove(name string) error { - name = filepath.Clean(name) - w.mu.Lock() - watchfd, ok := w.watches[name] - w.mu.Unlock() - if !ok { - return fmt.Errorf("can't remove non-existent kevent watch for: %s", name) - } - - const registerRemove = unix.EV_DELETE - if err := register(w.kq, []int{watchfd}, registerRemove, 0); err != nil { - return err - } - - unix.Close(watchfd) - - w.mu.Lock() - isDir := w.paths[watchfd].isDir - delete(w.watches, name) - delete(w.paths, watchfd) - delete(w.dirFlags, name) - w.mu.Unlock() - - // Find all watched paths that are in this directory that are not external. - if isDir { - var pathsToRemove []string - w.mu.Lock() - for _, path := range w.paths { - wdir, _ := filepath.Split(path.name) - if filepath.Clean(wdir) == name { - if !w.externalWatches[path.name] { - pathsToRemove = append(pathsToRemove, path.name) - } - } - } - w.mu.Unlock() - for _, name := range pathsToRemove { - // Since these are internal, not much sense in propagating error - // to the user, as that will just confuse them with an error about - // a path they did not explicitly watch themselves. - w.Remove(name) - } - } - - return nil -} - -// WatchList returns the directories and files that are being monitered. -func (w *Watcher) WatchList() []string { - w.mu.Lock() - defer w.mu.Unlock() - - entries := make([]string, 0, len(w.watches)) - for pathname := range w.watches { - entries = append(entries, pathname) - } - - return entries -} - -// Watch all events (except NOTE_EXTEND, NOTE_LINK, NOTE_REVOKE) -const noteAllEvents = unix.NOTE_DELETE | unix.NOTE_WRITE | unix.NOTE_ATTRIB | unix.NOTE_RENAME - -// keventWaitTime to block on each read from kevent -var keventWaitTime = durationToTimespec(100 * time.Millisecond) - -// addWatch adds name to the watched file set. -// The flags are interpreted as described in kevent(2). -// Returns the real path to the file which was added, if any, which may be different from the one passed in the case of symlinks. -func (w *Watcher) addWatch(name string, flags uint32) (string, error) { - var isDir bool - // Make ./name and name equivalent - name = filepath.Clean(name) - - w.mu.Lock() - if w.isClosed { - w.mu.Unlock() - return "", errors.New("kevent instance already closed") - } - watchfd, alreadyWatching := w.watches[name] - // We already have a watch, but we can still override flags. - if alreadyWatching { - isDir = w.paths[watchfd].isDir - } - w.mu.Unlock() - - if !alreadyWatching { - fi, err := os.Lstat(name) - if err != nil { - return "", err - } - - // Don't watch sockets. - if fi.Mode()&os.ModeSocket == os.ModeSocket { - return "", nil - } - - // Don't watch named pipes. - if fi.Mode()&os.ModeNamedPipe == os.ModeNamedPipe { - return "", nil - } - - // Follow Symlinks - // Unfortunately, Linux can add bogus symlinks to watch list without - // issue, and Windows can't do symlinks period (AFAIK). To maintain - // consistency, we will act like everything is fine. There will simply - // be no file events for broken symlinks. - // Hence the returns of nil on errors. - if fi.Mode()&os.ModeSymlink == os.ModeSymlink { - name, err = filepath.EvalSymlinks(name) - if err != nil { - return "", nil - } - - w.mu.Lock() - _, alreadyWatching = w.watches[name] - w.mu.Unlock() - - if alreadyWatching { - return name, nil - } - - fi, err = os.Lstat(name) - if err != nil { - return "", nil - } - } - - watchfd, err = unix.Open(name, openMode, 0700) - if watchfd == -1 { - return "", err - } - - isDir = fi.IsDir() - } - - const registerAdd = unix.EV_ADD | unix.EV_CLEAR | unix.EV_ENABLE - if err := register(w.kq, []int{watchfd}, registerAdd, flags); err != nil { - unix.Close(watchfd) - return "", err - } - - if !alreadyWatching { - w.mu.Lock() - w.watches[name] = watchfd - w.paths[watchfd] = pathInfo{name: name, isDir: isDir} - w.mu.Unlock() - } - - if isDir { - // Watch the directory if it has not been watched before, - // or if it was watched before, but perhaps only a NOTE_DELETE (watchDirectoryFiles) - w.mu.Lock() - - watchDir := (flags&unix.NOTE_WRITE) == unix.NOTE_WRITE && - (!alreadyWatching || (w.dirFlags[name]&unix.NOTE_WRITE) != unix.NOTE_WRITE) - // Store flags so this watch can be updated later - w.dirFlags[name] = flags - w.mu.Unlock() - - if watchDir { - if err := w.watchDirectoryFiles(name); err != nil { - return "", err - } - } - } - return name, nil -} - -// readEvents reads from kqueue and converts the received kevents into -// Event values that it sends down the Events channel. -func (w *Watcher) readEvents() { - eventBuffer := make([]unix.Kevent_t, 10) - -loop: - for { - // See if there is a message on the "done" channel - select { - case <-w.done: - break loop - default: - } - - // Get new events - kevents, err := read(w.kq, eventBuffer, &keventWaitTime) - // EINTR is okay, the syscall was interrupted before timeout expired. - if err != nil && err != unix.EINTR { - select { - case w.Errors <- err: - case <-w.done: - break loop - } - continue - } - - // Flush the events we received to the Events channel - for len(kevents) > 0 { - kevent := &kevents[0] - watchfd := int(kevent.Ident) - mask := uint32(kevent.Fflags) - w.mu.Lock() - path := w.paths[watchfd] - w.mu.Unlock() - event := newEvent(path.name, mask) - - if path.isDir && !(event.Op&Remove == Remove) { - // Double check to make sure the directory exists. This can happen when - // we do a rm -fr on a recursively watched folders and we receive a - // modification event first but the folder has been deleted and later - // receive the delete event - if _, err := os.Lstat(event.Name); os.IsNotExist(err) { - // mark is as delete event - event.Op |= Remove - } - } - - if event.Op&Rename == Rename || event.Op&Remove == Remove { - w.Remove(event.Name) - w.mu.Lock() - delete(w.fileExists, event.Name) - w.mu.Unlock() - } - - if path.isDir && event.Op&Write == Write && !(event.Op&Remove == Remove) { - w.sendDirectoryChangeEvents(event.Name) - } else { - // Send the event on the Events channel. - select { - case w.Events <- event: - case <-w.done: - break loop - } - } - - if event.Op&Remove == Remove { - // Look for a file that may have overwritten this. - // For example, mv f1 f2 will delete f2, then create f2. - if path.isDir { - fileDir := filepath.Clean(event.Name) - w.mu.Lock() - _, found := w.watches[fileDir] - w.mu.Unlock() - if found { - // make sure the directory exists before we watch for changes. When we - // do a recursive watch and perform rm -fr, the parent directory might - // have gone missing, ignore the missing directory and let the - // upcoming delete event remove the watch from the parent directory. - if _, err := os.Lstat(fileDir); err == nil { - w.sendDirectoryChangeEvents(fileDir) - } - } - } else { - filePath := filepath.Clean(event.Name) - if fileInfo, err := os.Lstat(filePath); err == nil { - w.sendFileCreatedEventIfNew(filePath, fileInfo) - } - } - } - - // Move to next event - kevents = kevents[1:] - } - } - - // cleanup - err := unix.Close(w.kq) - if err != nil { - // only way the previous loop breaks is if w.done was closed so we need to async send to w.Errors. - select { - case w.Errors <- err: - default: - } - } - close(w.Events) - close(w.Errors) -} - -// newEvent returns an platform-independent Event based on kqueue Fflags. -func newEvent(name string, mask uint32) Event { - e := Event{Name: name} - if mask&unix.NOTE_DELETE == unix.NOTE_DELETE { - e.Op |= Remove - } - if mask&unix.NOTE_WRITE == unix.NOTE_WRITE { - e.Op |= Write - } - if mask&unix.NOTE_RENAME == unix.NOTE_RENAME { - e.Op |= Rename - } - if mask&unix.NOTE_ATTRIB == unix.NOTE_ATTRIB { - e.Op |= Chmod - } - return e -} - -func newCreateEvent(name string) Event { - return Event{Name: name, Op: Create} -} - -// watchDirectoryFiles to mimic inotify when adding a watch on a directory -func (w *Watcher) watchDirectoryFiles(dirPath string) error { - // Get all files - files, err := ioutil.ReadDir(dirPath) - if err != nil { - return err - } - - for _, fileInfo := range files { - filePath := filepath.Join(dirPath, fileInfo.Name()) - filePath, err = w.internalWatch(filePath, fileInfo) - if err != nil { - return err - } - - w.mu.Lock() - w.fileExists[filePath] = true - w.mu.Unlock() - } - - return nil -} - -// sendDirectoryEvents searches the directory for newly created files -// and sends them over the event channel. This functionality is to have -// the BSD version of fsnotify match Linux inotify which provides a -// create event for files created in a watched directory. -func (w *Watcher) sendDirectoryChangeEvents(dirPath string) { - // Get all files - files, err := ioutil.ReadDir(dirPath) - if err != nil { - select { - case w.Errors <- err: - case <-w.done: - return - } - } - - // Search for new files - for _, fileInfo := range files { - filePath := filepath.Join(dirPath, fileInfo.Name()) - err := w.sendFileCreatedEventIfNew(filePath, fileInfo) - - if err != nil { - return - } - } -} - -// sendFileCreatedEvent sends a create event if the file isn't already being tracked. -func (w *Watcher) sendFileCreatedEventIfNew(filePath string, fileInfo os.FileInfo) (err error) { - w.mu.Lock() - _, doesExist := w.fileExists[filePath] - w.mu.Unlock() - if !doesExist { - // Send create event - select { - case w.Events <- newCreateEvent(filePath): - case <-w.done: - return - } - } - - // like watchDirectoryFiles (but without doing another ReadDir) - filePath, err = w.internalWatch(filePath, fileInfo) - if err != nil { - return err - } - - w.mu.Lock() - w.fileExists[filePath] = true - w.mu.Unlock() - - return nil -} - -func (w *Watcher) internalWatch(name string, fileInfo os.FileInfo) (string, error) { - if fileInfo.IsDir() { - // mimic Linux providing delete events for subdirectories - // but preserve the flags used if currently watching subdirectory - w.mu.Lock() - flags := w.dirFlags[name] - w.mu.Unlock() - - flags |= unix.NOTE_DELETE | unix.NOTE_RENAME - return w.addWatch(name, flags) - } - - // watch file to mimic Linux inotify - return w.addWatch(name, noteAllEvents) -} - -// kqueue creates a new kernel event queue and returns a descriptor. -func kqueue() (kq int, err error) { - kq, err = unix.Kqueue() - if kq == -1 { - return kq, err - } - return kq, nil -} - -// register events with the queue -func register(kq int, fds []int, flags int, fflags uint32) error { - changes := make([]unix.Kevent_t, len(fds)) - - for i, fd := range fds { - // SetKevent converts int to the platform-specific types: - unix.SetKevent(&changes[i], fd, unix.EVFILT_VNODE, flags) - changes[i].Fflags = fflags - } - - // register the events - success, err := unix.Kevent(kq, changes, nil, nil) - if success == -1 { - return err - } - return nil -} - -// read retrieves pending events, or waits until an event occurs. -// A timeout of nil blocks indefinitely, while 0 polls the queue. -func read(kq int, events []unix.Kevent_t, timeout *unix.Timespec) ([]unix.Kevent_t, error) { - n, err := unix.Kevent(kq, nil, events, timeout) - if err != nil { - return nil, err - } - return events[0:n], nil -} - -// durationToTimespec prepares a timeout value -func durationToTimespec(d time.Duration) unix.Timespec { - return unix.NsecToTimespec(d.Nanoseconds()) -} diff --git a/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go b/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go deleted file mode 100644 index 36cc3845b..000000000 --- a/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build freebsd || openbsd || netbsd || dragonfly -// +build freebsd openbsd netbsd dragonfly - -package fsnotify - -import "golang.org/x/sys/unix" - -const openMode = unix.O_NONBLOCK | unix.O_RDONLY | unix.O_CLOEXEC diff --git a/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go b/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go deleted file mode 100644 index 98cd8476f..000000000 --- a/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build darwin -// +build darwin - -package fsnotify - -import "golang.org/x/sys/unix" - -// note: this constant is not defined on BSD -const openMode = unix.O_EVTONLY | unix.O_CLOEXEC diff --git a/vendor/github.com/fsnotify/fsnotify/system_bsd.go b/vendor/github.com/fsnotify/fsnotify/system_bsd.go new file mode 100644 index 000000000..f65e8fe3e --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/system_bsd.go @@ -0,0 +1,7 @@ +//go:build freebsd || openbsd || netbsd || dragonfly + +package fsnotify + +import "golang.org/x/sys/unix" + +const openMode = unix.O_NONBLOCK | unix.O_RDONLY | unix.O_CLOEXEC diff --git a/vendor/github.com/fsnotify/fsnotify/system_darwin.go b/vendor/github.com/fsnotify/fsnotify/system_darwin.go new file mode 100644 index 000000000..a29fc7aab --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/system_darwin.go @@ -0,0 +1,8 @@ +//go:build darwin + +package fsnotify + +import "golang.org/x/sys/unix" + +// note: this constant is not defined on BSD +const openMode = unix.O_EVTONLY | unix.O_CLOEXEC diff --git a/vendor/github.com/fsnotify/fsnotify/windows.go b/vendor/github.com/fsnotify/fsnotify/windows.go deleted file mode 100644 index 02ce7deb0..000000000 --- a/vendor/github.com/fsnotify/fsnotify/windows.go +++ /dev/null @@ -1,586 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build windows -// +build windows - -package fsnotify - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "reflect" - "runtime" - "sync" - "syscall" - "unsafe" -) - -// Watcher watches a set of files, delivering events to a channel. -type Watcher struct { - Events chan Event - Errors chan error - isClosed bool // Set to true when Close() is first called - mu sync.Mutex // Map access - port syscall.Handle // Handle to completion port - watches watchMap // Map of watches (key: i-number) - input chan *input // Inputs to the reader are sent on this channel - quit chan chan<- error -} - -// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. -func NewWatcher() (*Watcher, error) { - port, e := syscall.CreateIoCompletionPort(syscall.InvalidHandle, 0, 0, 0) - if e != nil { - return nil, os.NewSyscallError("CreateIoCompletionPort", e) - } - w := &Watcher{ - port: port, - watches: make(watchMap), - input: make(chan *input, 1), - Events: make(chan Event, 50), - Errors: make(chan error), - quit: make(chan chan<- error, 1), - } - go w.readEvents() - return w, nil -} - -// Close removes all watches and closes the events channel. -func (w *Watcher) Close() error { - if w.isClosed { - return nil - } - w.isClosed = true - - // Send "quit" message to the reader goroutine - ch := make(chan error) - w.quit <- ch - if err := w.wakeupReader(); err != nil { - return err - } - return <-ch -} - -// Add starts watching the named file or directory (non-recursively). -func (w *Watcher) Add(name string) error { - if w.isClosed { - return errors.New("watcher already closed") - } - in := &input{ - op: opAddWatch, - path: filepath.Clean(name), - flags: sysFSALLEVENTS, - reply: make(chan error), - } - w.input <- in - if err := w.wakeupReader(); err != nil { - return err - } - return <-in.reply -} - -// Remove stops watching the the named file or directory (non-recursively). -func (w *Watcher) Remove(name string) error { - in := &input{ - op: opRemoveWatch, - path: filepath.Clean(name), - reply: make(chan error), - } - w.input <- in - if err := w.wakeupReader(); err != nil { - return err - } - return <-in.reply -} - -// WatchList returns the directories and files that are being monitered. -func (w *Watcher) WatchList() []string { - w.mu.Lock() - defer w.mu.Unlock() - - entries := make([]string, 0, len(w.watches)) - for _, entry := range w.watches { - for _, watchEntry := range entry { - entries = append(entries, watchEntry.path) - } - } - - return entries -} - -const ( - // Options for AddWatch - sysFSONESHOT = 0x80000000 - sysFSONLYDIR = 0x1000000 - - // Events - sysFSACCESS = 0x1 - sysFSALLEVENTS = 0xfff - sysFSATTRIB = 0x4 - sysFSCLOSE = 0x18 - sysFSCREATE = 0x100 - sysFSDELETE = 0x200 - sysFSDELETESELF = 0x400 - sysFSMODIFY = 0x2 - sysFSMOVE = 0xc0 - sysFSMOVEDFROM = 0x40 - sysFSMOVEDTO = 0x80 - sysFSMOVESELF = 0x800 - - // Special events - sysFSIGNORED = 0x8000 - sysFSQOVERFLOW = 0x4000 -) - -func newEvent(name string, mask uint32) Event { - e := Event{Name: name} - if mask&sysFSCREATE == sysFSCREATE || mask&sysFSMOVEDTO == sysFSMOVEDTO { - e.Op |= Create - } - if mask&sysFSDELETE == sysFSDELETE || mask&sysFSDELETESELF == sysFSDELETESELF { - e.Op |= Remove - } - if mask&sysFSMODIFY == sysFSMODIFY { - e.Op |= Write - } - if mask&sysFSMOVE == sysFSMOVE || mask&sysFSMOVESELF == sysFSMOVESELF || mask&sysFSMOVEDFROM == sysFSMOVEDFROM { - e.Op |= Rename - } - if mask&sysFSATTRIB == sysFSATTRIB { - e.Op |= Chmod - } - return e -} - -const ( - opAddWatch = iota - opRemoveWatch -) - -const ( - provisional uint64 = 1 << (32 + iota) -) - -type input struct { - op int - path string - flags uint32 - reply chan error -} - -type inode struct { - handle syscall.Handle - volume uint32 - index uint64 -} - -type watch struct { - ov syscall.Overlapped - ino *inode // i-number - path string // Directory path - mask uint64 // Directory itself is being watched with these notify flags - names map[string]uint64 // Map of names being watched and their notify flags - rename string // Remembers the old name while renaming a file - buf [4096]byte -} - -type indexMap map[uint64]*watch -type watchMap map[uint32]indexMap - -func (w *Watcher) wakeupReader() error { - e := syscall.PostQueuedCompletionStatus(w.port, 0, 0, nil) - if e != nil { - return os.NewSyscallError("PostQueuedCompletionStatus", e) - } - return nil -} - -func getDir(pathname string) (dir string, err error) { - attr, e := syscall.GetFileAttributes(syscall.StringToUTF16Ptr(pathname)) - if e != nil { - return "", os.NewSyscallError("GetFileAttributes", e) - } - if attr&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 { - dir = pathname - } else { - dir, _ = filepath.Split(pathname) - dir = filepath.Clean(dir) - } - return -} - -func getIno(path string) (ino *inode, err error) { - h, e := syscall.CreateFile(syscall.StringToUTF16Ptr(path), - syscall.FILE_LIST_DIRECTORY, - syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, - nil, syscall.OPEN_EXISTING, - syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OVERLAPPED, 0) - if e != nil { - return nil, os.NewSyscallError("CreateFile", e) - } - var fi syscall.ByHandleFileInformation - if e = syscall.GetFileInformationByHandle(h, &fi); e != nil { - syscall.CloseHandle(h) - return nil, os.NewSyscallError("GetFileInformationByHandle", e) - } - ino = &inode{ - handle: h, - volume: fi.VolumeSerialNumber, - index: uint64(fi.FileIndexHigh)<<32 | uint64(fi.FileIndexLow), - } - return ino, nil -} - -// Must run within the I/O thread. -func (m watchMap) get(ino *inode) *watch { - if i := m[ino.volume]; i != nil { - return i[ino.index] - } - return nil -} - -// Must run within the I/O thread. -func (m watchMap) set(ino *inode, watch *watch) { - i := m[ino.volume] - if i == nil { - i = make(indexMap) - m[ino.volume] = i - } - i[ino.index] = watch -} - -// Must run within the I/O thread. -func (w *Watcher) addWatch(pathname string, flags uint64) error { - dir, err := getDir(pathname) - if err != nil { - return err - } - if flags&sysFSONLYDIR != 0 && pathname != dir { - return nil - } - ino, err := getIno(dir) - if err != nil { - return err - } - w.mu.Lock() - watchEntry := w.watches.get(ino) - w.mu.Unlock() - if watchEntry == nil { - if _, e := syscall.CreateIoCompletionPort(ino.handle, w.port, 0, 0); e != nil { - syscall.CloseHandle(ino.handle) - return os.NewSyscallError("CreateIoCompletionPort", e) - } - watchEntry = &watch{ - ino: ino, - path: dir, - names: make(map[string]uint64), - } - w.mu.Lock() - w.watches.set(ino, watchEntry) - w.mu.Unlock() - flags |= provisional - } else { - syscall.CloseHandle(ino.handle) - } - if pathname == dir { - watchEntry.mask |= flags - } else { - watchEntry.names[filepath.Base(pathname)] |= flags - } - if err = w.startRead(watchEntry); err != nil { - return err - } - if pathname == dir { - watchEntry.mask &= ^provisional - } else { - watchEntry.names[filepath.Base(pathname)] &= ^provisional - } - return nil -} - -// Must run within the I/O thread. -func (w *Watcher) remWatch(pathname string) error { - dir, err := getDir(pathname) - if err != nil { - return err - } - ino, err := getIno(dir) - if err != nil { - return err - } - w.mu.Lock() - watch := w.watches.get(ino) - w.mu.Unlock() - if watch == nil { - return fmt.Errorf("can't remove non-existent watch for: %s", pathname) - } - if pathname == dir { - w.sendEvent(watch.path, watch.mask&sysFSIGNORED) - watch.mask = 0 - } else { - name := filepath.Base(pathname) - w.sendEvent(filepath.Join(watch.path, name), watch.names[name]&sysFSIGNORED) - delete(watch.names, name) - } - return w.startRead(watch) -} - -// Must run within the I/O thread. -func (w *Watcher) deleteWatch(watch *watch) { - for name, mask := range watch.names { - if mask&provisional == 0 { - w.sendEvent(filepath.Join(watch.path, name), mask&sysFSIGNORED) - } - delete(watch.names, name) - } - if watch.mask != 0 { - if watch.mask&provisional == 0 { - w.sendEvent(watch.path, watch.mask&sysFSIGNORED) - } - watch.mask = 0 - } -} - -// Must run within the I/O thread. -func (w *Watcher) startRead(watch *watch) error { - if e := syscall.CancelIo(watch.ino.handle); e != nil { - w.Errors <- os.NewSyscallError("CancelIo", e) - w.deleteWatch(watch) - } - mask := toWindowsFlags(watch.mask) - for _, m := range watch.names { - mask |= toWindowsFlags(m) - } - if mask == 0 { - if e := syscall.CloseHandle(watch.ino.handle); e != nil { - w.Errors <- os.NewSyscallError("CloseHandle", e) - } - w.mu.Lock() - delete(w.watches[watch.ino.volume], watch.ino.index) - w.mu.Unlock() - return nil - } - e := syscall.ReadDirectoryChanges(watch.ino.handle, &watch.buf[0], - uint32(unsafe.Sizeof(watch.buf)), false, mask, nil, &watch.ov, 0) - if e != nil { - err := os.NewSyscallError("ReadDirectoryChanges", e) - if e == syscall.ERROR_ACCESS_DENIED && watch.mask&provisional == 0 { - // Watched directory was probably removed - if w.sendEvent(watch.path, watch.mask&sysFSDELETESELF) { - if watch.mask&sysFSONESHOT != 0 { - watch.mask = 0 - } - } - err = nil - } - w.deleteWatch(watch) - w.startRead(watch) - return err - } - return nil -} - -// readEvents reads from the I/O completion port, converts the -// received events into Event objects and sends them via the Events channel. -// Entry point to the I/O thread. -func (w *Watcher) readEvents() { - var ( - n, key uint32 - ov *syscall.Overlapped - ) - runtime.LockOSThread() - - for { - e := syscall.GetQueuedCompletionStatus(w.port, &n, &key, &ov, syscall.INFINITE) - watch := (*watch)(unsafe.Pointer(ov)) - - if watch == nil { - select { - case ch := <-w.quit: - w.mu.Lock() - var indexes []indexMap - for _, index := range w.watches { - indexes = append(indexes, index) - } - w.mu.Unlock() - for _, index := range indexes { - for _, watch := range index { - w.deleteWatch(watch) - w.startRead(watch) - } - } - var err error - if e := syscall.CloseHandle(w.port); e != nil { - err = os.NewSyscallError("CloseHandle", e) - } - close(w.Events) - close(w.Errors) - ch <- err - return - case in := <-w.input: - switch in.op { - case opAddWatch: - in.reply <- w.addWatch(in.path, uint64(in.flags)) - case opRemoveWatch: - in.reply <- w.remWatch(in.path) - } - default: - } - continue - } - - switch e { - case syscall.ERROR_MORE_DATA: - if watch == nil { - w.Errors <- errors.New("ERROR_MORE_DATA has unexpectedly null lpOverlapped buffer") - } else { - // The i/o succeeded but the buffer is full. - // In theory we should be building up a full packet. - // In practice we can get away with just carrying on. - n = uint32(unsafe.Sizeof(watch.buf)) - } - case syscall.ERROR_ACCESS_DENIED: - // Watched directory was probably removed - w.sendEvent(watch.path, watch.mask&sysFSDELETESELF) - w.deleteWatch(watch) - w.startRead(watch) - continue - case syscall.ERROR_OPERATION_ABORTED: - // CancelIo was called on this handle - continue - default: - w.Errors <- os.NewSyscallError("GetQueuedCompletionPort", e) - continue - case nil: - } - - var offset uint32 - for { - if n == 0 { - w.Events <- newEvent("", sysFSQOVERFLOW) - w.Errors <- errors.New("short read in readEvents()") - break - } - - // Point "raw" to the event in the buffer - raw := (*syscall.FileNotifyInformation)(unsafe.Pointer(&watch.buf[offset])) - // TODO: Consider using unsafe.Slice that is available from go1.17 - // https://stackoverflow.com/questions/51187973/how-to-create-an-array-or-a-slice-from-an-array-unsafe-pointer-in-golang - // instead of using a fixed syscall.MAX_PATH buf, we create a buf that is the size of the path name - size := int(raw.FileNameLength / 2) - var buf []uint16 - sh := (*reflect.SliceHeader)(unsafe.Pointer(&buf)) - sh.Data = uintptr(unsafe.Pointer(&raw.FileName)) - sh.Len = size - sh.Cap = size - name := syscall.UTF16ToString(buf) - fullname := filepath.Join(watch.path, name) - - var mask uint64 - switch raw.Action { - case syscall.FILE_ACTION_REMOVED: - mask = sysFSDELETESELF - case syscall.FILE_ACTION_MODIFIED: - mask = sysFSMODIFY - case syscall.FILE_ACTION_RENAMED_OLD_NAME: - watch.rename = name - case syscall.FILE_ACTION_RENAMED_NEW_NAME: - if watch.names[watch.rename] != 0 { - watch.names[name] |= watch.names[watch.rename] - delete(watch.names, watch.rename) - mask = sysFSMOVESELF - } - } - - sendNameEvent := func() { - if w.sendEvent(fullname, watch.names[name]&mask) { - if watch.names[name]&sysFSONESHOT != 0 { - delete(watch.names, name) - } - } - } - if raw.Action != syscall.FILE_ACTION_RENAMED_NEW_NAME { - sendNameEvent() - } - if raw.Action == syscall.FILE_ACTION_REMOVED { - w.sendEvent(fullname, watch.names[name]&sysFSIGNORED) - delete(watch.names, name) - } - if w.sendEvent(fullname, watch.mask&toFSnotifyFlags(raw.Action)) { - if watch.mask&sysFSONESHOT != 0 { - watch.mask = 0 - } - } - if raw.Action == syscall.FILE_ACTION_RENAMED_NEW_NAME { - fullname = filepath.Join(watch.path, watch.rename) - sendNameEvent() - } - - // Move to the next event in the buffer - if raw.NextEntryOffset == 0 { - break - } - offset += raw.NextEntryOffset - - // Error! - if offset >= n { - w.Errors <- errors.New("Windows system assumed buffer larger than it is, events have likely been missed.") - break - } - } - - if err := w.startRead(watch); err != nil { - w.Errors <- err - } - } -} - -func (w *Watcher) sendEvent(name string, mask uint64) bool { - if mask == 0 { - return false - } - event := newEvent(name, uint32(mask)) - select { - case ch := <-w.quit: - w.quit <- ch - case w.Events <- event: - } - return true -} - -func toWindowsFlags(mask uint64) uint32 { - var m uint32 - if mask&sysFSACCESS != 0 { - m |= syscall.FILE_NOTIFY_CHANGE_LAST_ACCESS - } - if mask&sysFSMODIFY != 0 { - m |= syscall.FILE_NOTIFY_CHANGE_LAST_WRITE - } - if mask&sysFSATTRIB != 0 { - m |= syscall.FILE_NOTIFY_CHANGE_ATTRIBUTES - } - if mask&(sysFSMOVE|sysFSCREATE|sysFSDELETE) != 0 { - m |= syscall.FILE_NOTIFY_CHANGE_FILE_NAME | syscall.FILE_NOTIFY_CHANGE_DIR_NAME - } - return m -} - -func toFSnotifyFlags(action uint32) uint64 { - switch action { - case syscall.FILE_ACTION_ADDED: - return sysFSCREATE - case syscall.FILE_ACTION_REMOVED: - return sysFSDELETE - case syscall.FILE_ACTION_MODIFIED: - return sysFSMODIFY - case syscall.FILE_ACTION_RENAMED_OLD_NAME: - return sysFSMOVEDFROM - case syscall.FILE_ACTION_RENAMED_NEW_NAME: - return sysFSMOVEDTO - } - return 0 -} diff --git a/vendor/github.com/go-logr/logr/README.md b/vendor/github.com/go-logr/logr/README.md index 8969526a6..7c7f0c69c 100644 --- a/vendor/github.com/go-logr/logr/README.md +++ b/vendor/github.com/go-logr/logr/README.md @@ -1,6 +1,7 @@ # A minimal logging API for Go [![Go Reference](https://pkg.go.dev/badge/github.com/go-logr/logr.svg)](https://pkg.go.dev/github.com/go-logr/logr) +[![Go Report Card](https://goreportcard.com/badge/github.com/go-logr/logr)](https://goreportcard.com/report/github.com/go-logr/logr) [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/go-logr/logr/badge)](https://securityscorecards.dev/viewer/?platform=github.com&org=go-logr&repo=logr) logr offers an(other) opinion on how Go programs and libraries can do logging diff --git a/vendor/github.com/go-logr/logr/funcr/funcr.go b/vendor/github.com/go-logr/logr/funcr/funcr.go index fb2f866f4..30568e768 100644 --- a/vendor/github.com/go-logr/logr/funcr/funcr.go +++ b/vendor/github.com/go-logr/logr/funcr/funcr.go @@ -236,15 +236,14 @@ func newFormatter(opts Options, outfmt outputFormat) Formatter { // implementation. It should be constructed with NewFormatter. Some of // its methods directly implement logr.LogSink. type Formatter struct { - outputFormat outputFormat - prefix string - values []any - valuesStr string - parentValuesStr string - depth int - opts *Options - group string // for slog groups - groupDepth int + outputFormat outputFormat + prefix string + values []any + valuesStr string + depth int + opts *Options + groupName string // for slog groups + groups []groupDef } // outputFormat indicates which outputFormat to use. @@ -257,6 +256,13 @@ const ( outputJSON ) +// groupDef represents a saved group. The values may be empty, but we don't +// know if we need to render the group until the final record is rendered. +type groupDef struct { + name string + values string +} + // PseudoStruct is a list of key-value pairs that gets logged as a struct. type PseudoStruct []any @@ -264,76 +270,102 @@ type PseudoStruct []any func (f Formatter) render(builtins, args []any) string { // Empirically bytes.Buffer is faster than strings.Builder for this. buf := bytes.NewBuffer(make([]byte, 0, 1024)) + if f.outputFormat == outputJSON { - buf.WriteByte('{') // for the whole line + buf.WriteByte('{') // for the whole record } + // Render builtins vals := builtins if hook := f.opts.RenderBuiltinsHook; hook != nil { vals = hook(f.sanitize(vals)) } - f.flatten(buf, vals, false, false) // keys are ours, no need to escape + f.flatten(buf, vals, false) // keys are ours, no need to escape continuing := len(builtins) > 0 - if f.parentValuesStr != "" { - if continuing { - buf.WriteByte(f.comma()) + // Turn the inner-most group into a string + argsStr := func() string { + buf := bytes.NewBuffer(make([]byte, 0, 1024)) + + vals = args + if hook := f.opts.RenderArgsHook; hook != nil { + vals = hook(f.sanitize(vals)) } - buf.WriteString(f.parentValuesStr) - continuing = true - } + f.flatten(buf, vals, true) // escape user-provided keys - groupDepth := f.groupDepth - if f.group != "" { - if f.valuesStr != "" || len(args) != 0 { - if continuing { - buf.WriteByte(f.comma()) - } - buf.WriteString(f.quoted(f.group, true)) // escape user-provided keys - buf.WriteByte(f.colon()) - buf.WriteByte('{') // for the group - continuing = false - } else { - // The group was empty - groupDepth-- + return buf.String() + }() + + // Render the stack of groups from the inside out. + bodyStr := f.renderGroup(f.groupName, f.valuesStr, argsStr) + for i := len(f.groups) - 1; i >= 0; i-- { + grp := &f.groups[i] + if grp.values == "" && bodyStr == "" { + // no contents, so we must elide the whole group + continue } + bodyStr = f.renderGroup(grp.name, grp.values, bodyStr) } - if f.valuesStr != "" { + if bodyStr != "" { if continuing { buf.WriteByte(f.comma()) } - buf.WriteString(f.valuesStr) - continuing = true + buf.WriteString(bodyStr) } - vals = args - if hook := f.opts.RenderArgsHook; hook != nil { - vals = hook(f.sanitize(vals)) + if f.outputFormat == outputJSON { + buf.WriteByte('}') // for the whole record } - f.flatten(buf, vals, continuing, true) // escape user-provided keys - for i := 0; i < groupDepth; i++ { - buf.WriteByte('}') // for the groups + return buf.String() +} + +// renderGroup returns a string representation of the named group with rendered +// values and args. If the name is empty, this will return the values and args, +// joined. If the name is not empty, this will return a single key-value pair, +// where the value is a grouping of the values and args. If the values and +// args are both empty, this will return an empty string, even if the name was +// specified. +func (f Formatter) renderGroup(name string, values string, args string) string { + buf := bytes.NewBuffer(make([]byte, 0, 1024)) + + needClosingBrace := false + if name != "" && (values != "" || args != "") { + buf.WriteString(f.quoted(name, true)) // escape user-provided keys + buf.WriteByte(f.colon()) + buf.WriteByte('{') + needClosingBrace = true } - if f.outputFormat == outputJSON { - buf.WriteByte('}') // for the whole line + continuing := false + if values != "" { + buf.WriteString(values) + continuing = true + } + + if args != "" { + if continuing { + buf.WriteByte(f.comma()) + } + buf.WriteString(args) + } + + if needClosingBrace { + buf.WriteByte('}') } return buf.String() } -// flatten renders a list of key-value pairs into a buffer. If continuing is -// true, it assumes that the buffer has previous values and will emit a -// separator (which depends on the output format) before the first pair it -// writes. If escapeKeys is true, the keys are assumed to have -// non-JSON-compatible characters in them and must be evaluated for escapes. +// flatten renders a list of key-value pairs into a buffer. If escapeKeys is +// true, the keys are assumed to have non-JSON-compatible characters in them +// and must be evaluated for escapes. // // This function returns a potentially modified version of kvList, which // ensures that there is a value for every key (adding a value if needed) and // that each key is a string (substituting a key if needed). -func (f Formatter) flatten(buf *bytes.Buffer, kvList []any, continuing bool, escapeKeys bool) []any { +func (f Formatter) flatten(buf *bytes.Buffer, kvList []any, escapeKeys bool) []any { // This logic overlaps with sanitize() but saves one type-cast per key, // which can be measurable. if len(kvList)%2 != 0 { @@ -354,7 +386,7 @@ func (f Formatter) flatten(buf *bytes.Buffer, kvList []any, continuing bool, esc } v := kvList[i+1] - if i > 0 || continuing { + if i > 0 { if f.outputFormat == outputJSON { buf.WriteByte(f.comma()) } else { @@ -766,46 +798,17 @@ func (f Formatter) sanitize(kvList []any) []any { // startGroup opens a new group scope (basically a sub-struct), which locks all // the current saved values and starts them anew. This is needed to satisfy // slog. -func (f *Formatter) startGroup(group string) { +func (f *Formatter) startGroup(name string) { // Unnamed groups are just inlined. - if group == "" { + if name == "" { return } - // Any saved values can no longer be changed. - buf := bytes.NewBuffer(make([]byte, 0, 1024)) - continuing := false - - if f.parentValuesStr != "" { - buf.WriteString(f.parentValuesStr) - continuing = true - } - - if f.group != "" && f.valuesStr != "" { - if continuing { - buf.WriteByte(f.comma()) - } - buf.WriteString(f.quoted(f.group, true)) // escape user-provided keys - buf.WriteByte(f.colon()) - buf.WriteByte('{') // for the group - continuing = false - } - - if f.valuesStr != "" { - if continuing { - buf.WriteByte(f.comma()) - } - buf.WriteString(f.valuesStr) - } - - // NOTE: We don't close the scope here - that's done later, when a log line - // is actually rendered (because we have N scopes to close). - - f.parentValuesStr = buf.String() + n := len(f.groups) + f.groups = append(f.groups[:n:n], groupDef{f.groupName, f.valuesStr}) // Start collecting new values. - f.group = group - f.groupDepth++ + f.groupName = name f.valuesStr = "" f.values = nil } @@ -900,7 +903,7 @@ func (f *Formatter) AddValues(kvList []any) { // Pre-render values, so we don't have to do it on each Info/Error call. buf := bytes.NewBuffer(make([]byte, 0, 1024)) - f.flatten(buf, vals, false, true) // escape user-provided keys + f.flatten(buf, vals, true) // escape user-provided keys f.valuesStr = buf.String() } diff --git a/vendor/modules.txt b/vendor/modules.txt index 4bb8c6176..f5baecac6 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -83,10 +83,11 @@ github.com/emicklei/go-restful/log # github.com/felixge/httpsnoop v1.0.4 ## explicit; go 1.13 github.com/felixge/httpsnoop -# github.com/fsnotify/fsnotify v1.5.4 -## explicit; go 1.16 +# github.com/fsnotify/fsnotify v1.8.0 +## explicit; go 1.17 github.com/fsnotify/fsnotify -# github.com/go-logr/logr v1.4.1 +github.com/fsnotify/fsnotify/internal +# github.com/go-logr/logr v1.4.2 ## explicit; go 1.18 github.com/go-logr/logr github.com/go-logr/logr/funcr