feat(obs): change parallelized file system to obs Bucket

Author: Zippo-Wang

cluster/images/obs-csi-plugin/Dockerfile

@@ -5,16 +5,13 @@ LABEL description="HuaweiCloud CSI Plugin"
 
 WORKDIR /obs-csi
 
-COPY obsfs_CentOS7.6_amd64.tar.gz ./
-COPY obsfs_Ubuntu16.04_amd64.tar.gz ./
 COPY entrypoint.sh entrypoint.sh
 COPY nsenter /nsenter
 COPY csi-connector-server ./
 COPY csi-connector.service ./
 COPY obs-csi-plugin obs-csi-plugin
-COPY install_obsfs.sh ./
+COPY install_s3fs.sh ./
 COPY stop-server.sh ./
-COPY huaweicloud-obs-obsfs.tar.gz ./
 
 RUN chmod +x obs-csi-plugin
 RUN chmod +x entrypoint.sh

cluster/images/obs-csi-plugin/csi-connector-server.go

@@ -25,7 +25,7 @@ const (
 	region        = "region"
 	cloud         = "cloud"
 	credential    = "credential"
-	defaultOpts   = "-o big_writes -o max_write=131072 -o use_ino"
+	defaultOpts   = "-o nonempty -o big_writes -o max_write=131072"
 )
 
 type ResponseBody struct {
@@ -119,7 +119,7 @@ func main() {
 	//nolint:errcheck
 	flag.CommandLine.Parse([]string{})
 
-	initObsfsUtil()
+	initS3fsUtil()
 	if checkFileExists(obs.SocketPath) {
 		if err := os.Remove(obs.SocketPath); err != nil {
 			log.Fatalf("Failed to remove path: %s, err: %v", obs.SocketPath, err)
@@ -153,26 +153,27 @@ func mountHandler(parameters map[string]string) error {
 	if parameters[cloud] == "prod-cloud-ocb.orange-business.com" {
 		obsName = "oss"
 	}
-
 	mountOpts := parameters[mountFlags]
 	if mountOpts == "" {
 		mountOpts = defaultOpts
 	}
+
 	options := []string{
-		"obsfs",
 		parameters[bucketName],
 		parameters[targetPath],
-		fmt.Sprintf("-o url=%s.%s.%s", obsName, parameters[region], parameters[cloud]),
-		fmt.Sprintf("-o passwd_file=%s", credentialFile),
-		mountOpts,
+		"-o",
+		fmt.Sprintf("url=https://%s.%s.%s", obsName, parameters[region], parameters[cloud]),
+		"-o",
+		fmt.Sprintf("passwd_file=%s", credentialFile),
 	}
+	mntOptsArr := strings.Split(mountOpts, " ")
+	options = append(options, mntOptsArr...)
 
-	cmd := exec.Command("sh", "-c")
-	cmd.Args = append(cmd.Args, strings.Join(options, " "))
+	cmd := exec.Command("s3fs", options...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		log.Errorf("failed to mount CMD: obsfs %s, output: %s, error: %v", strings.Join(options, " "), string(out), err)
-		return fmt.Errorf("failed to mount CMD: obsfs %s, output: %s, error: %v", strings.Join(options, " "), string(out), err)
+		log.Errorf("failed to mount bucket to node, CMD: s3fs %s, output: %s, error: %v", strings.Join(options, " "), string(out), err)
+		return fmt.Errorf("failed to mount bucket to node, CMD: s3fs %s, output: %s, error: %v", strings.Join(options, " "), string(out), err)
 	}
 	log.Infof("success to mount CMD: %s", strings.Join(options, " "))
 	return nil
@@ -186,11 +187,11 @@ func deleteCredential(credential string) {
 	}
 }
 
-func initObsfsUtil() {
-	cmd := fmt.Sprintf("sh %s/install_obsfs.sh >> %s/connector.log 2>&1 &", credentialDir, credentialDir)
+func initS3fsUtil() {
+	cmd := fmt.Sprintf("sh %s/install_s3fs.sh > %s/connector.log 2>&1 &", credentialDir, credentialDir)
 	out, err := exec.Command("sh", "-c", cmd).CombinedOutput()
-	log.Infof("install obsfs %s", string(out))
+	log.Infof("install s3fs %s", string(out))
 	if err != nil {
-		log.Errorf("error install obsfs: %s", err)
+		log.Errorf("error install s3fs: %s", err)
 	}
 }

cluster/images/obs-csi-plugin/entrypoint.sh

@@ -3,10 +3,7 @@
 HOST_CMD="/nsenter --mount=/proc/1/ns/mnt"
 mkdir -p /var/lib/csi/
 
-cp -f /obs-csi/huaweicloud-obs-obsfs.tar.gz /var/lib/csi/huaweicloud-obs-obsfs.tar.gz
-cp -f /obs-csi/obsfs_CentOS7.6_amd64.tar.gz /var/lib/csi/obsfs_CentOS7.6_amd64.tar.gz
-cp -f /obs-csi/obsfs_Ubuntu16.04_amd64.tar.gz /var/lib/csi/obsfs_Ubuntu16.04_amd64.tar.gz
-cp -f /obs-csi/install_obsfs.sh /var/lib/csi/install_obsfs.sh
+cp -f /obs-csi/install_s3fs.sh /var/lib/csi/install_s3fs.sh
 
 echo "Starting install obs csi-connector-server...."
 $HOST_CMD systemctl stop csi-connector.service

cluster/images/obs-csi-plugin/huaweicloud-obs-obsfs.tar.gz

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+checkS3fsInstalled() {
+  echo "[INFO] Check version and check if it works"
+  isInstalled=1
+  command -v s3fs >/dev/null 2>&1 || { isInstalled=0; }
+  if [ ${isInstalled} = 0 ]; then
+  	echo "[WARN] has no command named s3fs"
+  	return ${isInstalled}
+  fi
+  echo "[INFO] s3fs has been installed"
+  return ${isInstalled}
+}
+
+checkLinuxOS() {
+  if grep -i -q "EulerOS" /etc/os-release; then
+    # EulerOS
+    return 5
+  elif [ -f "/etc/redhat-release" ]; then
+    # CentOS or RHEL or EulerOS
+    if grep -i -q "CentOS" /etc/redhat-release; then
+      return 1
+    elif grep -i -q "Fedora" /etc/redhat-release; then
+      return 4
+    elif grep -i -q "EulerOS" /etc/redhat-release; then
+      return 5
+    fi
+  elif [ -f "/etc/issue" ]; then
+    # Ubuntu or Debian or Euler
+    if grep -i -q "ubuntu" /etc/issue; then
+      return 2
+    elif grep -i -q "debian" /etc/issue; then
+      return 3
+    elif grep -i -q "Euler" /etc/os-release; then
+      return 5
+    fi
+  elif [ -f "/etc/fedora-release" ]; then
+    # Fedora
+    if grep -i -q "Fedora" /etc/fedora-release; then
+      return 4
+    fi
+  else
+    # unknown OS
+    return 0
+  fi
+}
+
+installS3fs() {
+  echo "[INFO] Install dependencies and s3fs"
+  pgkManageCmd=${1}
+
+  if which yum >/dev/null 2>&1 && [ ${pgkManageCmd} = "yum" ]; then
+    echo "[INFO] command 'yum' is available, trying to install s3fs with yum"
+    sudo yum -y install epel-release
+    sudo yum -y install s3fs-fuse
+    return
+  elif which apt >/dev/null 2>&1 && [ ${pgkManageCmd} = "apt" ]; then
+    echo "[INFO] command 'apt' is available, trying to install s3fs with apt"
+    sudo apt -y install s3fs
+    return
+  elif which dnf >/dev/null 2>&1 && [ ${pgkManageCmd} = "dnf" ]; then
+    echo "[INFO] command 'dnf' is available, trying to install s3fs with dnf"
+    sudo dnf -y install s3fs-fuse
+    return
+  else
+    echo "[ERROR] unsupported systems, please install s3fs manually"
+  fi
+}
+
+# pre install
+checkS3fsInstalled
+isInstalled=$?
+if [ ${isInstalled} = 1 ]; then
+  exit 1
+fi
+
+# install
+checkLinuxOS
+osCode=$?
+echo "[INFO] OS Code is ${osCode}"
+
+if [ ${osCode} = 1 ] || [ ${osCode} = 5 ]; then
+  echo "[INFO] OS is CentOS or EulerOS, use yum to install s3fs"
+  installS3fs "yum"
+elif [ ${osCode} = 2 ] || [ ${osCode} = 3 ]; then
+  echo "[INFO] OS is Ubuntu or Debian, use apt to install s3fs"
+  installS3fs "apt"
+elif [ ${osCode} = 4 ]; then
+  echo "[INFO] OS is Fedora, use dnf to install s3fs"
+  installS3fs "dnf"
+fi
+
+# post install
+checkS3fsInstalled
+isInstalled=$?
+if [ ${isInstalled} = 1 ]; then
+  exit 1
+fi
+echo "[ERROR] failed to install s3fs, please install s3fs manually"

cluster/images/obs-csi-plugin/install_obsfs.sh

@@ -6,7 +6,6 @@ require (
 	github.com/chnsz/golangsdk v0.0.0-20230808032048-e9d83e61c98d
 	github.com/container-storage-interface/spec v1.5.0
 	github.com/golang/glog v1.1.0
-	github.com/google/uuid v1.3.0
 	github.com/huaweicloud/huaweicloud-sdk-go-obs v3.22.11+incompatible
 	github.com/kubernetes-csi/csi-lib-utils v0.9.1
 	github.com/onsi/ginkgo/v2 v2.1.4

cluster/images/obs-csi-plugin/install_s3fs.sh

@@ -197,8 +197,6 @@ github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLe
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=