Skip to content

Commit 7bd65af

Browse files
Zippo-WangZippo-Wang
committed
feat(obs): change parallelized file system to obs Bucket
1 parent b082aad commit 7bd65af

File tree

13 files changed

+150
-157
lines changed

13 files changed

+150
-157
lines changed

‎cluster/images/obs-csi-plugin/Dockerfile‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,13 @@ LABEL description="HuaweiCloud CSI Plugin"
55

66
WORKDIR /obs-csi
77

8-
COPY obsfs_CentOS7.6_amd64.tar.gz ./
9-
COPY obsfs_Ubuntu16.04_amd64.tar.gz ./
108
COPY entrypoint.sh entrypoint.sh
119
COPY nsenter /nsenter
1210
COPY csi-connector-server ./
1311
COPY csi-connector.service ./
1412
COPY obs-csi-plugin obs-csi-plugin
15-
COPY install_obsfs.sh ./
13+
COPY install_s3fs.sh ./
1614
COPY stop-server.sh ./
17-
COPY huaweicloud-obs-obsfs.tar.gz ./
1815

1916
RUN chmod +x obs-csi-plugin
2017
RUN chmod +x entrypoint.sh

‎cluster/images/obs-csi-plugin/csi-connector-server.go‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ const (
2525
region = "region"
2626
cloud = "cloud"
2727
credential = "credential"
28-
defaultOpts = "-o big_writes -o max_write=131072 -o use_ino"
28+
defaultOpts = "-o nonempty -o big_writes -o max_write=131072"
2929
)
3030

3131
type ResponseBody struct {
@@ -119,7 +119,7 @@ func main() {
119119
//nolint:errcheck
120120
flag.CommandLine.Parse([]string{})
121121

122-
initObsfsUtil()
122+
initS3fsUtil()
123123
if checkFileExists(obs.SocketPath) {
124124
if err := os.Remove(obs.SocketPath); err != nil {
125125
log.Fatalf("Failed to remove path: %s, err: %v", obs.SocketPath, err)
@@ -153,26 +153,27 @@ func mountHandler(parameters map[string]string) error {
153153
if parameters[cloud] == "prod-cloud-ocb.orange-business.com" {
154154
obsName = "oss"
155155
}
156-
157156
mountOpts := parameters[mountFlags]
158157
if mountOpts == "" {
159158
mountOpts = defaultOpts
160159
}
160+
161161
options := []string{
162-
"obsfs",
163162
parameters[bucketName],
164163
parameters[targetPath],
165-
fmt.Sprintf("-o url=%s.%s.%s", obsName, parameters[region], parameters[cloud]),
166-
fmt.Sprintf("-o passwd_file=%s", credentialFile),
167-
mountOpts,
164+
"-o",
165+
fmt.Sprintf("url=https://%s.%s.%s", obsName, parameters[region], parameters[cloud]),
166+
"-o",
167+
fmt.Sprintf("passwd_file=%s", credentialFile),
168168
}
169+
mntOptsArr := strings.Split(mountOpts, " ")
170+
options = append(options, mntOptsArr...)
169171

170-
cmd := exec.Command("sh", "-c")
171-
cmd.Args = append(cmd.Args, strings.Join(options, " "))
172+
cmd := exec.Command("s3fs", options...)
172173
out, err := cmd.CombinedOutput()
173174
if err != nil {
174-
log.Errorf("failed to mount CMD: obsfs %s, output: %s, error: %v", strings.Join(options, " "), string(out), err)
175-
return fmt.Errorf("failed to mount CMD: obsfs %s, output: %s, error: %v", strings.Join(options, " "), string(out), err)
175+
log.Errorf("failed to mount bucket to node, CMD: s3fs %s, output: %s, error: %v", strings.Join(options, " "), string(out), err)
176+
return fmt.Errorf("failed to mount bucket to node, CMD: s3fs %s, output: %s, error: %v", strings.Join(options, " "), string(out), err)
176177
}
177178
log.Infof("success to mount CMD: %s", strings.Join(options, " "))
178179
return nil
@@ -186,11 +187,11 @@ func deleteCredential(credential string) {
186187
}
187188
}
188189

189-
func initObsfsUtil() {
190-
cmd := fmt.Sprintf("sh %s/install_obsfs.sh >> %s/connector.log 2>&1 &", credentialDir, credentialDir)
190+
func initS3fsUtil() {
191+
cmd := fmt.Sprintf("sh %s/install_s3fs.sh > %s/connector.log 2>&1 &", credentialDir, credentialDir)
191192
out, err := exec.Command("sh", "-c", cmd).CombinedOutput()
192-
log.Infof("install obsfs %s", string(out))
193+
log.Infof("install s3fs %s", string(out))
193194
if err != nil {
194-
log.Errorf("error install obsfs: %s", err)
195+
log.Errorf("error install s3fs: %s", err)
195196
}
196197
}

‎cluster/images/obs-csi-plugin/entrypoint.sh‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,7 @@
33
HOST_CMD="/nsenter --mount=/proc/1/ns/mnt"
44
mkdir -p /var/lib/csi/
55

6-
cp -f /obs-csi/huaweicloud-obs-obsfs.tar.gz /var/lib/csi/huaweicloud-obs-obsfs.tar.gz
7-
cp -f /obs-csi/obsfs_CentOS7.6_amd64.tar.gz /var/lib/csi/obsfs_CentOS7.6_amd64.tar.gz
8-
cp -f /obs-csi/obsfs_Ubuntu16.04_amd64.tar.gz /var/lib/csi/obsfs_Ubuntu16.04_amd64.tar.gz
9-
cp -f /obs-csi/install_obsfs.sh /var/lib/csi/install_obsfs.sh
6+
cp -f /obs-csi/install_s3fs.sh /var/lib/csi/install_s3fs.sh
107

118
echo "Starting install obs csi-connector-server...."
129
$HOST_CMD systemctl stop csi-connector.service

‎cluster/images/obs-csi-plugin/huaweicloud-obs-obsfs.tar.gz‎

-1.13 MB
Binary file not shown.

‎cluster/images/obs-csi-plugin/install_obsfs.sh‎

Lines changed: 0 additions & 96 deletions
This file was deleted.

‎cluster/images/obs-csi-plugin/install_s3fs.sh‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
#!/bin/bash
2+
3+
checkS3fsInstalled() {
4+
echo "[INFO] Check version and check if it works"
5+
isInstalled=1
6+
command -v s3fs >/dev/null 2>&1 || { isInstalled=0; }
7+
if [ ${isInstalled} = 0 ]; then
8+
echo "[WARN] has no command named s3fs"
9+
return ${isInstalled}
10+
fi
11+
echo "[INFO] s3fs has been installed"
12+
return ${isInstalled}
13+
}
14+
15+
checkLinuxOS() {
16+
if grep -i -q "EulerOS" /etc/os-release; then
17+
# EulerOS
18+
return 5
19+
elif [ -f "/etc/redhat-release" ]; then
20+
# CentOS or RHEL or EulerOS
21+
if grep -i -q "CentOS" /etc/redhat-release; then
22+
return 1
23+
elif grep -i -q "Fedora" /etc/redhat-release; then
24+
return 4
25+
elif grep -i -q "EulerOS" /etc/redhat-release; then
26+
return 5
27+
fi
28+
elif [ -f "/etc/issue" ]; then
29+
# Ubuntu or Debian or Euler
30+
if grep -i -q "ubuntu" /etc/issue; then
31+
return 2
32+
elif grep -i -q "debian" /etc/issue; then
33+
return 3
34+
elif grep -i -q "Euler" /etc/os-release; then
35+
return 5
36+
fi
37+
elif [ -f "/etc/fedora-release" ]; then
38+
# Fedora
39+
if grep -i -q "Fedora" /etc/fedora-release; then
40+
return 4
41+
fi
42+
else
43+
# unknown OS
44+
return 0
45+
fi
46+
}
47+
48+
installS3fs() {
49+
echo "[INFO] Install dependencies and s3fs"
50+
pgkManageCmd=${1}
51+
52+
if which yum >/dev/null 2>&1 && [ ${pgkManageCmd} = "yum" ]; then
53+
echo "[INFO] command 'yum' is available, trying to install s3fs with yum"
54+
sudo yum -y install epel-release
55+
sudo yum -y install s3fs-fuse
56+
return
57+
elif which apt >/dev/null 2>&1 && [ ${pgkManageCmd} = "apt" ]; then
58+
echo "[INFO] command 'apt' is available, trying to install s3fs with apt"
59+
sudo apt -y install s3fs
60+
return
61+
elif which dnf >/dev/null 2>&1 && [ ${pgkManageCmd} = "dnf" ]; then
62+
echo "[INFO] command 'dnf' is available, trying to install s3fs with dnf"
63+
sudo dnf -y install s3fs-fuse
64+
return
65+
else
66+
echo "[ERROR] unsupported systems, please install s3fs manually"
67+
fi
68+
}
69+
70+
# pre install
71+
checkS3fsInstalled
72+
isInstalled=$?
73+
if [ ${isInstalled} = 1 ]; then
74+
exit 1
75+
fi
76+
77+
# install
78+
checkLinuxOS
79+
osCode=$?
80+
echo "[INFO] OS Code is ${osCode}"
81+
82+
if [ ${osCode} = 1 ] || [ ${osCode} = 5 ]; then
83+
echo "[INFO] OS is CentOS or EulerOS, use yum to install s3fs"
84+
installS3fs "yum"
85+
elif [ ${osCode} = 2 ] || [ ${osCode} = 3 ]; then
86+
echo "[INFO] OS is Ubuntu or Debian, use apt to install s3fs"
87+
installS3fs "apt"
88+
elif [ ${osCode} = 4 ]; then
89+
echo "[INFO] OS is Fedora, use dnf to install s3fs"
90+
installS3fs "dnf"
91+
fi
92+
93+
# post install
94+
checkS3fsInstalled
95+
isInstalled=$?
96+
if [ ${isInstalled} = 1 ]; then
97+
exit 1
98+
fi
99+
echo "[ERROR] failed to install s3fs, please install s3fs manually"

‎cluster/images/obs-csi-plugin/obsfs_CentOS7.6_amd64.tar.gz‎

-2.11 MB
Binary file not shown.

‎cluster/images/obs-csi-plugin/obsfs_Ubuntu16.04_amd64.tar.gz‎

-2.27 MB
Binary file not shown.

‎pkg/obs/controllerserver.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ func (cs *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolum
4343
return nil, err
4444
}
4545

46-
if volume, err := services.GetParallelFSBucket(credentials, volName); err != nil && status.Code(err) != codes.NotFound {
46+
if volume, err := services.GetObsBucket(credentials, volName); err != nil && status.Code(err) != codes.NotFound {
4747
return nil, err
4848
} else if volume != nil {
4949
log.Infof("Volume %s existence, skip creating", volName)
@@ -61,7 +61,7 @@ func (cs *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolum
6161
}
6262
}
6363

64-
volume, err := services.GetParallelFSBucket(credentials, volName)
64+
volume, err := services.GetObsBucket(credentials, volName)
6565
if err != nil {
6666
return nil, err
6767
}
@@ -80,7 +80,7 @@ func (cs *controllerServer) DeleteVolume(_ context.Context, req *csi.DeleteVolum
8080
}
8181

8282
credentials := cs.Driver.cloud
83-
volume, err := services.GetParallelFSBucket(credentials, volName)
83+
volume, err := services.GetObsBucket(credentials, volName)
8484
if err != nil {
8585
if common.IsNotFound(err) {
8686
log.Infof("Volume %s does not exist, skip deleting", volName)
@@ -115,7 +115,7 @@ func (cs *controllerServer) ControllerGetVolume(_ context.Context, req *csi.Cont
115115
return nil, status.Error(codes.InvalidArgument, "Validation failed, volume ID cannot be empty")
116116
}
117117

118-
bucket, err := services.GetParallelFSBucket(cs.Driver.cloud, volumeID)
118+
bucket, err := services.GetObsBucket(cs.Driver.cloud, volumeID)
119119
if err != nil {
120120
return nil, err
121121
}
@@ -216,7 +216,7 @@ func (cs *controllerServer) ValidateVolumeCapabilities(_ context.Context, req *c
216216
if len(volumeID) == 0 {
217217
return nil, status.Error(codes.InvalidArgument, "Validation failed, volume ID cannot be empty")
218218
}
219-
if _, err := services.GetParallelFSBucket(cs.Driver.cloud, volumeID); err != nil {
219+
if _, err := services.GetObsBucket(cs.Driver.cloud, volumeID); err != nil {
220220
return nil, err
221221
}
222222

@@ -263,7 +263,7 @@ func (cs *controllerServer) ControllerExpandVolume(_ context.Context, req *csi.C
263263
"Validation failed, after round-up volume size %v exceeds the max size %v", sizeBytes, maxSizeBytes)
264264
}
265265

266-
volume, err := services.GetParallelFSBucket(cc, volumeID)
266+
volume, err := services.GetObsBucket(cc, volumeID)
267267
if err != nil {
268268
return nil, err
269269
}

‎pkg/obs/mount.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,14 @@ func sendCommand(cmd CommandRPC, mountClient http.Client) error {
2626
if err != nil {
2727
return err
2828
}
29-
log.Infof("Start sending command: %s", string(marshal))
3029
response, err := mountClient.Post("http://unix", "application/json", bytes.NewReader(marshal))
3130
if err != nil {
3231
return status.Errorf(codes.Internal, "Failed to post command, err: %v", err)
3332
}
3433
defer response.Body.Close()
3534
if response.StatusCode != http.StatusOK {
3635
respBody, err := ioutil.ReadAll(response.Body)
36+
log.Infof("start to mount bucket, cmd: %v", string(respBody))
3737
if err != nil {
3838
return status.Errorf(codes.Internal, "Failed to read responseBody, err: %v", err)
3939
}

0 commit comments

Comments
 (0)