Reboot instance if gpu is not accesible (#383)

* Reboot instance if gpu is not accesible

* tests

Author: DavidGOrtega

cml/setup.sh

@@ -11,8 +11,8 @@ if [ ! -f "$FILE" ]; then
   sudo add-apt-repository ppa:git-core/ppa -y
   sudo apt update && sudo apt-get install -y software-properties-common build-essential git
 
-  sudo curl -fsSL https://get.docker.com -o get-docker.sh && sudo sh get-docker.sh &&
-    sudo usermod -aG docker ubuntu
+  sudo curl -fsSL https://get.docker.com -o get-docker.sh && sudo sh get-docker.sh
+  sudo usermod -aG docker ubuntu
   sudo setfacl --modify user:ubuntu:rw /var/run/docker.sock
 
   curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add -

iterative/resource_runner.go

@@ -299,6 +299,8 @@ func renderScript(data map[string]interface{}) (string, error) {
 
 	tmpl, err := template.New("deploy").Funcs(template.FuncMap{"escape": shellescape.Quote}).Parse(
 		`#!/bin/sh
+sudo systemctl is-enabled cml.service && return 0
+
 {{- if not .container}}
 {{- if .setup}}{{.setup}}{{- end}}
 sudo npm config set user 0 && sudo npm install --global @dvcorg/cml
@@ -362,7 +364,11 @@ EOF'
 
 {{- if .cloud}}
 sudo systemctl daemon-reload
-sudo systemctl enable cml.service --now
+sudo systemctl enable cml.service
+{{- if .instance_gpu}}
+nvidia-smi &>/dev/null || reboot
+{{- end}}
+sudo systemctl start cml.service
 {{- end}}
 
 {{- end}}

iterative/testdata/script_template_cloud_aws.golden

@@ -1,4 +1,5 @@
 #!/bin/sh
+sudo systemctl is-enabled cml.service && return 0
 FILE=/var/log/cml_stack.log
 if [ ! -f "$FILE" ]; then
   DEBIAN_FRONTEND=noninteractive
@@ -65,4 +66,6 @@ sudo bash -c 'cat << EOF > /etc/systemd/system/cml.service
   WantedBy=default.target
 EOF'
 sudo systemctl daemon-reload
-sudo systemctl enable cml.service --now
+sudo systemctl enable cml.service
+nvidia-smi &>/dev/null || reboot
+sudo systemctl start cml.service

iterative/testdata/script_template_cloud_azure.golden

@@ -1,4 +1,5 @@
 #!/bin/sh
+sudo systemctl is-enabled cml.service && return 0
 FILE=/var/log/cml_stack.log
 if [ ! -f "$FILE" ]; then
   DEBIAN_FRONTEND=noninteractive
@@ -66,4 +67,6 @@ sudo bash -c 'cat << EOF > /etc/systemd/system/cml.service
   WantedBy=default.target
 EOF'
 sudo systemctl daemon-reload
-sudo systemctl enable cml.service --now
+sudo systemctl enable cml.service
+nvidia-smi &>/dev/null || reboot
+sudo systemctl start cml.service

iterative/testdata/script_template_cloud_gcp.golden

@@ -1,4 +1,5 @@
 #!/bin/sh
+sudo systemctl is-enabled cml.service && return 0
 FILE=/var/log/cml_stack.log
 if [ ! -f "$FILE" ]; then
   DEBIAN_FRONTEND=noninteractive
@@ -63,4 +64,6 @@ sudo bash -c 'cat << EOF > /etc/systemd/system/cml.service
   WantedBy=default.target
 EOF'
 sudo systemctl daemon-reload
-sudo systemctl enable cml.service --now
+sudo systemctl enable cml.service
+nvidia-smi &>/dev/null || reboot
+sudo systemctl start cml.service

iterative/testdata/script_template_cloud_invalid.golden

@@ -1,4 +1,5 @@
 #!/bin/sh
+sudo systemctl is-enabled cml.service && return 0
 FILE=/var/log/cml_stack.log
 if [ ! -f "$FILE" ]; then
   DEBIAN_FRONTEND=noninteractive
@@ -62,4 +63,6 @@ sudo bash -c 'cat << EOF > /etc/systemd/system/cml.service
   WantedBy=default.target
 EOF'
 sudo systemctl daemon-reload
-sudo systemctl enable cml.service --now
+sudo systemctl enable cml.service
+nvidia-smi &>/dev/null || reboot
+sudo systemctl start cml.service

iterative/testdata/script_template_cloud_kubernetes.golden

@@ -1,4 +1,5 @@
 #!/bin/sh
+sudo systemctl is-enabled cml.service && return 0
 export KUBERNETES_CONFIGURATION='8 value with "quotes" and spaces'
 
 HOME="$(mktemp -d)" exec cml-runner \