Kubernetes GPU: ensure nvidia drivers install after docker is installed (#3517)

Author: jackfrancis

parts/k8s/kubernetesagentcustomdata.yml

@@ -171,9 +171,9 @@ runcmd:
 - . /opt/azure/containers/provision_source.sh
 - timeout 10 apt-mark hold walinuxagent{{GetKubernetesAgentPreprovisionYaml .}}
 - timeout 10 apt-mark unhold walinuxagent
+- mkdir -p /var/log/azure && touch /var/log/azure/cloud-init.complete
 {{if IsNSeriesSKU .}}
 - . /opt/azure/containers/provision_source.sh
   {{GetGPUDriversInstallScript .}}
 {{end}}
-- mkdir -p /var/log/azure && touch /var/log/azure/cloud-init.complete
 {{end}}

parts/k8s/kubernetescustomscript.sh

@@ -197,6 +197,7 @@ function installDocker() {
     apt_get_install 20 30 120 docker-engine || exit $ERR_DOCKER_INSTALL_TIMEOUT
     echo "ExecStartPost=/sbin/iptables -P FORWARD ACCEPT" >> /etc/systemd/system/docker.service.d/exec_start.conf
     usermod -aG docker ${ADMINUSER}
+    touch /var/log/azure/docker-install.complete
 }
 
 function runAptDaily() {

pkg/acsengine/engine.go

@@ -472,6 +472,7 @@ func getGPUDriversInstallScript(profile *api.AgentPoolProfile) string {
 - sh -c "echo \"blacklist nouveau\" >> /etc/modprobe.d/blacklist.conf"
 - update-initramfs -u
 - mkdir -p %s
+- wait_for_file 900 1 /var/log/azure/docker-install.complete
 - cd %s`, dest, dest)
 
 	/*
@@ -507,7 +508,7 @@ func getGPUDriversInstallScript(profile *api.AgentPoolProfile) string {
 - sh nvidia-drivers-%s --silent --accept-license --no-drm --utility-prefix="%s" --opengl-prefix="%s"
 - echo "%s" > /etc/ld.so.conf.d/nvidia.conf
 - sudo ldconfig
-- umount /usr/lib/x86_64-linux-gnu
+- umount -l /usr/lib/x86_64-linux-gnu
 - nvidia-modprobe -u -c0
 - %s/bin/nvidia-smi
 - sudo ldconfig