vmtest: add new _ensure_ssh() that will retry 3x to login

Small helper to retry ssh to avoid flakyness when a VM closes the
connection too early.

Author: mvo5

vmtest/vm.py

@@ -57,12 +57,26 @@ def _sshpass(self, password):
             return []
         return ["sshpass", "-p", password]
 
+    def _ensure_ssh(self, cmd, user, password="", keyfile=None):
+        if not self.running():
+            self.start()
+        for i in range(3):
+            try:
+                ret, _ = self._run("true", user, password, keyfile)
+                if ret == 0:
+                    return
+            except Exception as e:
+                print(f"ssh not ready {e}")
+                time.sleep(10)
+
     def run(self, cmd, user, password="", keyfile=None):
+        self._ensure_ssh(cmd, user, password, keyfile)
+        return self._run(cmd, user, password, keyfile)
+
+    def _run(self, cmd, user, password="", keyfile=None):
         """
         Run a command on the VM via SSH using the provided credentials.
         """
-        if not self.running():
-            self.start()
         ssh_cmd = self._sshpass(password) + [
             "ssh", "-p", str(self._ssh_port),
         ] + _non_interactive_ssh
@@ -82,8 +96,7 @@ def run(self, cmd, user, password="", keyfile=None):
         return p.returncode, output.getvalue()
 
     def scp(self, src, dst, user, password="", keyfile=None):
-        if not self.running():
-            self.start()
+        self._ensure_ssh(user, password, keyfile)
         scp_cmd = self._sshpass(password) + [
             "scp", "-P", str(self._ssh_port),
         ] + _non_interactive_ssh