vm: drop paramiko, use opeensh and add retry

.github/workflows/tests.yml

@@ -106,7 +106,7 @@ jobs:
     - name: Install test dependencies
       run: |
         sudo apt update
-        sudo apt install -y python3-pytest python3-paramiko python3-boto3 flake8 pylint libosinfo-bin squashfs-tools
+        sudo apt install -y python3-pytest python3-boto3 flake8 pylint libosinfo-bin squashfs-tools sshpass
     - name: Diskspace (before)
       run: |
         df -h

test/test_build_iso.py

@@ -202,4 +202,4 @@ def test_bootc_installer_iso_installs(tmp_path, build_container, container_ref):
             assert exit_status == 0
             exit_status, output = vm.run("bootc status", user="root", keyfile=ssh_keyfile_private_path)
             assert exit_status == 0
-            assert f"Booted image: {container_ref}" in output
+            assert f"image: {container_ref}" in output

vmtest/vm.py

@@ -2,7 +2,6 @@
 import os
 import pathlib
 import platform
-import logging
 import shutil
 import subprocess
 import sys
@@ -12,18 +11,17 @@
 from io import StringIO
 
 import boto3
-import paramiko
 from botocore.exceptions import ClientError
-from paramiko.client import AutoAddPolicy, SSHClient
-from scp import SCPClient
 from vmtest.util import get_free_port, wait_ssh_ready
 
 AWS_REGION = "us-east-1"
 
-# XXX: find better way to control this
-if os.environ.get("OSBUILD_TEST_QEMU_VERBOSE"):
-    logging.getLogger("paramiko").setLevel(logging.DEBUG)
-    logging.getLogger("paramiko").addHandler(logging.StreamHandler(sys.stderr))
+
+_non_interactive_ssh = [
+    "-o", "UserKnownHostsFile=/dev/null",
+    "-o", "StrictHostKeyChecking=no",
+    "-o", "LogLevel=ERROR",
+]
 
 
 class VM(abc.ABC):
@@ -54,50 +52,62 @@ def force_stop(self):
         Stop the VM and clean up any resources that were created when setting up and starting the machine.
         """
 
-    def _get_ssh_transport(self, user, password="", keyfile=None):
+    def _sshpass(self, password):
+        if not password:
+            return []
+        return ["sshpass", "-p", password]
+
+    def _ensure_ssh(self, user, password="", keyfile=None):
         if not self.running():
             self.start()
-        client = SSHClient()
-        client.set_missing_host_key_policy(AutoAddPolicy)
-        # workaround, see https://github.com/paramiko/paramiko/issues/2048
-        pkey = None
-        if keyfile:
-            for klass in paramiko.key_classes:
-                try:
-                    pkey = klass.from_private_key_file(keyfile)
-                    break
-                except paramiko.ssh_exception.SSHException:
-                    continue
-            if pkey is None:
-                raise RuntimeError(f"cannot load {keyfile}, tried {paramiko.key_classes}")
-        client.connect(
-            self._address, self._ssh_port,
-            user, password, pkey=pkey,
-            allow_agent=False, look_for_keys=False)
-        return client.get_transport()
+        n_retries = 3
+        wait_sec = 10
+        for _ in range(n_retries):
+            try:
+                ret, _ = self._run("true", user, password, keyfile)
+                if ret == 0:
+                    return
+            except Exception as e:
+                print(f"ssh not ready {e}")
+                time.sleep(wait_sec)
+        raise RuntimeError(f"no ssh after {n_retries} retries of {wait_sec}")
 
     def run(self, cmd, user, password="", keyfile=None):
+        self._ensure_ssh(user, password, keyfile)
+        return self._run(cmd, user, password, keyfile)
+
+    def _run(self, cmd, user, password="", keyfile=None):
         """
         Run a command on the VM via SSH using the provided credentials.
         """
-        tr = self._get_ssh_transport(user, password, keyfile)
-        chan = tr.open_session()
-        chan.get_pty()
-        chan.exec_command(cmd)
-        stdout_f = chan.makefile()
+        ssh_cmd = self._sshpass(password) + [
+            "ssh", "-p", str(self._ssh_port),
+        ] + _non_interactive_ssh
+        if keyfile:
+            ssh_cmd.extend(["-i", keyfile])
+        ssh_cmd.append(f"{user}@{self._address}")
+        ssh_cmd.append(cmd)
         output = StringIO()
-        while True:
-            out = stdout_f.readline()
-            if not out:
-                break
-            self._log(out)
-            output.write(out)
-        exit_status = stdout_f.channel.recv_exit_status()
-        return exit_status, output.getvalue()
+        with subprocess.Popen(
+                ssh_cmd,
+                stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                text=True, bufsize=1,
+        ) as p:
+            for out in p.stdout:
+                self._log(out)
+                output.write(out)
+        return p.returncode, output.getvalue()
 
     def scp(self, src, dst, user, password="", keyfile=None):
-        with SCPClient(self._get_ssh_transport(user, password, keyfile)) as scp:
-            scp.put(src, dst)
+        self._ensure_ssh(user, password, keyfile)
+        scp_cmd = self._sshpass(password) + [
+            "scp", "-P", str(self._ssh_port),
+        ] + _non_interactive_ssh
+        if keyfile:
+            scp_cmd.extend(["-i", keyfile])
+        scp_cmd.append(src)
+        scp_cmd.append(f"{user}@{self._address}:{dst}")
+        subprocess.check_call(scp_cmd)
 
     @abc.abstractmethod
     def running(self):