From 625215ad07b435ca58f6a90ddd8701d29e08e744 Mon Sep 17 00:00:00 2001 From: Den Shchedrivyi Date: Mon, 8 Jun 2026 13:29:13 -0700 Subject: [PATCH] [4.20] [VIRT] Add SSH retry after migration/hotplug operations (#5135) Add wait_timeout=TIMEOUT_2MIN to run_ssh_commands calls in functions invoked after migration, hotplug, or pause operations to handle transient SSH connection issues. Manual cherry-pick of PR #5135 from main branch. Co-Authored-By: Claude Sonnet 4.5 Signed-off-by: Denys Shchedrivyi --- tests/utils.py | 9 ++++++--- tests/virt/utils.py | 1 + utilities/virt.py | 4 +++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index cb6806c726..84a5c6aef6 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -30,6 +30,7 @@ TCP_TIMEOUT_30SEC, TIMEOUT_1MIN, TIMEOUT_1SEC, + TIMEOUT_2MIN, TIMEOUT_3MIN, TIMEOUT_5SEC, TIMEOUT_10MIN, @@ -233,17 +234,19 @@ def get_os_cpu_count(vm): cmd = shlex.split("echo %NUMBER_OF_PROCESSORS%") else: cmd = shlex.split("nproc --all") - return int(run_ssh_commands(host=vm.ssh_exec, commands=cmd)[0].strip()) + return int(run_ssh_commands(host=vm.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN)[0].strip()) def get_os_memory_value(vm): if "windows" in vm.name: cmd = shlex.split("wmic ComputerSystem get TotalPhysicalMemory") - wmic_total_mem = run_ssh_commands(host=vm.ssh_exec, commands=cmd)[0].strip().split()[1] + wmic_total_mem = ( + run_ssh_commands(host=vm.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN)[0].strip().split()[1] + ) return f"{round(float(bitmath.Bit(int(wmic_total_mem)).to_Gib()))}Gi" else: cmd = shlex.split("awk \"'{print$2/1024/1024;exit}'\" /proc/meminfo") - meminfo = run_ssh_commands(host=vm.ssh_exec, commands=cmd)[0].strip() + meminfo = run_ssh_commands(host=vm.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN)[0].strip() return f"{round(float(meminfo))}Gi" diff --git a/tests/virt/utils.py b/tests/virt/utils.py index 90fa063763..857b969d48 100644 --- a/tests/virt/utils.py +++ b/tests/virt/utils.py @@ -113,6 +113,7 @@ def get_stress_ng_pid(ssh_exec, windows=False): host=ssh_exec, commands=shlex.split(f"{command_prefix} bash -c 'pgrep {stress}'"), tcp_timeout=TCP_TIMEOUT_30SEC, + wait_timeout=TIMEOUT_2MIN, )[0].split("\n")[0] diff --git a/utilities/virt.py b/utilities/virt.py index 9e7a709d0f..a6f1e2b66a 100644 --- a/utilities/virt.py +++ b/utilities/virt.py @@ -2362,6 +2362,7 @@ def fetch_pid_from_linux_vm(vm, process_name): cmd_res = run_ssh_commands( host=vm.ssh_exec, commands=shlex.split(f"pgrep {process_name} -x || true"), + wait_timeout=TIMEOUT_2MIN, )[0].strip() assert cmd_res, f"VM {vm.name}, '{process_name}' process not found" return int(cmd_res) @@ -2384,6 +2385,7 @@ def fetch_pid_from_windows_vm(vm, process_name): host=vm.ssh_exec, commands=shlex.split(f"powershell -Command (Get-Process -Name {process_name.removesuffix('.exe')}).Id"), tcp_timeout=TCP_TIMEOUT_30SEC, + wait_timeout=TIMEOUT_2MIN, )[0].strip() assert cmd_res, f"Process '{process_name}' not in output: {cmd_res}" return int(cmd_res) @@ -2620,7 +2622,7 @@ def validate_virtctl_guest_agent_data_over_time(vm: VirtualMachineForTests) -> b def get_vm_boot_time(vm: VirtualMachineForTests) -> str: boot_command = 'net statistics workstation | findstr "Statistics since"' if "windows" in vm.name else "who -b" - return run_ssh_commands(host=vm.ssh_exec, commands=shlex.split(boot_command))[0] + return run_ssh_commands(host=vm.ssh_exec, commands=shlex.split(boot_command), wait_timeout=TIMEOUT_2MIN)[0] def username_password_from_cloud_init(vm_volumes: list[dict[str, Any]]) -> tuple[str, str]: