fix: Retry KVM_CREATE_VM on EINTR

It is known that KVM_CREATE_VM fails with EINTR on heavily loaded
machines with many VMs. It might be a kernel bug but apparently has not
been fixed. To mitigate it, QEMU does an infinitely retry on EINTR.
Similar, do retries up to 5 times.

Signed-off-by: Takahiro Itazuri <[email protected]>

Author: zulinx86

src/vmm/src/vstate/vm/aarch64.rs

@@ -30,7 +30,7 @@ pub enum ArchVmError {
 impl ArchVm {
     /// Create a new `Vm` struct.
     pub fn new(kvm: &Kvm) -> Result<ArchVm, VmError> {
-        let fd = kvm.fd.create_vm().map_err(VmError::CreateVm)?;
+        let fd = Self::create_vm(kvm)?;
         Ok(ArchVm {
             fd,
             irqchip_handle: None,

src/vmm/src/vstate/vm/mod.rs

@@ -9,6 +9,7 @@ use kvm_bindings::{kvm_userspace_memory_region, KVM_MEM_LOG_DIRTY_PAGES};
 use kvm_ioctls::VmFd;
 use vmm_sys_util::eventfd::EventFd;
 
+use crate::logger::info;
 use crate::vstate::memory::{Address, GuestMemory, GuestMemoryMmap, GuestMemoryRegion};
 
 #[cfg(target_arch = "x86_64")]
@@ -42,6 +43,36 @@ pub enum VmError {
 
 /// Contains Vm functions that are usable across CPU architectures
 impl Vm {
+    fn create_vm(kvm: &crate::vstate::kvm::Kvm) -> Result<VmFd, VmError> {
+        // It is known that KVM_CREATE_VM spuriously fails with EINTR on heavily loaded machines
+        // with many VMs. It might be a kernel bug but apparently has not been fixed.
+        //
+        // To mitigate it, QEMU does an inifinite retry on EINTR:
+        // - https://github.com/qemu/qemu/commit/94ccff133820552a859c0fb95e33a539e0b90a75
+        // - https://github.com/qemu/qemu/commit/bbde13cd14ad4eec18529ce0bf5876058464e124
+        //
+        // Similarly, we do retries up to 5 times until the bug is fixed.
+        const MAX_ATTEMPTS: u8 = 5;
+        let mut attempts = 0;
+        loop {
+            match kvm.fd.create_vm() {
+                Ok(fd) => return Ok(fd),
+                Err(err) if err.errno() == libc::EINTR => {
+                    attempts += 1;
+                    info!("KVM_CREATE_VM returned EINTR: {attempts} of {MAX_ATTEMPTS} attempts");
+                    if attempts == MAX_ATTEMPTS {
+                        return Err(VmError::CreateVm(err));
+                    }
+                    // Exponential backoff (1us, 2us, 4us, 8us => 15ms in total)
+                    std::thread::sleep(std::time::Duration::from_micros(
+                        2u64.pow(attempts as u32 - 1),
+                    ));
+                }
+                Err(err) => return Err(VmError::CreateVm(err)),
+            }
+        }
+    }
+
     /// Creates the specified number of [`Vcpu`]s.
     ///
     /// The returned [`EventFd`] is written to whenever any of the vcpus exit.

src/vmm/src/vstate/vm/x86_64.rs

@@ -53,7 +53,8 @@ pub struct ArchVm {
 impl ArchVm {
     /// Create a new `Vm` struct.
     pub fn new(kvm: &crate::vstate::kvm::Kvm) -> Result<ArchVm, VmError> {
-        let fd = kvm.fd.create_vm().map_err(VmError::CreateVm)?;
+        let fd = Self::create_vm(kvm)?;
+
         let msrs_to_save = kvm.msrs_to_save().map_err(ArchVmError::GetMsrsToSave)?;
 
         fd.set_tss_address(u64_to_usize(crate::arch::x86_64::layout::KVM_TSS_ADDRESS))