From 8264c0d73f07f6c3e6768d5a41b56c621f912a37 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 10:44:39 -0700 Subject: [PATCH 01/29] KVM: Reject attempts to lock all vCPUs if vCPU creation is in-progress Reject locking of all vCPUs if vCPU creation is in-progress, i.e. if the number of "created" vCPUs doesn't match the number of "onlined" vCPUs. It's simply not possible to guarantee that KVM has truly locked all vCPUs if one or more vCPUs are actively being created. Holding kvm->lock does prevent in-flight vCPUs from being fully onlined, but it's infeasible for common KVM to know whether or not that provides sufficient protection. In practice, this is likely a minor bug fix for the ARM and RISC-V usage of kvm_trylock_all_vcpus(), and a glorified nop for everything else. E.g. ARM's kvm_timer_vcpu_init() can race kvm_vm_ioctl_set_counter_offset() with respect to observing KVM_ARCH_FLAG_VM_COUNTER_OFFSET. Opportunistically drop x86's existing manual checks on vCPU creation being in-progress as all of x86's checks immediately precede or follow locking of all vCPUs. Leave arm64 and RISC-V alone for the moment, as their checks aren't as obviously redundant/equivalent. Signed-off-by: Sean Christopherson Tested-by: Jean-Christophe Guillain Tested-by: Naveen N Rao (AMD) Message-ID: <20260921174445.911676-2-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/sev.c | 10 ---------- arch/x86/kvm/vmx/tdx.c | 5 ----- virt/kvm/kvm_main.c | 6 ++++++ 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 63eb2155a774..6aa86a78711e 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -1125,9 +1125,6 @@ static int sev_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) if (!sev_es_guest(kvm)) return -ENOTTY; - if (kvm_is_vcpu_creation_in_progress(kvm)) - return -EBUSY; - ret = kvm_lock_all_vcpus(kvm); if (ret) return ret; @@ -2121,10 +2118,6 @@ static int sev_check_source_vcpus(struct kvm *dst, struct kvm *src) struct kvm_vcpu *src_vcpu; unsigned long i; - if (kvm_is_vcpu_creation_in_progress(src) || - kvm_is_vcpu_creation_in_progress(dst)) - return -EBUSY; - if (!sev_es_guest(src)) return 0; @@ -2520,9 +2513,6 @@ static int snp_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) unsigned long i; int ret; - if (kvm_is_vcpu_creation_in_progress(kvm)) - return -EBUSY; - ret = kvm_lock_all_vcpus(kvm); if (ret) return ret; diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 6c842e9191a5..00850a3a77d6 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -2728,11 +2728,6 @@ static tdx_vm_state_guard_t tdx_acquire_vm_state_locks(struct kvm *kvm) mutex_lock(&kvm->lock); - if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus)) { - r = -EBUSY; - goto out_err; - } - r = kvm_lock_all_vcpus(kvm); if (r) goto out_err; diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 85f42289748d..aad8cf00a1c8 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1363,6 +1363,9 @@ int kvm_trylock_all_vcpus(struct kvm *kvm) lockdep_assert_held(&kvm->lock); + if (kvm_is_vcpu_creation_in_progress(kvm)) + return -EBUSY; + kvm_for_each_vcpu(i, vcpu, kvm) if (!mutex_trylock_nest_lock(&vcpu->mutex, &kvm->lock)) goto out_unlock; @@ -1386,6 +1389,9 @@ int kvm_lock_all_vcpus(struct kvm *kvm) lockdep_assert_held(&kvm->lock); + if (kvm_is_vcpu_creation_in_progress(kvm)) + return -EBUSY; + kvm_for_each_vcpu(i, vcpu, kvm) { r = mutex_lock_killable_nest_lock(&vcpu->mutex, &kvm->lock); if (r) From e3995b05e95d03953954dd60ba803ebfa7755253 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 10:44:40 -0700 Subject: [PATCH 02/29] KVM: arm64: vgic: Rely on vCPU creation check in "trylock all vCPUs" Now that KVM's APIs for locking all vCPUs return -EBUSY if vCPU creation is in-progress, drop the manual check for the same from vGIC creation, and update the comments accordingly. Note, while KVM arm64 guards many vGIC operations with its arch-specific config_lock, holding kvm->lock is sufficient to guarantee a stable result for "is vCPU creation in-progress". So, no functional change intended. Note #2, the open coded check in vgic_init() is racy when called without kvm->lock held, e.g. via vgic_lazy_init(). I.e. that check needs to stay open coded to avoid triggering a lockdep assert. Whether or not the race is "fine" is a problem for a different day. Signed-off-by: Sean Christopherson Tested-by: Jean-Christophe Guillain Tested-by: Naveen N Rao (AMD) Message-ID: <20260921174445.911676-3-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/arm64/kvm/vgic/vgic-init.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 4012df6002ea..a58575df36e9 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -97,6 +97,9 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) /* * - Acquiring the vCPU mutex for every *online* vCPU to prevent * concurrent vCPU ioctls for vCPUs already visible to userspace. + * This also ensures KVM isn't in the middle of creating a vCPU, + * i.e. that there are no vCPUs that have been created but aren't + * yet fully online. */ ret = -EBUSY; if (kvm_trylock_all_vcpus(kvm)) @@ -105,18 +108,11 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) /* * - Taking the config_lock which protects VGIC data structures such * as the per-vCPU arrays of private IRQs (SGIs, PPIs). - */ - mutex_lock(&kvm->arch.config_lock); - - /* - * - Bailing on the entire thing if a vCPU is in the middle of creation, - * dropped the kvm->lock, but hasn't reached kvm_arch_vcpu_create(). * * The whole combination of this guarantees that no vCPU can get into * KVM with a VGIC configuration inconsistent with the VM's VGIC. */ - if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus)) - goto out_unlock; + mutex_lock(&kvm->arch.config_lock); if (irqchip_in_kernel(kvm)) { ret = -EEXIST; From 10f4082b754f18ed82f94d47f0e959ef80038895 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 10:44:41 -0700 Subject: [PATCH 03/29] KVM: RISC-V: Use kvm_is_vcpu_creation_in_progress() instead of open-coded equivalent Use kvm_is_vcpu_creation_in_progress() instead of an open-coded equivalent during AIA initialization. Unlike similar vGIC code in arm64, the relevant RISC-V code runs under kvm->lock, i.e. can use the standard API without hitting lockdep false positive. No functional change intended. Signed-off-by: Sean Christopherson Tested-by: Jean-Christophe Guillain Tested-by: Naveen N Rao (AMD) Message-ID: <20260921174445.911676-4-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/riscv/kvm/aia_device.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/kvm/aia_device.c b/arch/riscv/kvm/aia_device.c index efc7c0bcfba9..97833a04268a 100644 --- a/arch/riscv/kvm/aia_device.c +++ b/arch/riscv/kvm/aia_device.c @@ -237,7 +237,7 @@ static int aia_init(struct kvm *kvm) return -EBUSY; /* We might be in the middle of creating a VCPU? */ - if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus)) + if (kvm_is_vcpu_creation_in_progress(kvm)) return -EBUSY; /* Number of sources should be less than or equals number of IDs */ From 6418b715d5a24fa952e8c4baa587dd812dd4765b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 10:44:42 -0700 Subject: [PATCH 04/29] KVM: Protect all of kvm_vm_ioctl_create_vcpu() with kvm->lock When creating a vCPU, don't drop kvm->lock to when doing the bulk of actual vCPU creation, as allowing multiple vCPUs to be created in parallel adds significant complexity in KVM (as evidenced by the many related bugs), and all known VMMs fully serialize vCPU creation. Remove all manually locking of kvm->lock from kvm_arch_vcpu_{post,}create() for obvious reasons. For many years, "everyone" has assumed that dropping kvm->lock was done for performance reasons optimization, e.g. to allow userspace to create all vCPUs concurrently for latency purposes. But as above, no known VMM does that. Looking at the history of this code, before commit 11ec28047118 ("KVM: Convert vm lock to a mutex"), kvm->lock was a spinlock. I.e. KVM *had* to drop kvm->lock when doing the bulk of vCPU creation, otherwise KVM couldn't do normal memory allocations. When kvm->lock got turned into a mutex for unrelated reasons, no one took advantage updated of the change to simplify vCPU creation. And 19 years later, everyone just assumed that KVM continued to deal with the complexity for performance reasons. Furthermore, naively parallelizing vCPU creation in userspace is likely a net negative due to the overheads of task creation. Unless a VMM carefully avoids the extra overhead related to parallelization, e.g. spawns each vCPU's thread before creating the vCPU, creating vCPUs concurrently is a net *negative* up until about ~64 vCPUs, after which the times are a wash. The absolute speed of light _is_ faster if KVM doesn't hold kvm-lock, but at vCPU counts of ~16 or less, it's probably in the noise when considering total VM creation time, as the added latency is less than 1ms up until 16 or so vCPUs. On top of all that, KVM has had a *lot* of fatal bugs (most often found by syzkaller) related to vCPUs being created while trying to do per-VM operations (basically, see every flow that locks all vCPUs). I.e. the parallel vCPU creation "support" is actively harmful as the only "use case" is for misbehaving userspace to exploit KVM bugs. Serializing vCPU creation will allow reverting commit 97d65b544f48 ("KVM: Check for duplicate vcpu_id as early as possible"), which had "minor" math error: the worst case scenario isn't "256 bytes per VM", it's "256 unsigned longs per VM", i.e. 2048 bytes per VM, which doubles the size of each VM and pushes several architectures into order-1 allocations. Signed-off-by: Sean Christopherson Tested-by: Jean-Christophe Guillain Tested-by: Naveen N Rao (AMD) Message-ID: <20260921174445.911676-5-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/powerpc/kvm/book3s_hv.c | 2 -- arch/s390/kvm/s390/s390.c | 5 +---- virt/kvm/kvm_main.c | 22 +++++----------------- 3 files changed, 6 insertions(+), 23 deletions(-) diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index dbac3573b2c8..de05d721edc4 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -3058,7 +3058,6 @@ static int kvmppc_core_vcpu_create_hv(struct kvm_vcpu *vcpu) init_waitqueue_head(&vcpu->arch.cpu_run); - mutex_lock(&kvm->lock); vcore = NULL; err = -EINVAL; if (cpu_has_feature(CPU_FTR_ARCH_300)) { @@ -3091,7 +3090,6 @@ static int kvmppc_core_vcpu_create_hv(struct kvm_vcpu *vcpu) mutex_unlock(&kvm->arch.mmu_setup_lock); } } - mutex_unlock(&kvm->lock); if (!vcore) return err; diff --git a/arch/s390/kvm/s390/s390.c b/arch/s390/kvm/s390/s390.c index efd77b9cd36c..e4912db67adc 100644 --- a/arch/s390/kvm/s390/s390.c +++ b/arch/s390/kvm/s390/s390.c @@ -3579,12 +3579,11 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) { - mutex_lock(&vcpu->kvm->lock); preempt_disable(); vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch; vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx; preempt_enable(); - mutex_unlock(&vcpu->kvm->lock); + if (!kvm_is_ucontrol(vcpu->kvm)) { vcpu->arch.gmap = vcpu->kvm->arch.gmap; sca_add_vcpu(vcpu); @@ -3757,13 +3756,11 @@ static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu) kvm_s390_vcpu_pci_setup(vcpu); - mutex_lock(&vcpu->kvm->lock); if (kvm_s390_pv_is_protected(vcpu->kvm)) { rc = kvm_s390_pv_create_cpu(vcpu, &uvrc, &uvrrc); if (rc) kvm_s390_vcpu_unsetup_cmma(vcpu); } - mutex_unlock(&vcpu->kvm->lock); return rc; } diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index aad8cf00a1c8..cae0a9cdc33e 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -4188,6 +4188,8 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) struct kvm_vcpu *vcpu; struct page *page; + guard(mutex)(&kvm->lock); + /* * KVM tracks vCPU IDs as 'int', be kind to userspace and reject * too-large values instead of silently truncating. @@ -4200,26 +4202,18 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) if (id >= KVM_MAX_VCPU_IDS) return -EINVAL; - mutex_lock(&kvm->lock); - if (kvm->created_vcpus >= kvm->max_vcpus) { - mutex_unlock(&kvm->lock); + if (kvm->created_vcpus >= kvm->max_vcpus) return -EINVAL; - } - if (test_bit(id, kvm->vcpu_ids)) { - mutex_unlock(&kvm->lock); + if (test_bit(id, kvm->vcpu_ids)) return -EEXIST; - } r = kvm_arch_vcpu_precreate(kvm, id); - if (r) { - mutex_unlock(&kvm->lock); + if (r) return r; - } kvm->created_vcpus++; __set_bit(id, kvm->vcpu_ids); - mutex_unlock(&kvm->lock); vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL_ACCOUNT); if (!vcpu) { @@ -4250,8 +4244,6 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) goto arch_vcpu_destroy; } - mutex_lock(&kvm->lock); - if (WARN_ON_ONCE(kvm_get_vcpu_by_id(kvm, id))) { r = -EEXIST; goto unlock_vcpu_destroy; @@ -4290,7 +4282,6 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) atomic_inc(&kvm->online_vcpus); mutex_unlock(&vcpu->mutex); - mutex_unlock(&kvm->lock); kvm_arch_vcpu_postcreate(vcpu); kvm_create_vcpu_debugfs(vcpu); return r; @@ -4301,7 +4292,6 @@ kvm_put_xa_erase: xa_erase(&kvm->vcpu_array, vcpu->vcpu_idx); unlock_vcpu_destroy: vcpu->vcpu_idx = -1; - mutex_unlock(&kvm->lock); kvm_dirty_ring_free(&vcpu->dirty_ring); arch_vcpu_destroy: kvm_arch_vcpu_destroy(vcpu); @@ -4310,10 +4300,8 @@ vcpu_free_run_page: vcpu_free: kmem_cache_free(kvm_vcpu_cache, vcpu); vcpu_decrement: - mutex_lock(&kvm->lock); kvm->created_vcpus--; __clear_bit(id, kvm->vcpu_ids); - mutex_unlock(&kvm->lock); return r; } From c5866d42f7116ab609e3ab6b4b699282e8287e3f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 10:44:43 -0700 Subject: [PATCH 05/29] KVM: Move check for existing vCPU ID to the top of vCPU creation Now that kvm->lock is held for the entirety of vCPU creation, check for a conflicting vCPU ID at the begnning of vCPU creation, before the arch precreate() hook is invoked. This will allow reverting commit 97d65b544f48 ("KVM: Check for duplicate vcpu_id as early as possible"). For now, keep the redundant vcpu_ids tracking as a sanity check. No functional change intended (absent KVM bugs, checking vcpu_ids and walking kvm_get_vcpu_by_id() should yield the same result). Signed-off-by: Sean Christopherson Tested-by: Jean-Christophe Guillain Tested-by: Naveen N Rao (AMD) Message-ID: <20260921174445.911676-6-seanjc@google.com> Signed-off-by: Paolo Bonzini --- virt/kvm/kvm_main.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index cae0a9cdc33e..0e5b639559ba 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -4205,7 +4205,10 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) if (kvm->created_vcpus >= kvm->max_vcpus) return -EINVAL; - if (test_bit(id, kvm->vcpu_ids)) + if (kvm_get_vcpu_by_id(kvm, id)) + return -EEXIST; + + if (WARN_ON_ONCE(test_bit(id, kvm->vcpu_ids))) return -EEXIST; r = kvm_arch_vcpu_precreate(kvm, id); @@ -4244,11 +4247,6 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) goto arch_vcpu_destroy; } - if (WARN_ON_ONCE(kvm_get_vcpu_by_id(kvm, id))) { - r = -EEXIST; - goto unlock_vcpu_destroy; - } - /* * Set the vCPU's index *before* the vCPU is reachable by other tasks. * Unwind the index back to -1 on failure so that KVM can use the index From 700f8d24436c45cb3cc8956f35c90a1b6a6487c2 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 10:44:44 -0700 Subject: [PATCH 06/29] Revert "KVM: Check for duplicate vcpu_id as early as possible" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that KVM uses kvm_get_vcpu_by_id() to check for an existing vCPU ID before doing any meaningful work, which was made possible by holding kvm->lock for the entirety of vCPU creation, revert the now-redundant "early" vCPU ID tracking. The claims about the impact of kvm->vcpu_ids on the memory footprint were a wee bit wrong: the worst case scenario isn't 256 bytes per VM, it's 256 "unsigned longs" per VM, i.e. 2048 bytes per VM. Increasing the size of "struct kvm" by 2048 nearly doubled the total size on many architectures, and tripped x86's KVM_SANITY_CHECK_VM_STRUCT_SIZE, which was added to detect this *exact* scenario, where a single change significantly increased the size of "struct kvm". I.e. attempting to build KVM with CONFIG_DEBUG_KERNEL=n fails on x86 (the build failures got missed because all build bots apparently test only CONFIG_DEBUG_KERNEL=y kernels, and maintainers' test flows were similarly lacking). This reverts commit 97d65b544f48b2ee49f6aea32145e3e7969955dc. Fixes: 97d65b544f48 ("KVM: Check for duplicate vcpu_id as early as possible") Reported-by: Jean-Christophe Guillain Closes: https://lore.kernel.org/all/56a4bc35ee605588b7cc36c8e45c12b5f3b506cb.camel@guillain.net Reported-by: Paweł S Closes: https://lore.kernel.org/all/CABD%3DWFOS4j4hDv%2BpW-eEM9HAM2q2GY_iYdAG%2BqvYcUEinUrcQQ@mail.gmail.com Tested-by: Jean-Christophe Guillain Signed-off-by: Sean Christopherson Tested-by: Naveen N Rao (AMD) Message-ID: <20260921174445.911676-7-seanjc@google.com> Signed-off-by: Paolo Bonzini --- include/linux/kvm_host.h | 1 - virt/kvm/kvm_main.c | 5 ----- 2 files changed, 6 deletions(-) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index cf7fe835c4ad..c844def6dd95 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -791,7 +791,6 @@ struct kvm { /* The current active memslot set for each address space */ struct kvm_memslots __rcu *memslots[KVM_MAX_NR_ADDRESS_SPACES]; struct xarray vcpu_array; - DECLARE_BITMAP(vcpu_ids, KVM_MAX_VCPU_IDS); /* * Protected by slots_lock, but can be read outside if an * incorrect answer is acceptable. diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 0e5b639559ba..02bab6923e03 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -4208,15 +4208,11 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) if (kvm_get_vcpu_by_id(kvm, id)) return -EEXIST; - if (WARN_ON_ONCE(test_bit(id, kvm->vcpu_ids))) - return -EEXIST; - r = kvm_arch_vcpu_precreate(kvm, id); if (r) return r; kvm->created_vcpus++; - __set_bit(id, kvm->vcpu_ids); vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL_ACCOUNT); if (!vcpu) { @@ -4299,7 +4295,6 @@ vcpu_free: kmem_cache_free(kvm_vcpu_cache, vcpu); vcpu_decrement: kvm->created_vcpus--; - __clear_bit(id, kvm->vcpu_ids); return r; } From 41e005e2c5126ccb906d6374b3b97fcb223f7f8f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 10:44:45 -0700 Subject: [PATCH 07/29] KVM: WARN if vCPU creation is in-progress when locking all vCPUs Now that KVM holds kvm->lock for the entirety of vCPU creation, from when created_vcpus is incremented until the new vCPU is fully onlined, WARN if the impossible happens and KVM somehow sees a discrepancy between the number of vCPUs "created" and "onlined". Signed-off-by: Sean Christopherson Tested-by: Jean-Christophe Guillain Tested-by: Naveen N Rao (AMD) Message-ID: <20260921174445.911676-8-seanjc@google.com> Signed-off-by: Paolo Bonzini --- virt/kvm/kvm_main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 02bab6923e03..5a782849521c 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1363,7 +1363,7 @@ int kvm_trylock_all_vcpus(struct kvm *kvm) lockdep_assert_held(&kvm->lock); - if (kvm_is_vcpu_creation_in_progress(kvm)) + if (WARN_ON_ONCE(kvm_is_vcpu_creation_in_progress(kvm))) return -EBUSY; kvm_for_each_vcpu(i, vcpu, kvm) @@ -1389,7 +1389,7 @@ int kvm_lock_all_vcpus(struct kvm *kvm) lockdep_assert_held(&kvm->lock); - if (kvm_is_vcpu_creation_in_progress(kvm)) + if (WARN_ON_ONCE(kvm_is_vcpu_creation_in_progress(kvm))) return -EBUSY; kvm_for_each_vcpu(i, vcpu, kvm) { From b6249c1d5665098c8bf12f8d3e4712bf04c775bc Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 25 Sep 2026 16:42:56 -0700 Subject: [PATCH 08/29] KVM: x86/mmu: Bail from shadow walks if the root is invalid or a dummy When walking shadow page tables, immediately terminate the walk if the root is a "dummy" root, i.e. a root whose top-level page table is backed by the zero page, but otherwise doesn't exist. If memslot creation races with a stage-2 page fault (EPT violation or #NPF) from L2, then if the stars align, KVM will attempt to walk shadow page tables using the zero page and hit a NULL pointer deref. BUG: kernel NULL pointer dereference, address: 0000000000000021 #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page PGD 0 P4D 0 Oops: Oops: 0000 [#1] SMP CPU: 30 UID: 1000 PID: 941 Comm: qemu Not tainted 7.2.0-rc2-1b731e5ded48-next-vm #1741 PREEMPTLAZY Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 0.0.0 02/06/2015 RIP: 0010:__kvm_mmu_invalidate_addr+0xea/0x210 [kvm] Call Trace: kvm_mmu_invalidate_addr+0x92/0xe0 [kvm] __kvm_inject_emulated_page_fault+0x67/0x80 [kvm] ept_page_fault+0x160/0x850 [kvm] kvm_mmu_do_page_fault+0x102/0x1f0 [kvm] kvm_mmu_page_fault+0x8e/0x6b0 [kvm] vmx_handle_exit+0x163/0x640 [kvm_intel] kvm_arch_vcpu_ioctl_run+0x960/0x2120 [kvm] kvm_vcpu_ioctl+0x2c7/0x970 [kvm] __x64_sys_ioctl+0x90/0xd0 do_syscall_64+0x67/0x5f0 entry_SYSCALL_64_after_hwframe+0x4b/0x53 RIP: 0033:0x7f606d0b53bb Opportunistically harden the shadow walks against fully invalid roots, but WARN, as all callers are expected/required to pre-check for a valid root. Don't WARN in the dummy root case as the whole point of using a dummy root is to provide a root that's valid enough to enter the guest, i.e. it should Just Work for all flows except those that *need* to know about dummy roots. Alternatively, KVM could allocate a dedicated page and associated shadow page structure for the dummy root, which is very tempting as it such an approach should be more resilient against unexpected behavior. But that would be a much larger and thus riskier change than simply terminating walks of dummy roots. Fixes: 0e3223d8d00a ("KVM: x86/mmu: Use dummy root, backed by zero page, for !visible guest roots") Reported-by: Gabriel Schneider Cc: stable@vger.kernel.org Signed-off-by: Sean Christopherson Message-ID: <20260925234256.2384816-1-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 8e62476e477b..a0d608e3fceb 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -2512,6 +2512,12 @@ static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterato iterator->addr = addr; iterator->shadow_addr = root; + + if (WARN_ON_ONCE(!VALID_PAGE(root)) || kvm_mmu_is_dummy_root(root)) { + iterator->level = 0; + return; + } + iterator->level = vcpu->arch.mmu->root_role.level; if (iterator->level >= PT64_ROOT_4LEVEL && From 60d93c27859fb3ec5b5a689e24de303f166477d0 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 4 Sep 2026 10:06:39 -0700 Subject: [PATCH 09/29] KVM: SVM: Preserve TLB control (i.e. pending TLB flush) on failed VMRUN Don't reset the VMCB's TLB control back to "do nothing" on a failed VMRUN, as empirical testing shows that the CPU performs the requested TLB flush if and only if VMRUN is successful, i.e. clearing TLB control on a failed VMRUN effectively drops a TLB flush. Explicitly track the need to flush all ASIDs on a per-CPU basis, as the ASID reuse condition is tied to the pCPU, not to the vCPU. As a bonus, this also obviates the need to avoid clobbering FLUSH_ALL_ASID with TLB_CONTROL_FLUSH_ASID, e.g. in svm_flush_tlb_asid(). Deliberately don't bother saving/restoring the "old" tlb_ctl on failure, in quotes because it's not exactly the old tlb_ctl, it's the tlb_ctl from after pre_svm_run(), but before updating tlb_ctl for flush_all_asids. If VMRUN fails and TLB_CONTROL_FLUSH_ALL_ASID is forced, then the next successful run of the VMCB *may* unnecessarily flush all ASIDs, which strictly speaking could result in noisy neighbor issues. However, the fact that new_asid() is already guest-triggerable, because of KVM's flawed behavior of clearing the ASID on emulated INIT, means that a guest can already trigger a flush of all ASIDs at roughly the same rate. And once KVM stops clobbering the ASID on emulated INIT, *or* assigns a static ASID to each vCPU, this flaw goes away. Fixes: 38e5e92fe8c0 ("KVM: SVM: Implement Flush-By-Asid feature") Cc: stable@vger.kernel.org Reported-by: Stefan Teodorescu Suggested-by: Yosry Ahmed Cc: Tom Lendacky Cc: Jim Mattson Signed-off-by: Sean Christopherson Message-ID: <20260904170642.3291466-2-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/svm.c | 13 ++++++++++--- arch/x86/kvm/svm/svm.h | 1 + 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 7d59d301e1e5..afbaaaab84ed 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -1902,8 +1902,7 @@ static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd) if (sd->next_asid > sd->max_asid) { ++sd->asid_generation; sd->next_asid = sd->min_asid; - svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID; - vmcb_mark_dirty(svm->vmcb, VMCB_ASID); + sd->flush_all_asids = true; } svm->current_vmcb->asid_generation = sd->asid_generation; @@ -4528,6 +4527,11 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) svm->vmcb->control.asid = svm->asid; vmcb_mark_dirty(svm->vmcb, VMCB_ASID); } + if (this_cpu_ptr(&svm_data)->flush_all_asids) { + svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID; + vmcb_mark_dirty(svm->vmcb, VMCB_ASID); + } + svm->vmcb->save.cr2 = vcpu->arch.cr2; if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS) && @@ -4618,7 +4622,10 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) vcpu->arch.nested_run_pending = 0; } - svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; + if (!svm_is_vmrun_failure(svm->vmcb->control.exit_code)) { + this_cpu_ptr(&svm_data)->flush_all_asids = false; + svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; + } /* * Unconditionally mask off the CLEAR_RAP bit, the AND is just as cheap diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index e958943b8162..84f19026d3e8 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -376,6 +376,7 @@ struct svm_cpu_data { u32 next_asid; u32 min_asid; + bool flush_all_asids; bool bp_spec_reduce_set; struct vmcb *save_area; From e27fc915c8f575ab5630e27d97465c2077d4b9d1 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 4 Sep 2026 10:06:40 -0700 Subject: [PATCH 10/29] KVM: SVM: Update control fields on #VMEXIT if and only if VMRUN succeeded Leave control.erap_ctl and control.clean as-is in the VMCS if VMRUN fails, because as per AMD: there's no explicit architectural guarantee about the behavior in the presence of VMRUN failures. So the best thing to do would be to assume that if VMRUN fails, the actions requested in the control fields may not have been performed. Cc: stable@vger.kernel.org Signed-off-by: Sean Christopherson Message-ID: <20260904170642.3291466-3-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/svm.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index afbaaaab84ed..b63e7c69aa1c 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -4625,17 +4625,17 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) if (!svm_is_vmrun_failure(svm->vmcb->control.exit_code)) { this_cpu_ptr(&svm_data)->flush_all_asids = false; svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; + + /* + * Unconditionally mask off the CLEAR_RAP bit, the AND is just + * as cheap as the TEST+Jcc to avoid it. + */ + if (cpu_feature_enabled(X86_FEATURE_ERAPS)) + svm->vmcb->control.erap_ctl &= ~ERAP_CONTROL_CLEAR_RAP; + + vmcb_mark_all_clean(svm->vmcb); } - /* - * Unconditionally mask off the CLEAR_RAP bit, the AND is just as cheap - * as the TEST+Jcc to avoid it. - */ - if (cpu_feature_enabled(X86_FEATURE_ERAPS)) - svm->vmcb->control.erap_ctl &= ~ERAP_CONTROL_CLEAR_RAP; - - vmcb_mark_all_clean(svm->vmcb); - /* if exit due to PF check for async PF */ if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) vcpu->arch.apf.host_apf_flags = From 64c5b15af71ba3c26b2088bdbb0bbab3dbe58e2c Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 4 Sep 2026 10:06:41 -0700 Subject: [PATCH 11/29] KVM: SVM: Don't mark ASID fields as dirty when setting control.tlb_ctl Don't mark the ASID as dirty in the VMCB when requesting a TLB flush via control.tlb_ctl. Per "15.15.3 VMCB Clean Field" of the July 2026, Revision 3.45 version of the APM: The following are explicitly not cached and not represented by Clean bits: * TLB_Control Fixes: 7e8e6eed75e2 ("KVM: SVM: Move asid to vcpu_svm") Suggested-by: Yosry Ahmed Signed-off-by: Sean Christopherson Message-ID: <20260904170642.3291466-4-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/sev.c | 1 - arch/x86/kvm/svm/svm.c | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 6aa86a78711e..3448d56520c6 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -3620,7 +3620,6 @@ int pre_sev_run(struct vcpu_svm *svm, int cpu) sd->sev_vmcbs[asid] = svm->vmcb; svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID; - vmcb_mark_dirty(svm->vmcb, VMCB_ASID); return 0; } diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index b63e7c69aa1c..830ace75e986 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -4527,10 +4527,8 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) svm->vmcb->control.asid = svm->asid; vmcb_mark_dirty(svm->vmcb, VMCB_ASID); } - if (this_cpu_ptr(&svm_data)->flush_all_asids) { + if (this_cpu_ptr(&svm_data)->flush_all_asids) svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID; - vmcb_mark_dirty(svm->vmcb, VMCB_ASID); - } svm->vmcb->save.cr2 = vcpu->arch.cr2; From 73392706d0e0f3aff299321de41d73abe3f53d9d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 4 Sep 2026 10:06:42 -0700 Subject: [PATCH 12/29] KVM: SVM: Sync guest's PERF_CNTR_GLOBAL_CTL from h/w only on successful VMRUN Don't (re)read PERF_CNTR_GLOBAL_CTL from hardware on a failed VMRUN, as the purpose of the read is to synchronize KVM's cache with any writes done by the guest, and the guest can't possibly have modified the MSR if it never got a chance to run. Signed-off-by: Sean Christopherson Message-ID: <20260904170642.3291466-5-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/svm.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 830ace75e986..f4f488328ea4 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -4632,6 +4632,10 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) svm->vmcb->control.erap_ctl &= ~ERAP_CONTROL_CLEAR_RAP; vmcb_mark_all_clean(svm->vmcb); + + if (!msr_write_intercepted(svm, MSR_AMD64_PERF_CNTR_GLOBAL_CTL)) + rdmsrq(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, + vcpu_to_pmu(vcpu)->global_ctrl); } /* if exit due to PF check for async PF */ @@ -4641,9 +4645,6 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) kvm_clear_available_registers(vcpu, SVM_REGS_LAZY_LOAD_SET); - if (!msr_write_intercepted(svm, MSR_AMD64_PERF_CNTR_GLOBAL_CTL)) - rdmsrq(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, vcpu_to_pmu(vcpu)->global_ctrl); - trace_kvm_exit(vcpu, KVM_ISA_SVM); svm_complete_interrupts(vcpu); From 9e06bbb9ade7cdf28d4a3b5b14e5812881c8df85 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 26 Aug 2026 12:58:33 -0700 Subject: [PATCH 13/29] KVM: SVM: Use the active VMCB's MSR bitmap when checking if MSR is intercepted Use the MSR permission bitmap of the active VMCB instead of assuming that KVM is always using vmcb02's bitmap when L2 is active, as KVM uses msrpm02 if and only if L1 wants to intercept MSR accesses, i.e. if and only if KVM needs to merge msprm01 with msrpm12. Don't bother tracking the virtual address of the bitmap that's being used, as __va() is cheap on x86, and caching the virtual address would introduce yet another source of potentially stale information. Fixes: b2ac58f90540 ("KVM/SVM: Allow direct access to MSR_IA32_SPEC_CTRL") Cc: stable@vger.kernel.org Reported-by: Stefan Teodorescu Signed-off-by: Sean Christopherson Message-ID: <20260826195833.844526-1-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/svm.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index f4f488328ea4..fe0cba4731b2 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -673,16 +673,7 @@ static void clr_dr_intercepts(struct vcpu_svm *svm) static bool msr_write_intercepted(struct vcpu_svm *svm, u32 msr) { - /* - * For non-nested case: - * If the L01 MSR bitmap does not intercept the MSR, then we need to - * save it. - * - * For nested case: - * If the L02 MSR bitmap does not intercept the MSR, then we need to - * save it. - */ - void *msrpm = is_guest_mode(&svm->vcpu) ? svm->nested.msrpm : svm->msrpm; + void *msrpm = __va(__sme_clr(svm->vmcb->control.msrpm_base_pa)); return svm_test_msr_bitmap_write(msrpm, msr); } From 2abb25273c82061dd4670690c81c319c03740bf6 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 13 Aug 2026 14:44:03 -0700 Subject: [PATCH 14/29] KVM: nVMX: Force MSR bitmap refresh if runtime eVMCS controls are modified Force a refresh of the vmcs02 MSR bitmap during nested VM-Enter if the runtime eVMCS controls (pin, primary, secondary, etc.) are being updated. If L1 isn't intercepting TPR writes, runs L2 with TPR virtualization, and then runs the same L2 with TPR virtualization disabled, KVM will fail to refresh msr_bitmap02 and leave TPR in passthrough mode even though TPR virtualization is disabled. I.e. failure to refresh the bitmap lets L2 (or L1 by proxy) read and write L0's TPR. Fixes: 502d2bf5f2fd ("KVM: nVMX: Implement Enlightened MSR Bitmap feature") Cc: stable@vger.kernel.org Reviewed-by: Vitaly Kuznetsov Signed-off-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 40c1a5f6fa8a..b25216862740 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -1754,6 +1754,9 @@ static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx) static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields) { #ifdef CONFIG_KVM_HYPERV + const u64 runtime_controls = HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1 | + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2 | + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC; struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx); struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(&vmx->vcpu); @@ -1762,6 +1765,9 @@ static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields vmcs12->tpr_threshold = evmcs->tpr_threshold; vmcs12->guest_rip = evmcs->guest_rip; + if ((hv_clean_fields & runtime_controls) != runtime_controls) + vmx->nested.force_msr_bitmap_recalc = true; + if (unlikely(!(hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_ENLIGHTENMENTSCONTROL))) { hv_vcpu->nested.pa_page_gpa = evmcs->partition_assist_page; From 8916c7a87455370b25523d5b1b9015b5244d18c8 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 10 Jul 2026 09:20:52 -0700 Subject: [PATCH 15/29] KVM: selftests: Add x2APIC MSR test for inhibiting APICv while nested Add a selftest to verify that KVM intercepts x2APIC MSR accesses for L1 after APICv is inhibited while L2 is active. This is a regression test for an AVIC bug where KVM would skip updating x2APIC MSR intercepts while L2 is active, thus giving L1 access to a wide swath of L0's x2APIC surface. Signed-off-by: Sean Christopherson Message-ID: <20260710162052.2188574-3-seanjc@google.com> Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/Makefile.kvm | 1 + .../selftests/kvm/x86/nested_x2apic_test.c | 117 ++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 tools/testing/selftests/kvm/x86/nested_x2apic_test.c diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 6a1482e3a286..752f81b070fa 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -103,6 +103,7 @@ TEST_GEN_PROGS_x86 += x86/nested_tdp_fault_test TEST_GEN_PROGS_x86 += x86/nested_tsc_adjust_test TEST_GEN_PROGS_x86 += x86/nested_tsc_scaling_test TEST_GEN_PROGS_x86 += x86/nested_vmsave_vmload_test +TEST_GEN_PROGS_x86 += x86/nested_x2apic_test TEST_GEN_PROGS_x86 += x86/platform_info_test TEST_GEN_PROGS_x86 += x86/pmu_counters_test TEST_GEN_PROGS_x86 += x86/pmu_event_filter_test diff --git a/tools/testing/selftests/kvm/x86/nested_x2apic_test.c b/tools/testing/selftests/kvm/x86/nested_x2apic_test.c new file mode 100644 index 000000000000..e77b4c347272 --- /dev/null +++ b/tools/testing/selftests/kvm/x86/nested_x2apic_test.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include "test_util.h" +#include "kvm_util.h" +#include "processor.h" +#include "vmx.h" +#include "svm_util.h" + +/* + * Use the kernel's posted interrupt vectors to minimize the risk of crashing + * the host if KVM is buggy. Note, the vectors aren't set in stone, ideally + * these will be kept up-to-date if the kernel vectors change, but it's "fine" + * if they are stale. + */ +#define POSTED_INTR_VECTOR 0xf2 +#define POSTED_INTR_WAKEUP_VECTOR 0xf1 +#define POSTED_INTR_NESTED_VECTOR 0xf0 + +static volatile unsigned int nr_irqs; + +static void guest_irq_handler(struct ex_regs *regs) +{ + nr_irqs++; + x2apic_write_reg(APIC_EOI, 0); +} + +static void l2_guest_code(void) +{ + wrmsr(MSR_IA32_APICBASE, rdmsr(MSR_IA32_APICBASE) & GENMASK_ULL(11, 0)); + asm volatile("cpuid" ::: "eax", "ebx", "ecx", "edx"); +} + +static void l1_svm_code(struct svm_test_data *svm) +{ + struct vmcb_control_area *ctrl = &svm->vmcb->control; + + generic_svm_setup(svm, l2_guest_code); + ctrl->intercept |= BIT_ULL(INTERCEPT_CPUID) | BIT_ULL(INTERCEPT_MSR_PROT); + + run_guest(svm->vmcb, svm->vmcb_gpa); + GUEST_ASSERT_EQ(ctrl->exit_code, SVM_EXIT_CPUID); + + stgi(); +} + +static void l1_vmx_code(struct vmx_pages *vmx) +{ + u64 control; + + GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true); + GUEST_ASSERT_EQ(load_vmcs(vmx), true); + + prepare_vmcs(vmx, NULL); + GUEST_ASSERT_EQ(vmwrite(GUEST_RIP, (unsigned long)l2_guest_code), 0); + + control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control |= CPU_BASED_USE_MSR_BITMAPS; + GUEST_ASSERT_EQ(vmwrite(CPU_BASED_VM_EXEC_CONTROL, control), 0); + + GUEST_ASSERT(!vmlaunch()); + GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_CPUID); +} + +static void l1_guest_code(void *test_data) +{ + x2apic_enable(); + + if (this_cpu_has(X86_FEATURE_SVM)) + l1_svm_code(test_data); + else + l1_vmx_code(test_data); + + sti_nop(); + + x2apic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_INT_ASSERT | POSTED_INTR_VECTOR); + x2apic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_INT_ASSERT | POSTED_INTR_WAKEUP_VECTOR); + x2apic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_INT_ASSERT | POSTED_INTR_NESTED_VECTOR); + GUEST_ASSERT_EQ(nr_irqs, 3); + GUEST_DONE(); +} + +int main(int argc, char *argv[]) +{ + gva_t nested_test_data_gva; + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct ucall uc; + + TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SVM) || kvm_cpu_has(X86_FEATURE_VMX)); + + vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); + vm_install_exception_handler(vm, POSTED_INTR_VECTOR, guest_irq_handler); + vm_install_exception_handler(vm, POSTED_INTR_WAKEUP_VECTOR, guest_irq_handler); + vm_install_exception_handler(vm, POSTED_INTR_NESTED_VECTOR, guest_irq_handler); + + if (kvm_cpu_has(X86_FEATURE_SVM)) + vcpu_alloc_svm(vm, &nested_test_data_gva); + else + vcpu_alloc_vmx(vm, &nested_test_data_gva); + + vcpu_args_set(vcpu, 1, nested_test_data_gva); + + vcpu_run(vcpu); + + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_DONE: + break; + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + default: + TEST_FAIL("Expected DONE, got unexpected ucall %lu", uc.cmd); + } + + kvm_vm_free(vm); +} From 0c19bc0ef11d8ea7f929a3ec3ce549cfe0781859 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 13 Aug 2026 15:36:07 -0700 Subject: [PATCH 16/29] KVM: selftests: Run the nested x2APIC with and without APICv being inhibited in L2 Signed-off-by: Sean Christopherson Message-ID: <20260813223610.2043560-4-seanjc@google.com> Signed-off-by: Paolo Bonzini --- .../selftests/kvm/x86/nested_x2apic_test.c | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/nested_x2apic_test.c b/tools/testing/selftests/kvm/x86/nested_x2apic_test.c index e77b4c347272..a1072bf499ee 100644 --- a/tools/testing/selftests/kvm/x86/nested_x2apic_test.c +++ b/tools/testing/selftests/kvm/x86/nested_x2apic_test.c @@ -15,6 +15,7 @@ #define POSTED_INTR_WAKEUP_VECTOR 0xf1 #define POSTED_INTR_NESTED_VECTOR 0xf0 +static bool inhibit_apicv; static volatile unsigned int nr_irqs; static void guest_irq_handler(struct ex_regs *regs) @@ -25,7 +26,8 @@ static void guest_irq_handler(struct ex_regs *regs) static void l2_guest_code(void) { - wrmsr(MSR_IA32_APICBASE, rdmsr(MSR_IA32_APICBASE) & GENMASK_ULL(11, 0)); + if (inhibit_apicv) + wrmsr(MSR_IA32_APICBASE, rdmsr(MSR_IA32_APICBASE) & GENMASK_ULL(11, 0)); asm volatile("cpuid" ::: "eax", "ebx", "ecx", "edx"); } @@ -78,20 +80,20 @@ static void l1_guest_code(void *test_data) GUEST_DONE(); } -int main(int argc, char *argv[]) +static void __test_x2apic_intercepts(void) { gva_t nested_test_data_gva; struct kvm_vcpu *vcpu; struct kvm_vm *vm; struct ucall uc; - TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SVM) || kvm_cpu_has(X86_FEATURE_VMX)); - vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); vm_install_exception_handler(vm, POSTED_INTR_VECTOR, guest_irq_handler); vm_install_exception_handler(vm, POSTED_INTR_WAKEUP_VECTOR, guest_irq_handler); vm_install_exception_handler(vm, POSTED_INTR_NESTED_VECTOR, guest_irq_handler); + sync_global_to_guest(vm, inhibit_apicv); + if (kvm_cpu_has(X86_FEATURE_SVM)) vcpu_alloc_svm(vm, &nested_test_data_gva); else @@ -115,3 +117,18 @@ int main(int argc, char *argv[]) kvm_vm_free(vm); } + +#define test_x2apic_intercepts(inhibit_apic_setting) \ +do { \ + inhibit_apic_setting; \ + \ + __test_x2apic_intercepts(); \ +} while (0) + +int main(int argc, char *argv[]) +{ + TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SVM) || kvm_cpu_has(X86_FEATURE_VMX)); + + test_x2apic_intercepts(inhibit_apicv = true); + test_x2apic_intercepts(inhibit_apicv = false); +} From 2f00ba93cc285092cd3b7ffebbc9abb6004f55e7 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 13 Aug 2026 15:36:08 -0700 Subject: [PATCH 17/29] KVM: selftests: Verify that L0's TPR doesn't get clobbered Signed-off-by: Sean Christopherson Message-ID: <20260813223610.2043560-5-seanjc@google.com> Signed-off-by: Paolo Bonzini --- .../selftests/kvm/x86/nested_x2apic_test.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tools/testing/selftests/kvm/x86/nested_x2apic_test.c b/tools/testing/selftests/kvm/x86/nested_x2apic_test.c index a1072bf499ee..3b59ba3e3342 100644 --- a/tools/testing/selftests/kvm/x86/nested_x2apic_test.c +++ b/tools/testing/selftests/kvm/x86/nested_x2apic_test.c @@ -28,6 +28,10 @@ static void l2_guest_code(void) { if (inhibit_apicv) wrmsr(MSR_IA32_APICBASE, rdmsr(MSR_IA32_APICBASE) & GENMASK_ULL(11, 0)); + + x2apic_write_reg(APIC_TASKPRI, 0xf0); + GUEST_ASSERT_EQ(x2apic_read_reg(APIC_TASKPRI), 0xf0); + asm volatile("cpuid" ::: "eax", "ebx", "ecx", "edx"); } @@ -73,10 +77,22 @@ static void l1_guest_code(void *test_data) sti_nop(); + GUEST_ASSERT_EQ(x2apic_read_reg(APIC_TASKPRI), 0xf0); + x2apic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_INT_ASSERT | POSTED_INTR_VECTOR); + GUEST_ASSERT_EQ(nr_irqs, 0); + + x2apic_write_reg(APIC_TASKPRI, 0xff); + GUEST_ASSERT_EQ(x2apic_read_reg(APIC_TASKPRI), 0xff); x2apic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_INT_ASSERT | POSTED_INTR_WAKEUP_VECTOR); + GUEST_ASSERT_EQ(nr_irqs, 0); + + x2apic_write_reg(APIC_TASKPRI, 0); + GUEST_ASSERT_EQ(x2apic_read_reg(APIC_TASKPRI), 0); + x2apic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_INT_ASSERT | POSTED_INTR_NESTED_VECTOR); GUEST_ASSERT_EQ(nr_irqs, 3); + GUEST_DONE(); } From b630929bcc4c867c9abc8cd5a0ebee3bb5ce5dcd Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Sat, 26 Sep 2026 02:04:03 -0400 Subject: [PATCH 18/29] KVM: selftests: Extend nested x2APIC test to validate disabling x2APIC virt Signed-off-by: Sean Christopherson Message-ID: <20260813223610.2043560-6-seanjc@google.com> Signed-off-by: Paolo Bonzini --- .../selftests/kvm/x86/nested_x2apic_test.c | 88 ++++++++++++++----- 1 file changed, 68 insertions(+), 20 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/nested_x2apic_test.c b/tools/testing/selftests/kvm/x86/nested_x2apic_test.c index 3b59ba3e3342..964937aefe39 100644 --- a/tools/testing/selftests/kvm/x86/nested_x2apic_test.c +++ b/tools/testing/selftests/kvm/x86/nested_x2apic_test.c @@ -29,10 +29,12 @@ static void l2_guest_code(void) if (inhibit_apicv) wrmsr(MSR_IA32_APICBASE, rdmsr(MSR_IA32_APICBASE) & GENMASK_ULL(11, 0)); - x2apic_write_reg(APIC_TASKPRI, 0xf0); - GUEST_ASSERT_EQ(x2apic_read_reg(APIC_TASKPRI), 0xf0); + for (;;) { + x2apic_write_reg(APIC_TASKPRI, 0xf0); + GUEST_ASSERT_EQ(x2apic_read_reg(APIC_TASKPRI), 0xf0); - asm volatile("cpuid" ::: "eax", "ebx", "ecx", "edx"); + asm volatile("cpuid" ::: "eax", "ebx", "ecx", "edx"); + } } static void l1_svm_code(struct svm_test_data *svm) @@ -46,6 +48,7 @@ static void l1_svm_code(struct svm_test_data *svm) GUEST_ASSERT_EQ(ctrl->exit_code, SVM_EXIT_CPUID); stgi(); + x2apic_write_reg(APIC_TASKPRI, 0); } static void l1_vmx_code(struct vmx_pages *vmx) @@ -58,22 +61,50 @@ static void l1_vmx_code(struct vmx_pages *vmx) prepare_vmcs(vmx, NULL); GUEST_ASSERT_EQ(vmwrite(GUEST_RIP, (unsigned long)l2_guest_code), 0); + control = vmreadz(PIN_BASED_VM_EXEC_CONTROL); + control |= PIN_BASED_EXT_INTR_MASK; + vmwrite(PIN_BASED_VM_EXEC_CONTROL, control); + control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); - control |= CPU_BASED_USE_MSR_BITMAPS; + control |= CPU_BASED_USE_MSR_BITMAPS | CPU_BASED_TPR_SHADOW; GUEST_ASSERT_EQ(vmwrite(CPU_BASED_VM_EXEC_CONTROL, control), 0); + control = vmreadz(SECONDARY_VM_EXEC_CONTROL); + control |= SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | + SECONDARY_EXEC_APIC_REGISTER_VIRT | + SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY; + control &= (rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2) >> 32); + GUEST_ASSERT_EQ(vmwrite(SECONDARY_VM_EXEC_CONTROL, control), 0); + GUEST_ASSERT(!vmlaunch()); GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_CPUID); + GUEST_ASSERT_EQ(vmwrite(GUEST_RIP, + vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN)), 0); } -static void l1_guest_code(void *test_data) +static void l1_vmx_code_part2(void) { - x2apic_enable(); + u64 control; - if (this_cpu_has(X86_FEATURE_SVM)) - l1_svm_code(test_data); - else - l1_vmx_code(test_data); + control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control &= ~CPU_BASED_TPR_SHADOW; + GUEST_ASSERT_EQ(vmwrite(CPU_BASED_VM_EXEC_CONTROL, control), 0); + + control = vmreadz(SECONDARY_VM_EXEC_CONTROL); + control &= ~(SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | + SECONDARY_EXEC_APIC_REGISTER_VIRT | + SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); + GUEST_ASSERT_EQ(vmwrite(SECONDARY_VM_EXEC_CONTROL, control), 0); + + GUEST_ASSERT(!vmresume()); + GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_CPUID); + GUEST_ASSERT_EQ(vmwrite(GUEST_RIP, + vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN)), 0); +} + +static void l1_test_x2apic_intercepts(void) +{ + GUEST_ASSERT_EQ(nr_irqs, 0); sti_nop(); @@ -93,16 +124,40 @@ static void l1_guest_code(void *test_data) x2apic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_INT_ASSERT | POSTED_INTR_NESTED_VECTOR); GUEST_ASSERT_EQ(nr_irqs, 3); + nr_irqs = 0; +} + +static void l1_guest_code(void *test_data) +{ + x2apic_enable(); + + if (this_cpu_has(X86_FEATURE_SVM)) + l1_svm_code(test_data); + else + l1_vmx_code(test_data); + + GUEST_ASSERT_EQ(x2apic_read_reg(APIC_TASKPRI), 0); + x2apic_write_reg(APIC_TASKPRI, 0xf0); + + l1_test_x2apic_intercepts(); + + if (this_cpu_has(X86_FEATURE_VMX)) { + l1_vmx_code_part2(); + l1_test_x2apic_intercepts(); + } + GUEST_DONE(); } -static void __test_x2apic_intercepts(void) +static void __test_x2apic_intercepts(bool with_inhibit_apicv) { gva_t nested_test_data_gva; struct kvm_vcpu *vcpu; struct kvm_vm *vm; struct ucall uc; + inhibit_apicv = with_inhibit_apicv; + vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); vm_install_exception_handler(vm, POSTED_INTR_VECTOR, guest_irq_handler); vm_install_exception_handler(vm, POSTED_INTR_WAKEUP_VECTOR, guest_irq_handler); @@ -134,17 +189,10 @@ static void __test_x2apic_intercepts(void) kvm_vm_free(vm); } -#define test_x2apic_intercepts(inhibit_apic_setting) \ -do { \ - inhibit_apic_setting; \ - \ - __test_x2apic_intercepts(); \ -} while (0) - int main(int argc, char *argv[]) { TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SVM) || kvm_cpu_has(X86_FEATURE_VMX)); - test_x2apic_intercepts(inhibit_apicv = true); - test_x2apic_intercepts(inhibit_apicv = false); + test_x2apic_intercepts(true); + test_x2apic_intercepts(false); } From b0bc51910f0391fea09c0a1d32352fec2acb2686 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 13 Aug 2026 15:36:10 -0700 Subject: [PATCH 19/29] KVM: selftests: Extend nested x2APIC test to validate using eVMCS for vmcs12 Signed-off-by: Sean Christopherson Message-ID: <20260813223610.2043560-7-seanjc@google.com> Signed-off-by: Paolo Bonzini --- .../selftests/kvm/x86/nested_x2apic_test.c | 42 +++++++++++++++---- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/tools/testing/selftests/kvm/x86/nested_x2apic_test.c b/tools/testing/selftests/kvm/x86/nested_x2apic_test.c index 964937aefe39..ce204ce29a9c 100644 --- a/tools/testing/selftests/kvm/x86/nested_x2apic_test.c +++ b/tools/testing/selftests/kvm/x86/nested_x2apic_test.c @@ -51,12 +51,24 @@ static void l1_svm_code(struct svm_test_data *svm) x2apic_write_reg(APIC_TASKPRI, 0); } -static void l1_vmx_code(struct vmx_pages *vmx) +static void l1_vmx_code(struct vmx_pages *vmx, struct hyperv_test_pages *hv_pages) { u64 control; + if (hv_pages) { + wrmsr(HV_X64_MSR_GUEST_OS_ID, HYPERV_LINUX_OS_ID); + enable_vp_assist(hv_pages->vp_assist_gpa, hv_pages->vp_assist); + evmcs_enable(); + } + GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true); - GUEST_ASSERT_EQ(load_vmcs(vmx), true); + + if (hv_pages) { + GUEST_ASSERT(load_evmcs(hv_pages)); + current_evmcs->hv_enlightenments_control.msr_bitmap = 1; + } else { + GUEST_ASSERT(load_vmcs(vmx)); + } prepare_vmcs(vmx, NULL); GUEST_ASSERT_EQ(vmwrite(GUEST_RIP, (unsigned long)l2_guest_code), 0); @@ -127,14 +139,14 @@ static void l1_test_x2apic_intercepts(void) nr_irqs = 0; } -static void l1_guest_code(void *test_data) +static void l1_guest_code(void *test_data, void *hv_pages) { x2apic_enable(); if (this_cpu_has(X86_FEATURE_SVM)) l1_svm_code(test_data); else - l1_vmx_code(test_data); + l1_vmx_code(test_data, hv_pages); GUEST_ASSERT_EQ(x2apic_read_reg(APIC_TASKPRI), 0); x2apic_write_reg(APIC_TASKPRI, 0xf0); @@ -149,9 +161,9 @@ static void l1_guest_code(void *test_data) GUEST_DONE(); } -static void __test_x2apic_intercepts(bool with_inhibit_apicv) +static void test_x2apic_intercepts(bool with_inhibit_apicv, bool use_evmcs) { - gva_t nested_test_data_gva; + gva_t nested_test_data_gva, hv_pages_gva = 0; struct kvm_vcpu *vcpu; struct kvm_vm *vm; struct ucall uc; @@ -170,7 +182,14 @@ static void __test_x2apic_intercepts(bool with_inhibit_apicv) else vcpu_alloc_vmx(vm, &nested_test_data_gva); - vcpu_args_set(vcpu, 1, nested_test_data_gva); + if (use_evmcs) { + vcpu_set_hv_cpuid(vcpu); + vcpu_enable_evmcs(vcpu); + + vcpu_alloc_hyperv_test_pages(vm, &hv_pages_gva); + } + + vcpu_args_set(vcpu, 2, nested_test_data_gva, hv_pages_gva); vcpu_run(vcpu); @@ -193,6 +212,11 @@ int main(int argc, char *argv[]) { TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SVM) || kvm_cpu_has(X86_FEATURE_VMX)); - test_x2apic_intercepts(true); - test_x2apic_intercepts(false); + test_x2apic_intercepts(true, false); + test_x2apic_intercepts(false, false); + + if (kvm_has_cap(KVM_CAP_HYPERV_ENLIGHTENED_VMCS)) { + test_x2apic_intercepts(true, true); + test_x2apic_intercepts(false, true); + } } From 8abbc76120a74bfba1851bd97528099d715e45c6 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 28 Sep 2026 08:46:43 -0700 Subject: [PATCH 20/29] KVM: SEV: Nullify "have run CPUs" mask pointer when freeing it Nullify have_run_cpus when freeing the mask, particularly in the error path of __sev_guest_init(), so that KVM doesn't have to subtly use sev->active to track whether or not the mask has been freed. As pointed out by Sashiko, blindly freeing the mask in sev_vm_destroy() results in a double-free if the mask is freed if __sev_guest_init() fails. Throw the logic in a helper as nullifying the pointer is frustratingly difficult and weird due to have_run_cpus being a single-entry array when CPUMASK_OFFSTACK=n. Deliberately don't use CPUMASK_VAR_NULL, as it's not directly assignable when the cpumask is on-stack, e.g. requires using a local variable and a memcpy(), which is beyond ridiculous. Furthermore, while clearing the on-stack bitmask is an unnecessary and arguably unwanted side effect, KVM absolutely relies on '0' being the "null" value given that the struct is zero-allocated. Opportunistically add an alloc() helper to pair with free(); there are just enough call sites to make doing so worthwhile. Fixes: 12c1f6e03f94 ("KVM: SEV: Free have_run_cpus during VM destruction even if VM is no longer SEV") Cc: stable@vger.kernel.org Reported-by: Sashiko Bot Closes: https://lore.kernel.org/all/20260923165349.CAAF01F000FF@smtp.kernel.org Signed-off-by: Sean Christopherson Message-ID: <20260928154644.2559454-2-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/sev.c | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 3448d56520c6..d3a2e6a51efc 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -488,6 +488,20 @@ static void snp_guest_req_cleanup(struct kvm *kvm) sev->guest_resp_buf = NULL; } +static int sev_alloc_have_run_cpus(struct kvm_sev_info *sev) +{ + if (!zalloc_cpumask_var(&sev->have_run_cpus, GFP_KERNEL_ACCOUNT)) + return -ENOMEM; + + return 0; +} + +static void sev_free_have_run_cpus(struct kvm_sev_info *sev) +{ + free_cpumask_var(sev->have_run_cpus); + memset(&sev->have_run_cpus, 0, sizeof(sev->have_run_cpus)); +} + static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp, struct kvm_sev_init *data, unsigned long vm_type) @@ -545,10 +559,9 @@ static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp, if (ret) goto e_free_asid; - if (!zalloc_cpumask_var(&sev->have_run_cpus, GFP_KERNEL_ACCOUNT)) { - ret = -ENOMEM; + ret = sev_alloc_have_run_cpus(sev); + if (ret) goto e_free_asid; - } /* This needs to happen after SEV/SNP firmware initialization. */ if (snp_active) { @@ -566,7 +579,7 @@ static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp, return 0; e_free: - free_cpumask_var(sev->have_run_cpus); + sev_free_have_run_cpus(sev); e_free_asid: argp->error = init_args.error; sev_asid_free(sev); @@ -2191,10 +2204,9 @@ int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd) * does not, i.e. KVM could skip flushes if memory is reclaimed from * the old VM but not the new VM. */ - if (!zalloc_cpumask_var(&dst_sev->have_run_cpus, GFP_KERNEL_ACCOUNT)) { - ret = -ENOMEM; + ret = sev_alloc_have_run_cpus(dst_sev); + if (ret) goto out_source_vcpu; - } sev_migrate_from(kvm, source_kvm); kvm_vm_dead(source_kvm); @@ -2888,10 +2900,9 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd) } mirror_sev = to_kvm_sev_info(kvm); - if (!zalloc_cpumask_var(&mirror_sev->have_run_cpus, GFP_KERNEL_ACCOUNT)) { - ret = -ENOMEM; + ret = sev_alloc_have_run_cpus(mirror_sev); + if (ret) goto e_unlock; - } /* * The mirror kvm holds an enc_context_owner ref so its asid can't @@ -2984,7 +2995,7 @@ void sev_vm_destroy(struct kvm *kvm) * Free the mask even if the VM is not *currently* an SEV VM, as it may * have been an SEV VM prior to intra-host migration. */ - free_cpumask_var(sev->have_run_cpus); + sev_free_have_run_cpus(sev); if (!sev_guest(kvm)) return; From 973ea70393e885e540f714904e51bc6cac80e3d7 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 28 Sep 2026 08:46:44 -0700 Subject: [PATCH 21/29] KVM: SEV: Do cache maintenance on the source VM *before* clearing SEV state Explicitly flush caches after intra-host migration before clearing "SEV active" on the source VM, as doing cache maintenance afterwards creates a tiny window where memory reclaim could return memory to the host without performing a cache flush, e.g. as pointed out by Sashiko: CPU1 in sev_migrate_from(): src->active = false; CPU2 running concurrent unmap: Since active is false, the automatic cache flush in sev_guest_memory_reclaimed is skipped. The host frees and reallocates the page. CPU1 in sev_migrate_from(): sev_writeback_caches(src_kvm); Executes a hardware cache flush (wbnoinvd), which writes the guest's old dirty ciphertext over the new page owner's data. Fixes: 93de2a6a4b91 ("KVM: SEV: Do cache maintenance on the source VM during intra-host migration") Cc: stable@vger.kernel.org Reported-by: Sashiko Bot Closes: https://lore.kernel.org/all/20260923165304.1662E1F000FF@smtp.kernel.org Signed-off-by: Sean Christopherson Message-ID: <20260928154644.2559454-3-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/sev.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index d3a2e6a51efc..0c1ebb16cec6 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2045,6 +2045,13 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm) struct kvm_sev_info *mirror; unsigned long i; + /* + * Do cache maintenance on the source VM *before* clearing "SEV active", + * as memory reclaim flows won't trigger cache maintenance on the VM + * once it's no longer an SEV VM. + */ + sev_writeback_caches(src_kvm); + dst->active = true; dst->asid = src->asid; dst->handle = src->handle; @@ -2058,12 +2065,6 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm) src->pages_locked = 0; src->es_active = false; - /* - * Do cache maintenance on the source VM as it is no longer an SEV VM, - * i.e. memory reclaim flows won't trigger cache maintenance on the VM. - */ - sev_writeback_caches(src_kvm); - list_cut_before(&dst->regions_list, &src->regions_list, &src->regions_list); mutex_lock(&sev_mirror_lock); From e079ef0b1c40c7a4a1be0c1ac5dbff95685c3beb Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 29 Sep 2026 10:35:44 +0100 Subject: [PATCH 22/29] KVM: arm64: vgic: Allow last_lr_irq to be NULL when LRs are not overflowing last_lr_irq is always populated when there is any interrupt populated in the AP list. Not only this is not necessary (it is only useful when we completely fill the LRs), but this is in the way of further fixes. Make sure last_lr_irq is kept to NULL when we LRs are not completely full. Fixes: 6da5e537f5afe ("KVM: arm64: vgic: Pick EOIcount deactivations from AP-list tail") Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260929093548.3598547-4-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-v2.c | 6 ++++-- arch/arm64/kvm/vgic/vgic-v3.c | 6 ++++-- arch/arm64/kvm/vgic/vgic.c | 8 +++----- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-v2.c b/arch/arm64/kvm/vgic/vgic-v2.c index 7182f63fc938..70cc53ba810a 100644 --- a/arch/arm64/kvm/vgic/vgic-v2.c +++ b/arch/arm64/kvm/vgic/vgic-v2.c @@ -122,6 +122,10 @@ void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu) for (int lr = 0; lr < vgic_cpu->vgic_v2.used_lrs; lr++) vgic_v2_fold_lr(vcpu, cpuif->vgic_lr[lr]); + cpuif->used_lrs = 0; + if (!irq) + return; + /* See the GICv3 equivalent for the EOIcount handling rationale */ list_for_each_entry_continue(irq, &vgic_cpu->ap_list_head, ap_list) { u32 lr; @@ -144,8 +148,6 @@ void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu) vgic_v2_fold_lr(vcpu, lr); eoicount--; } - - cpuif->used_lrs = 0; } void vgic_v2_deactivate(struct kvm_vcpu *vcpu, u32 val) diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 726e20a1da6e..346bacb3198f 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -155,6 +155,10 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu) for (int lr = 0; lr < cpuif->used_lrs; lr++) vgic_v3_fold_lr(vcpu, cpuif->vgic_lr[lr]); + cpuif->used_lrs = 0; + if (!irq) + return; + /* * EOIMode=0: use EOIcount to emulate deactivation. We are * guaranteed to deactivate in reverse order of the activation, so @@ -188,8 +192,6 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu) vgic_v3_fold_lr(vcpu, lr); eoicount--; } - - cpuif->used_lrs = 0; } void vgic_v3_deactivate(struct kvm_vcpu *vcpu, u64 val) diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index b25303d9919f..425503e0c582 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -866,9 +866,6 @@ static void vgic_fold_state(struct kvm_vcpu *vcpu) return; } - if (!*host_data_ptr(last_lr_irq)) - return; - if (kvm_vgic_global_state.type == VGIC_V2) vgic_v2_fold_lr_state(vcpu); else @@ -1021,11 +1018,12 @@ static void vgic_flush_lr_state(struct kvm_vcpu *vcpu) scoped_guard(raw_spinlock, &irq->irq_lock) { if (likely(vgic_target_oracle(irq) == vcpu)) { vgic_populate_lr(vcpu, irq, count++); - *host_data_ptr(last_lr_irq) = irq; + if (count == kvm_vgic_global_state.nr_lr) + *host_data_ptr(last_lr_irq) = irq; } } - if (count == kvm_vgic_global_state.nr_lr) + if (*host_data_ptr(last_lr_irq)) break; } From 19bc5d79ece613f242b5ec03921a3b465321bfed Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 29 Sep 2026 10:35:45 +0100 Subject: [PATCH 23/29] KVM: arm64: vgic: Take a refcount on IRQs referenced by last_lr_irq Referencing the last interrupt inserted in an LR is rather fragile, as this interrupt can vanish if a concurrently unmapped LPI. Solve this by bumping up the refcount on the interrupt when populating last_lr_irq, and drop it at vgic_prune_ap_list() time, when LPIs are being reclaimed. Fixes: 6da5e537f5afe ("KVM: arm64: vgic: Pick EOIcount deactivations from AP-list tail") Reported-by: Yuchao Zhang Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260929093548.3598547-5-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 425503e0c582..5cf5a1ef86cd 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -853,6 +853,17 @@ retry: goto retry; } + /* + * Fix the last_lr_irq refcount which was obtained while + * populating the LRs. This can also result in the LPI being + * deleted. + */ + irq = *host_data_ptr(last_lr_irq); + if (irq) { + deleted_lpis |= vgic_put_irq_norelease(vcpu->kvm, irq); + *host_data_ptr(last_lr_irq) = NULL; + } + raw_spin_unlock(&vgic_cpu->ap_list_lock); if (unlikely(deleted_lpis)) @@ -1018,8 +1029,10 @@ static void vgic_flush_lr_state(struct kvm_vcpu *vcpu) scoped_guard(raw_spinlock, &irq->irq_lock) { if (likely(vgic_target_oracle(irq) == vcpu)) { vgic_populate_lr(vcpu, irq, count++); - if (count == kvm_vgic_global_state.nr_lr) + if (count == kvm_vgic_global_state.nr_lr) { + vgic_get_irq_ref(irq); *host_data_ptr(last_lr_irq) = irq; + } } } From 751f4641560be8568dcc5af400c0e55a283abf51 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 29 Sep 2026 10:35:47 +0100 Subject: [PATCH 24/29] KVM: arm64: vgic-its: Fix MOVALL handling of source redistributor The MOVALL command moves the pending state of all LPIs targeting a given redistributor to another one. However, we seem to have lost the filtering on the source RD, which means we move all LPIs to the target. Not quite what the spec mandates. Hack update_affinity() to take an optional source vcpu that is used as a filter when non-NULL, restoring the filtering that was performed by vgic_copy_lpi_list() back in the days. Fixes: 11f4f8f3e6e06 ("KVM: arm64: vgic-its: Walk LPI xarray in vgic_its_cmd_handle_movall()") Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260929093548.3598547-7-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic-its.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 0904ae850c35..c10a11373e10 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -319,12 +319,16 @@ static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq, return ret; } -static int update_affinity(struct vgic_irq *irq, struct kvm_vcpu *vcpu) +static int update_affinity(struct vgic_irq *irq, + struct kvm_vcpu *from_vcpu, struct kvm_vcpu *vcpu) { struct its_vlpi_map map; int ret; guard(raw_spinlock_irqsave)(&irq->irq_lock); + if (from_vcpu && irq->target_vcpu != from_vcpu) + return 0; + irq->target_vcpu = vcpu; if (!irq->hw) @@ -362,7 +366,7 @@ static void update_affinity_ite(struct kvm *kvm, struct its_ite *ite) return; vcpu = collection_to_vcpu(kvm, ite->collection); - update_affinity(ite->irq, vcpu); + update_affinity(ite->irq, NULL, vcpu); } /* @@ -856,7 +860,7 @@ static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its, vgic_its_invalidate_cache(its); - return update_affinity(ite->irq, vcpu); + return update_affinity(ite->irq, NULL, vcpu); } static bool __is_visible_gfn_locked(struct vgic_its *its, gpa_t gpa) @@ -1383,7 +1387,7 @@ static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its, if (!irq) continue; - update_affinity(irq, vcpu2); + update_affinity(irq, vcpu1, vcpu2); vgic_put_irq(kvm, irq); } From 4bdd2b3a708c1feb701cb8b62feb8ad2fd77bb07 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 29 Sep 2026 10:00:28 +0100 Subject: [PATCH 25/29] KVM: arm64: Apply the fine-grained UNDEFs without FEAT_FGT FGUs don't actually require hardware support, so don't gate fgt trap config insertion on FGT. This is the only action required to allow FGUs always, as kvm_calculate_traps() already computes the FGU bits with or without FEAT_FGT. This also fixes a spurious WARN when a TLBI OS is run on a guest in vEL1 without FEAT_TLBIOS on non FEAT_FGT hardware. In this configuration the course-grained HCR_EL2.TTLBOS trap reaches the handler handle_tlbi_el1(), which expects an EL1 TLBI only from vEL2. With FGUs enabled the UNDEF will be injected without needing the specific handler. Fixes: f5a5a406b4b8b ("KVM: arm64: Propagate and handle Fine-Grained UNDEF bits") Cc: stable@vger.kernel.org Signed-off-by: Fuad Tabba Reviewed-by: Wei-Lin Chang Link: https://patch.msgid.link/20260929090031.3829185-2-fuad.tabba@linux.dev [oliver: Take Wei-Lin's suggested changelog] Signed-off-by: Oliver Upton --- arch/arm64/kvm/emulate-nested.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index 3806ff0920fe..f59d89d04eca 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -2386,9 +2386,6 @@ int __init populate_nv_trap_config(void) print_nv_trap_error(fgt, "FGT bit is reserved", ret); } - if (!cpus_have_final_cap(ARM64_HAS_FGT)) - continue; - prev = xa_store(&sr_forward_xa, enc, xa_mk_value(tc.val), GFP_KERNEL); From 80ae56184b57176b908cd6a7ececf274a7e4b3c7 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 29 Sep 2026 10:00:29 +0100 Subject: [PATCH 26/29] KVM: arm64: Clear HCR_EL2.RW for 32-bit non-protected vCPUs pKVM keeps its own copy of each vCPU's HCR_EL2 at EL2, initialised with RW set unconditionally. Nothing stops userspace from creating a non-protected AArch32 VM, and with RW set, entering one of its vCPUs is an illegal exception return: KVM_RUN fails with KVM_EXIT_FAIL_ENTRY. Clear RW for a vCPU that is AArch32 at EL1, when the CPU has AArch32 EL1. On a CPU without it, RW stays set and the entry still fails, rather than EL2 switching *32_EL2 registers that are UNDEFINED there. The host already rejects such a vCPU, but EL2 takes the vCPU's features from the host and doesn't rely on that. Fixes: b56680de9c648 ("KVM: arm64: Initialize trap register values in hyp in pKVM") Cc: stable@vger.kernel.org Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260929090031.3829185-3-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/pkvm.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index bb3e0dc0676e..cd09d8888202 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -54,6 +54,15 @@ static void pkvm_vcpu_reset_hcr(struct kvm_vcpu *vcpu) else vcpu->arch.hcr_el2 |= HCR_TID2; + /* + * Without AArch32 EL1, leave RW set and let the entry fail with an + * illegal exception return: the *32_EL2 registers EL2 would otherwise + * switch are UNDEFINED there. + */ + if (vcpu_has_feature(vcpu, KVM_ARM_VCPU_EL1_32BIT) && + cpus_have_final_cap(ARM64_HAS_32BIT_EL1)) + vcpu->arch.hcr_el2 &= ~HCR_EL2_RW; + if (vcpu_has_ptrauth(vcpu)) vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK); From 04447b95c62bef2b3591486c66244c49591a3dbd Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 29 Sep 2026 10:00:30 +0100 Subject: [PATCH 27/29] KVM: arm64: Use the host's HCR_EL2 for non-protected VMs in pKVM For each vCPU, pKVM's EL2 builds its own HCR_EL2 and takes only TWI, TWE and VSE from the host's value on each entry. For a non-protected VM it has fallen behind the host's: on a CPU with MTE the guest can read GMID_EL1, on one with FEAT_EVT2 but no FGT it can execute a TLBI OS its ID registers hide, an interrupt injected without a vGIC (VI, VF) never arrives, and set/way emulation loses the TVM trap it relies on. For a non-protected VM, also take the bits the host varies with the VM's configuration or at runtime: VI, VF, TVM, TID2, TID4, TID5 and TTLBOS. They only pend interrupts for, or add traps to, a VM the host controls. The traps exit to the host, which handles them as it does without pKVM. The bits that change what EL2 does on entry and exit (E2H, RW, API/APK) or depend only on the CPU (TEA, TERR, FWB) stay EL2's. A protected VM still takes only TWI, TWE and VSE. EL2 now sets TID2 or TID4 only for a protected VM, and never sets ATA, as the host rejects KVM_CAP_ARM_MTE in pKVM. Fixes: b56680de9c648 ("KVM: arm64: Initialize trap register values in hyp in pKVM") Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260929090031.3829185-4-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/include/nvhe/pkvm.h | 9 +++++++++ arch/arm64/kvm/hyp/nvhe/hyp-main.c | 7 ++++--- arch/arm64/kvm/hyp/nvhe/pkvm.c | 18 ++++++++---------- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index c904647d2f76..5c050f21066a 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -12,6 +12,15 @@ #include #include +/* + * HCR_EL2 bits EL2 takes from the host on each entry, per VM type. The rest + * are EL2's own and nothing the host sets there reaches the guest. + */ +#define PKVM_HCR_EL2_HOST_PVM (HCR_EL2_TWI | HCR_EL2_TWE | HCR_EL2_VSE) +#define PKVM_HCR_EL2_HOST_NPVM (PKVM_HCR_EL2_HOST_PVM | HCR_EL2_VI | HCR_EL2_VF | \ + HCR_EL2_TVM | HCR_EL2_TID2 | HCR_EL2_TID4 | \ + HCR_EL2_TID5 | HCR_EL2_TTLBOS) + /* * Holds the relevant data for maintaining the vcpu state completely at hyp. */ diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 9a3b92e626ad..ac64a036b0a9 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -216,6 +216,7 @@ static void sync_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + u64 host_hcr_mask = PKVM_HCR_EL2_HOST_PVM; fpsimd_sve_flush(); flush_debug_state(hyp_vcpu); @@ -228,6 +229,7 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { if (vcpu_get_flag(host_vcpu, PKVM_HOST_STATE_DIRTY)) flush_hyp_vcpu_state(hyp_vcpu); + host_hcr_mask = PKVM_HCR_EL2_HOST_NPVM; } else { hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt; } @@ -241,9 +243,8 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) * trap-control bit, so it must flow to the hyp vCPU alongside TWI/TWE * for the vSError to be delivered. sync_hyp_vcpu() reflects it back. */ - hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWI | HCR_TWE | HCR_VSE); - hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) & - (HCR_TWI | HCR_TWE | HCR_VSE); + hyp_vcpu->vcpu.arch.hcr_el2 &= ~host_hcr_mask; + hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) & host_hcr_mask; hyp_vcpu->vcpu.arch.iflags = host_vcpu->arch.iflags; diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index cd09d8888202..6290c4b62659 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -30,6 +30,7 @@ unsigned int kvm_host_sve_max_vl; */ static DEFINE_PER_CPU(struct pkvm_hyp_vcpu *, loaded_hyp_vcpu); +/* The PKVM_HCR_EL2_HOST_{PVM,NPVM} bits of this value come from the host on each entry. */ static void pkvm_vcpu_reset_hcr(struct kvm_vcpu *vcpu) { vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS; @@ -47,13 +48,6 @@ static void pkvm_vcpu_reset_hcr(struct kvm_vcpu *vcpu) if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) vcpu->arch.hcr_el2 |= HCR_FWB; - if (cpus_have_final_cap(ARM64_HAS_EVT) && - !cpus_have_final_cap(ARM64_MISMATCHED_CACHE_TYPE) && - kvm_read_vm_id_reg(vcpu->kvm, SYS_CTR_EL0) == read_cpuid(CTR_EL0)) - vcpu->arch.hcr_el2 |= HCR_TID4; - else - vcpu->arch.hcr_el2 |= HCR_TID2; - /* * Without AArch32 EL1, leave RW set and let the entry fail with an * illegal exception return: the *32_EL2 registers EL2 would otherwise @@ -65,9 +59,6 @@ static void pkvm_vcpu_reset_hcr(struct kvm_vcpu *vcpu) if (vcpu_has_ptrauth(vcpu)) vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK); - - if (kvm_has_mte(vcpu->kvm)) - vcpu->arch.hcr_el2 |= HCR_ATA; } static void pvm_init_traps_hcr(struct kvm_vcpu *vcpu) @@ -85,6 +76,13 @@ static void pvm_init_traps_hcr(struct kvm_vcpu *vcpu) */ val |= HCR_TACR | HCR_TIDCP | HCR_TID3 | HCR_TID1; + if (cpus_have_final_cap(ARM64_HAS_EVT) && + !cpus_have_final_cap(ARM64_MISMATCHED_CACHE_TYPE) && + kvm_read_vm_id_reg(kvm, SYS_CTR_EL0) == read_cpuid(CTR_EL0)) + val |= HCR_EL2_TID4; + else + val |= HCR_EL2_TID2; + if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) { val |= HCR_TERR | HCR_TEA; val &= ~(HCR_FIEN); From afb4334fb52c7b416bbc9c4b166f6806509f734b Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 29 Sep 2026 10:00:31 +0100 Subject: [PATCH 28/29] KVM: arm64: selftests: Check a feature hidden in an ID register is UNDEF Userspace can hide a feature from a guest by clearing its field in a writable ID register, and KVM then makes the feature's instructions UNDEFINED in the guest by trapping or disabling them. No selftest checks that. Add a test that runs the instruction of each of TLBI OS, MOPS, TCR2_EL1 and FPMR once with its field as advertised and once with it cleared, and expects an UNDEF only when cleared. A feature the vCPU doesn't advertise is skipped, as is hidden TLBI OS on a CPU with neither FGT nor FEAT_EVT2, where KVM can't trap it. Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260929090031.3829185-5-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- tools/testing/selftests/kvm/Makefile.kvm | 1 + .../selftests/kvm/arm64/hidden_features.c | 184 ++++++++++++++++++ 2 files changed, 185 insertions(+) create mode 100644 tools/testing/selftests/kvm/arm64/hidden_features.c diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 6a1482e3a286..51b98f47044f 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -195,6 +195,7 @@ TEST_GEN_PROGS_arm64 += arm64/vgic_v5 TEST_GEN_PROGS_arm64 += arm64/vpmu_counter_access TEST_GEN_PROGS_arm64 += arm64/no-vgic TEST_GEN_PROGS_arm64 += arm64/idreg-idst +TEST_GEN_PROGS_arm64 += arm64/hidden_features TEST_GEN_PROGS_arm64 += arm64/kvm-uuid TEST_GEN_PROGS_arm64 += access_tracking_perf_test TEST_GEN_PROGS_arm64 += arch_timer diff --git a/tools/testing/selftests/kvm/arm64/hidden_features.c b/tools/testing/selftests/kvm/arm64/hidden_features.c new file mode 100644 index 000000000000..194d746e7605 --- /dev/null +++ b/tools/testing/selftests/kvm/arm64/hidden_features.c @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * hidden_features - Check that a feature's instruction runs in the guest when + * its ID register field is advertised, and is UNDEFINED when userspace clears + * the field. + * + * Copyright (c) 2026 Google LLC + * Author: Fuad Tabba + */ +#include "kvm_util.h" +#include "processor.h" +#include "test_util.h" + +static volatile bool undef; + +static void guest_tlbi_os(void) +{ + /* tlbi vmalle1os */ + asm volatile("sys #0, c8, c1, #0\n\tdsb ish\n\tisb" ::: "memory"); +} + +static void guest_mops(void) +{ + register u64 *d asm("x0"); + register u64 n asm("x1"); + register u64 s asm("x2"); + u64 buf[8]; + + d = buf; + n = sizeof(buf); + s = 0; + /* setp [x0]!, x1!, x2; setm; sete */ + asm volatile(".inst 0x19c20420\n\t.inst 0x19c24420\n\t.inst 0x19c28420" + : "+r"(d), "+r"(n) : "r"(s) : "cc", "memory"); +} + +static void guest_tcr2(void) +{ + read_sysreg_s(SYS_TCR2_EL1); +} + +static void guest_fpmr(void) +{ + read_sysreg_s(SYS_FPMR); +} + +struct feature { + const char *name; + u64 id_reg; + u64 mask; + u8 shift; + u64 min; + void (*insn)(void); + bool (*trappable)(struct kvm_vcpu *vcpu); +}; + +/* Without FGT, KVM traps a hidden TLBI OS only through HCR_EL2.TTLBOS (FEAT_EVT2). */ +static bool tlbi_os_trappable(struct kvm_vcpu *vcpu) +{ + u64 mmfr0 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64MMFR0_EL1)); + u64 mmfr2 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64MMFR2_EL1)); + + return SYS_FIELD_GET(ID_AA64MMFR0_EL1, FGT, mmfr0) >= ID_AA64MMFR0_EL1_FGT_IMP || + SYS_FIELD_GET(ID_AA64MMFR2_EL1, EVT, mmfr2) >= ID_AA64MMFR2_EL1_EVT_TTLBxS; +} + +#define FEATURE(n, reg, field, min_val, fn, trap) \ +{ \ + .name = n, \ + .id_reg = SYS_##reg, \ + .mask = reg##_##field##_MASK, \ + .shift = reg##_##field##_SHIFT, \ + .min = reg##_##field##_##min_val, \ + .insn = fn, \ + .trappable = trap, \ +} + +static const struct feature features[] = { + FEATURE("TLBI OS", ID_AA64ISAR0_EL1, TLB, OS, guest_tlbi_os, tlbi_os_trappable), + FEATURE("MOPS", ID_AA64ISAR2_EL1, MOPS, IMP, guest_mops, NULL), + FEATURE("TCR2_EL1", ID_AA64MMFR3_EL1, TCRX, IMP, guest_tcr2, NULL), + FEATURE("FPMR", ID_AA64PFR2_EL1, FPMR, IMP, guest_fpmr, NULL), +}; + +static void guest_code(const struct feature *feat) +{ + undef = false; + feat->insn(); + GUEST_SYNC(undef); + GUEST_DONE(); +} + +static void guest_undef_handler(struct ex_regs *regs) +{ + undef = true; + regs->pc += 4; +} + +static bool run(const struct feature *feat, bool hide) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct ucall uc; + bool got = false; + u64 val; + + vm = vm_create_with_one_vcpu(&vcpu, (void *)guest_code); + vm_init_descriptor_tables(vm); + vcpu_init_descriptor_tables(vcpu); + vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT, ESR_ELx_EC_UNKNOWN, guest_undef_handler); + vcpu_args_set(vcpu, 1, feat); + + if (hide) { + val = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(feat->id_reg)); + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(feat->id_reg), val & ~feat->mask); + } + + for (;;) { + vcpu_run(vcpu); + switch (get_ucall(vcpu, &uc)) { + case UCALL_SYNC: + got = uc.args[1]; + break; + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + case UCALL_DONE: + kvm_vm_free(vm); + return got; + default: + TEST_FAIL("Unknown ucall %lu", uc.cmd); + } + } +} + +static void probe_feature(const struct feature *feat, bool *present, bool *trappable) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + u64 val; + + vm = vm_create_with_one_vcpu(&vcpu, NULL); + val = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(feat->id_reg)); + *present = ((val & feat->mask) >> feat->shift) >= feat->min; + *trappable = !feat->trappable || feat->trappable(vcpu); + kvm_vm_free(vm); +} + +int main(void) +{ + const struct feature *feat; + bool present, trappable; + int i; + + test_disable_default_vgic(); + + ksft_print_header(); + ksft_set_plan(ARRAY_SIZE(features) * 2); + + for (i = 0; i < ARRAY_SIZE(features); i++) { + feat = &features[i]; + + probe_feature(feat, &present, &trappable); + if (!present) { + ksft_test_result_skip("%s advertised, not supported\n", feat->name); + ksft_test_result_skip("%s hidden, not supported\n", feat->name); + continue; + } + + if (run(feat, false)) + ksft_test_result_fail("%s advertised, UNDEF\n", feat->name); + else + ksft_test_result_pass("%s advertised\n", feat->name); + + if (!trappable) + ksft_test_result_skip("%s hidden, not trappable\n", feat->name); + else if (run(feat, true)) + ksft_test_result_pass("%s hidden\n", feat->name); + else + ksft_test_result_fail("%s hidden, no UNDEF\n", feat->name); + } + + ksft_finished(); +} From 71cc2c67fb8f8d5aa8154eb482e9846d2214f11b Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Tue, 29 Sep 2026 20:02:09 +0000 Subject: [PATCH 29/29] KVM: arm64: Use stage-1 leaf size for VM_PFNMAP When commit 2aa53d68cee6 ("KVM: arm64: Try stage2 block mapping for host device MMIO") added VM_PFNMAP support to get_vma_page_shift(), stage-1 page tables did not support huge PFNMAP (as in VFIO-PCI vfio_pci_mmap_huge_fault()) and transparent_hugepage_adjust() was unsafe for MMIO as it dereferenced struct page. Since commit 6011cf68c885 ("KVM: arm64: Walk userspace page tables to compute the THP mapping size"), transparent_hugepage_adjust() instead walks the host stage-1 page tables via get_user_mapping_size() without touching struct page. Meanwhile, commit 3e509c9b03f9 ("mm/arm64: support large pfn mappings") enabled stage-1 huge PFNMAP. So. we can drop the VMA-based VM_PFNMAP size calculation in get_vma_page_shift() and let transparent_hugepage_adjust() derive the stage-2 mapping size directly from the populated stage-1 leaf for non-cacheable mappings as well. Assisted-by: LLM Cc: stable@vger.kernel.org Fixes: 2aa53d68cee6 ("KVM: arm64: Try stage2 block mapping for host device MMIO") Reviewed-by: Marc Zyngier Signed-off-by: Mostafa Saleh Signed-off-by: Oliver Upton --- arch/arm64/kvm/mmu.c | 47 ++++++-------------------------------------- 1 file changed, 6 insertions(+), 41 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 2d44cd6a5aed..bf6d6526639f 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1468,32 +1468,11 @@ transparent_hugepage_adjust(struct kvm *kvm, struct kvm_memory_slot *memslot, return PAGE_SIZE; } -static int get_vma_page_shift(struct vm_area_struct *vma, unsigned long hva) +static int get_vma_page_shift(struct vm_area_struct *vma) { - unsigned long pa; - - if (is_vm_hugetlb_page(vma) && !(vma->vm_flags & VM_PFNMAP)) + if (is_vm_hugetlb_page(vma)) return huge_page_shift(hstate_vma(vma)); - if (!(vma->vm_flags & VM_PFNMAP)) - return PAGE_SHIFT; - - VM_BUG_ON(is_vm_hugetlb_page(vma)); - - pa = (vma->vm_pgoff << PAGE_SHIFT) + (hva - vma->vm_start); - -#ifndef __PAGETABLE_PMD_FOLDED - if ((hva & (PUD_SIZE - 1)) == (pa & (PUD_SIZE - 1)) && - ALIGN_DOWN(hva, PUD_SIZE) >= vma->vm_start && - ALIGN(hva, PUD_SIZE) <= vma->vm_end) - return PUD_SHIFT; -#endif - - if ((hva & (PMD_SIZE - 1)) == (pa & (PMD_SIZE - 1)) && - ALIGN_DOWN(hva, PMD_SIZE) >= vma->vm_start && - ALIGN(hva, PMD_SIZE) <= vma->vm_end) - return PMD_SHIFT; - return PAGE_SHIFT; } @@ -1794,7 +1773,7 @@ static short kvm_s2_resolve_vma_size(const struct kvm_s2_fault_desc *s2fd, vma_shift = PAGE_SHIFT; } else { s2vi->max_map_size = PUD_SIZE; - vma_shift = get_vma_page_shift(vma, s2fd->hva); + vma_shift = get_vma_page_shift(vma); } switch (vma_shift) { @@ -1952,16 +1931,6 @@ static int kvm_s2_fault_pin_pfn(const struct kvm_s2_fault_desc *s2fd, return -EFAULT; } } else { - /* - * If the page was identified as device early by looking at - * the VMA flags, vma_pagesize is already representing the - * largest quantity we can map. If instead it was mapped - * via __kvm_faultin_pfn(), vma_pagesize is set to PAGE_SIZE - * and must not be upgraded. - * - * In both cases, we don't let transparent_hugepage_adjust() - * change things at the last minute. - */ s2vi->map_non_cacheable = true; } @@ -2051,10 +2020,10 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, /* * If we are not forced to use page mapping, check if we are - * backed by a THP and thus use block mapping if possible. + * backed by a huge stage-1 mapping and thus use block mapping if + * possible. */ - if (mapping_size == PAGE_SIZE && - !(s2vi->max_map_size == PAGE_SIZE || s2vi->map_non_cacheable)) { + if (mapping_size == PAGE_SIZE && s2vi->max_map_size != PAGE_SIZE) { if (perm_fault_granule > PAGE_SIZE) { mapping_size = perm_fault_granule; } else { @@ -2135,10 +2104,6 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd) return ret; } - /* - * Let's check if we will get back a huge page backed by hugetlbfs, or - * get block mapping for device MMIO region. - */ ret = kvm_s2_fault_pin_pfn(s2fd, &s2vi); if (ret != 1) return ret;