diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 9ba86450fe4a..2d44cd6a5aed 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -59,27 +59,36 @@ static phys_addr_t stage2_range_addr_end(phys_addr_t addr, phys_addr_t end) * long will also starve other vCPUs. We have to also make sure that the page * tables are not freed while we released the lock. */ -static int stage2_apply_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, +static int stage2_apply_range(struct kvm_s2_mmu *mmu, phys_addr_t start, phys_addr_t end, int (*fn)(struct kvm_pgtable *, u64, u64), bool resched) { struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); + bool lock_dropped = false; + phys_addr_t addr = start; int ret; u64 next; do { struct kvm_pgtable *pgt = mmu->pgt; + /* + * We may be raced on PGT teardown when we release the + * kvm->mmu_lock. That's fine as the PGT is legitimately no + * longer present. + */ if (!pgt) - return -EINVAL; + return lock_dropped ? 0 : -EINVAL; next = stage2_range_addr_end(addr, end); ret = fn(pgt, addr, next - addr); if (ret) break; - if (resched && next != end) + if (resched && next != end) { cond_resched_rwlock_write(&kvm->mmu_lock); + lock_dropped = true; + } } while (addr = next, addr != end); return ret;