diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h index cab27f7bd423..ec85a9547120 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h @@ -55,6 +55,7 @@ bool addr_is_memory(phys_addr_t phys); bool addr_is_hyp_text(phys_addr_t phys); int host_stage2_idmap_locked(phys_addr_t addr, u64 size, enum kvm_pgtable_prot prot); int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id); +bool host_stage2_pte_is_hyp_owned(kvm_pte_t pte); int kvm_host_prepare_stage2(void *pgt_pool_base); int kvm_guest_prepare_stage2(struct pkvm_hyp_vm *vm, void *pgd); void kvm_guest_destroy_stage2(struct pkvm_hyp_vm *vm); diff --git a/arch/arm64/kvm/hyp/include/nvhe/mm.h b/arch/arm64/kvm/hyp/include/nvhe/mm.h index 6e83ce35c2f2..31cae95ddb71 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mm.h @@ -29,6 +29,7 @@ int __pkvm_create_private_mapping(phys_addr_t phys, size_t size, enum kvm_pgtable_prot prot, unsigned long *haddr); int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr); +int pkvm_check_host_ownership(void); int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr); #endif /* __KVM_HYP_MM_H */ diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index d026f446bd8e..a6a47c1e058b 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -641,6 +641,18 @@ int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id) return ret; } +bool host_stage2_pte_is_hyp_owned(kvm_pte_t pte) +{ + if (kvm_pte_valid(pte)) + return false; + + if (FIELD_GET(KVM_INVALID_PTE_TYPE_MASK, pte) != + KVM_HOST_INVALID_PTE_TYPE_DONATION) + return false; + + return FIELD_GET(KVM_HOST_DONATION_PTE_OWNER_MASK, pte) == PKVM_ID_HYP; +} + #define KVM_HOST_PTE_OWNER_GUEST_HANDLE_MASK GENMASK(15, 0) /* We need 40 bits for the GFN to cover a 52-bit IPA with 4k pages and LPA2 */ #define KVM_HOST_PTE_OWNER_GUEST_GFN_MASK GENMASK(55, 16) diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index 422ee57be956..29ab5ee9d57f 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -472,6 +472,65 @@ int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr) return ret; } +static int check_page_ownership(phys_addr_t phys) +{ + kvm_pte_t pte; + bool host_ok; + int ret; + + if (addr_is_memory(phys)) { + struct hyp_page *page = hyp_phys_to_page(phys); + + if (get_hyp_state(page) != PKVM_PAGE_OWNED || + get_host_state(page) != PKVM_NOPAGE) + return -EPERM; + } + + ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, NULL); + if (ret) + return ret; + + /* Hyp text may stay host-readable, see fix_host_ownership_walker(). */ + if (kvm_pte_valid(pte) && addr_is_hyp_text(phys)) + host_ok = !(kvm_pgtable_stage2_pte_prot(pte) & KVM_PGTABLE_PROT_W); + else + host_ok = host_stage2_pte_is_hyp_owned(pte); + + return host_ok ? 0 : -EPERM; +} + +static int check_host_ownership_walker(const struct kvm_pgtable_visit_ctx *ctx, + enum kvm_pgtable_walk_flags visit) +{ + phys_addr_t phys, end; + int ret; + + if (!kvm_pte_valid(ctx->old)) + return 0; + + phys = kvm_pte_to_phys(ctx->old); + end = phys + kvm_granule_size(ctx->level); + for (; phys < end; phys += PAGE_SIZE) { + ret = check_page_ownership(phys); + if (ret) + return ret; + } + + return 0; +} + +int pkvm_check_host_ownership(void) +{ + struct kvm_pgtable_walker walker = { + .cb = check_host_ownership_walker, + .flags = KVM_PGTABLE_WALK_LEAF, + }; + + /* The private range and the vmemmap share one quarter of the VA space. */ + return kvm_pgtable_walk(&pkvm_pgtable, __io_map_base, + BIT(pkvm_pgtable.ia_bits - 2), &walker); +} + static void *admit_host_page(void *arg) { struct kvm_hyp_memcache *host_mc = arg; diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index bb667cd7080b..45ac5f2ba4f7 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -334,6 +334,10 @@ void __noreturn __pkvm_init_finalise(void) if (ret) goto out; + ret = pkvm_check_host_ownership(); + if (ret) + goto out; + ret = hyp_ffa_init(ffa_proxy_pages); if (ret) goto out;