diff --git a/arch/arm64/kvm/hyp-init.S b/arch/arm64/kvm/hyp-init.S
index a409482fc19b..29c47f7d6ecf 100644
--- a/arch/arm64/kvm/hyp-init.S
+++ b/arch/arm64/kvm/hyp-init.S
@@ -86,9 +86,10 @@ __do_hyp_init:
 
 	/*
 	 * Preserve all the RES1 bits while setting the default flags,
-	 * as well as the EE bit on BE.
+	 * as well as the EE bit on BE. Drop the A flag since the compiler
+	 * is allowed to generate unaligned accesses.
 	 */
-	ldr	x4, =(SCTLR_EL2_RES1 | SCTLR_EL2_FLAGS)
+	ldr	x4, =(SCTLR_EL2_RES1 | (SCTLR_EL2_FLAGS & ~SCTLR_EL2_A))
 CPU_BE(	orr	x4, x4, #SCTLR_EL2_EE)
 	msr	sctlr_el2, x4
 	isb