diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index d6c2dbf8455c..5874ca30f4f6 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -453,9 +453,10 @@ config SMP config NR_CPUS int "Maximum number of CPUs (2-512)" depends on SMP - range 2 512 if !RISCV_SBI_V01 - range 2 32 if RISCV_SBI_V01 && 32BIT + range 2 512 if !RISCV_SBI_V01 && 64BIT + range 2 128 if !RISCV_SBI_V01 && 32BIT range 2 64 if RISCV_SBI_V01 && 64BIT + range 2 32 if RISCV_SBI_V01 && 32BIT default "32" if 32BIT default "64" if RISCV_SBI_V01 && 64BIT default "256" if !RISCV_SBI_V01 && 64BIT diff --git a/arch/riscv/Kconfig.errata b/arch/riscv/Kconfig.errata index 3c945d086c7d..622ade116a2d 100644 --- a/arch/riscv/Kconfig.errata +++ b/arch/riscv/Kconfig.errata @@ -46,7 +46,7 @@ config ERRATA_MIPS_P8700_PAUSE_OPCODE config ERRATA_SIFIVE bool "SiFive errata" - depends on RISCV_ALTERNATIVE + depends on RISCV_ALTERNATIVE && 64BIT help All SiFive errata Kconfig depend on this Kconfig. Disabling this Kconfig will disable all SiFive errata. Please say "Y" @@ -96,6 +96,20 @@ config ERRATA_STARFIVE_JH7100 Say "Y" if you want to support the BeagleV Starlight and/or StarFive VisionFive V1 boards. +config ERRATA_SIFIVE_MAL_9092 + bool "Apply SiFive HLV/HLVX/HSV fence errata" + depends on ERRATA_SIFIVE && 64BIT + default y + help + This will apply the SiFive MAL-9092 errata to add fence.rw + instructions around all HLV/HLVX/HSV instructions. + + Note: HSTATUS.HU is cleared during KVM initialization. If future + changes enable HSTATUS.HU to allow hypervisor instructions in + U-mode, additional errata handling may be required. + + If you don't know what to do here, say "Y". + config ERRATA_THEAD bool "T-HEAD errata" depends on RISCV_ALTERNATIVE @@ -146,6 +160,7 @@ config ERRATA_THEAD_PMU config ERRATA_THEAD_GHOSTWRITE bool "Apply T-Head Ghostwrite errata" depends on ERRATA_THEAD && RISCV_ISA_XTHEADVECTOR + select RISCV_ALTERNATIVE_EARLY default y help The T-Head C9xx cores have a vulnerability in the xtheadvector diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs index 429e07589306..cecbce0c4983 100644 --- a/arch/riscv/Kconfig.socs +++ b/arch/riscv/Kconfig.socs @@ -34,6 +34,7 @@ config ARCH_RENESAS config ARCH_SIFIVE bool "SiFive SoCs" select ERRATA_SIFIVE + depends on 64BIT help This enables support for SiFive SoC platform hardware. diff --git a/arch/riscv/errata/sifive/errata.c b/arch/riscv/errata/sifive/errata.c index df80c9614df1..1a5d223ebf35 100644 --- a/arch/riscv/errata/sifive/errata.c +++ b/arch/riscv/errata/sifive/errata.c @@ -51,6 +51,22 @@ static bool errata_cip_1200_check_func(unsigned long arch_id, unsigned long imp return true; } +static bool errata_mal_9092_check_func(unsigned long arch_id, unsigned long impid) +{ + /* + * Affected cores: + * Architecture ID: 0x8000000000000109 + * Implementation ID: 0x19251031, 0x19251217, 0x19260320 + */ + + if (arch_id != 0x8000000000000109) + return false; + if (impid != 0x19251031 && impid != 0x19251217 && impid != 0x19260320) + return false; + + return true; +} + static struct errata_info_t errata_list[ERRATA_SIFIVE_NUMBER] = { { .name = "cip-453", @@ -60,6 +76,10 @@ static struct errata_info_t errata_list[ERRATA_SIFIVE_NUMBER] = { .name = "cip-1200", .check_func = errata_cip_1200_check_func }, + { + .name = "mal-9092", + .check_func = errata_mal_9092_check_func + }, }; static u32 __init_or_module sifive_errata_probe(unsigned long archid, diff --git a/arch/riscv/include/asm/cpufeature.h b/arch/riscv/include/asm/cpufeature.h index 739fcc84bf7b..5efa72823475 100644 --- a/arch/riscv/include/asm/cpufeature.h +++ b/arch/riscv/include/asm/cpufeature.h @@ -40,6 +40,8 @@ extern u32 thead_vlenb_of; void __init riscv_user_isa_enable(void); +void riscv_clear_hypervisor_csr(void); + #define _RISCV_ISA_EXT_DATA(_name, _id, _subset_exts, _subset_exts_size, _validate) { \ .name = #_name, \ .property = #_name, \ diff --git a/arch/riscv/include/asm/errata_list_vendors.h b/arch/riscv/include/asm/errata_list_vendors.h index ec7eba373437..c62a82f60468 100644 --- a/arch/riscv/include/asm/errata_list_vendors.h +++ b/arch/riscv/include/asm/errata_list_vendors.h @@ -11,7 +11,8 @@ #ifdef CONFIG_ERRATA_SIFIVE #define ERRATA_SIFIVE_CIP_453 0 #define ERRATA_SIFIVE_CIP_1200 1 -#define ERRATA_SIFIVE_NUMBER 2 +#define ERRATA_SIFIVE_MAL_9092 2 +#define ERRATA_SIFIVE_NUMBER 3 #endif #ifdef CONFIG_ERRATA_THEAD diff --git a/arch/riscv/include/asm/futex.h b/arch/riscv/include/asm/futex.h index 90c86b115e00..6f3bc9bdb85e 100644 --- a/arch/riscv/include/asm/futex.h +++ b/arch/riscv/include/asm/futex.h @@ -40,6 +40,7 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; + uaddr = untagged_addr(uaddr); switch (op) { case FUTEX_OP_SET: @@ -82,6 +83,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; + uaddr = untagged_addr(uaddr); __enable_user_access(); __asm__ __volatile__ ( diff --git a/arch/riscv/include/asm/insn-def.h b/arch/riscv/include/asm/insn-def.h index 7c6daf116756..8fcc848c196e 100644 --- a/arch/riscv/include/asm/insn-def.h +++ b/arch/riscv/include/asm/insn-def.h @@ -192,18 +192,28 @@ INSN_R(OPCODE_SYSTEM, FUNC3(0), FUNC7(49), \ __RD(0), RS1(gaddr), RS2(vmid)) +#define ALT_SIFIVE_MAL_9092_FENCE \ +ALTERNATIVE("nop", "fence rw, rw", SIFIVE_VENDOR_ID, \ + ERRATA_SIFIVE_MAL_9092, CONFIG_ERRATA_SIFIVE_MAL_9092) + #define HLVX_HU(dest, addr) \ + ALT_SIFIVE_MAL_9092_FENCE "\n" \ INSN_R(OPCODE_SYSTEM, FUNC3(4), FUNC7(50), \ - RD(dest), RS1(addr), __RS2(3)) + RD(dest), RS1(addr), __RS2(3)) "\n" \ + ALT_SIFIVE_MAL_9092_FENCE #define HLV_W(dest, addr) \ + ALT_SIFIVE_MAL_9092_FENCE "\n" \ INSN_R(OPCODE_SYSTEM, FUNC3(4), FUNC7(52), \ - RD(dest), RS1(addr), __RS2(0)) + RD(dest), RS1(addr), __RS2(0)) "\n" \ + ALT_SIFIVE_MAL_9092_FENCE #ifdef CONFIG_64BIT #define HLV_D(dest, addr) \ + ALT_SIFIVE_MAL_9092_FENCE "\n" \ INSN_R(OPCODE_SYSTEM, FUNC3(4), FUNC7(54), \ - RD(dest), RS1(addr), __RS2(0)) + RD(dest), RS1(addr), __RS2(0)) "\n" \ + ALT_SIFIVE_MAL_9092_FENCE #else #define HLV_D(dest, addr) \ __ASM_STR(.error "hlv.d requires 64-bit support") diff --git a/arch/riscv/include/asm/scs.h b/arch/riscv/include/asm/scs.h index 023a412fe38d..49e40eef4ed7 100644 --- a/arch/riscv/include/asm/scs.h +++ b/arch/riscv/include/asm/scs.h @@ -4,6 +4,7 @@ #ifdef __ASSEMBLER__ #include +#include #ifdef CONFIG_SHADOW_CALL_STACK @@ -22,9 +23,17 @@ REG_L gp, TASK_TI_SCS_SP(tp) .endm -/* Load task_scs_sp(current) to gp, but only if tp has changed. */ -.macro scs_load_current_if_task_changed prev - beq \prev, tp, _skip_scs +/* + * Load task_scs_sp(current) to gp, but only when the trap came from U-mode. + * The source privilege level is taken from the saved sstatus.SPP bit (written + * by hardware on trap entry), not by comparing tp. tp is a user-writable + * register, so gating the shadow call stack reload on it let a user task that + * set tp == ¤t skip the reload and run the kernel with an attacker + * controlled gp (the shadow call stack pointer). + */ +.macro scs_load_current_if_from_user status, tmp + andi \tmp, \status, SR_SPP + bnez \tmp, _skip_scs scs_load_current _skip_scs: .endm @@ -42,7 +51,7 @@ _skip_scs: .endm .macro scs_load_current .endm -.macro scs_load_current_if_task_changed prev +.macro scs_load_current_if_from_user status, tmp .endm .macro scs_save_current .endm diff --git a/arch/riscv/include/asm/vector.h b/arch/riscv/include/asm/vector.h index fffe72a77208..1f8dcae3af51 100644 --- a/arch/riscv/include/asm/vector.h +++ b/arch/riscv/include/asm/vector.h @@ -137,8 +137,8 @@ static __always_inline void __vstate_csr_save(struct __riscv_v_ext_state *dest) "csrr %0, " __stringify(CSR_VSTART) "\n\t" "csrr %1, " __stringify(CSR_VTYPE) "\n\t" "csrr %2, " __stringify(CSR_VL) "\n\t" - : "=r" (dest->vstart), "=r" (dest->vtype), "=r" (dest->vl), - "=r" (dest->vcsr) : :); + : "=r" (dest->vstart), "=r" (dest->vtype), "=r" (dest->vl) + : :); if (has_xtheadvector()) { unsigned long status; @@ -230,7 +230,7 @@ static inline void __riscv_v_vstate_save(struct __riscv_v_ext_state *save_to, "add %1, %1, %0\n\t" "vse8.v v24, (%1)\n\t" ".option pop\n\t" - : "=&r" (vl) : "r" (datap) : "memory"); + : "=&r" (vl), "+r" (datap) : : "memory"); } riscv_v_disable(); } @@ -266,7 +266,7 @@ static inline void __riscv_v_vstate_restore(struct __riscv_v_ext_state *restore_ "add %1, %1, %0\n\t" "vle8.v v24, (%1)\n\t" ".option pop\n\t" - : "=&r" (vl) : "r" (datap) : "memory"); + : "=&r" (vl), "+r" (datap) : : "memory"); } __vstate_csr_restore(restore_from); riscv_v_disable(); diff --git a/arch/riscv/include/uapi/asm/vendor/mips.h b/arch/riscv/include/uapi/asm/vendor/mips.h index e65ab268b265..a31c23afe74f 100644 --- a/arch/riscv/include/uapi/asm/vendor/mips.h +++ b/arch/riscv/include/uapi/asm/vendor/mips.h @@ -1,3 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#define RISCV_HWPROBE_VENDOR_EXT_XMIPSEXECTL BIT(0) +#include + +#define RISCV_HWPROBE_VENDOR_EXT_XMIPSEXECTL _BITULL(0) diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 61d21f714830..7329792978cc 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -1233,6 +1233,18 @@ void __init riscv_user_isa_enable(void) pr_warn("Zicbop disabled as it is unavailable on some harts\n"); } +void riscv_clear_hypervisor_csr(void) +{ + if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_H)) + return; + + /* + * Clear HSTATUS.HU to restrict hypervisor instructions to HS-mode. + * This prevents user-mode from executing HLV/HSV instructions. + */ + csr_clear(CSR_HSTATUS, HSTATUS_HU); +} + #ifdef CONFIG_RISCV_ALTERNATIVE /* * Alternative patch sites consider 48 bits when determining when to patch diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index d799c4e56f80..61b7a7716697 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -208,7 +208,7 @@ SYM_CODE_START(handle_exception) load_global_pointer /* Load the kernel shadow call stack pointer if coming from userspace */ - scs_load_current_if_task_changed s5 + scs_load_current_if_from_user s1, s5 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE move a0, sp diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c index 2239c28981bc..2b5adf01c550 100644 --- a/arch/riscv/kernel/patch.c +++ b/arch/riscv/kernel/patch.c @@ -48,7 +48,7 @@ static __always_inline void *patch_map(void *addr, const unsigned int fixmap) if (!IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) return addr; phys = __pa_symbol(addr); - } else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) { + } else if (IS_ENABLED(CONFIG_EXECMEM)) { struct page *page = vmalloc_to_page(addr); BUG_ON(!page); diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c index f336a183667e..a0055743d3ba 100644 --- a/arch/riscv/kernel/ptrace.c +++ b/arch/riscv/kernel/ptrace.c @@ -230,6 +230,7 @@ static int riscv_vr_set(struct task_struct *target, return -ENODATA; /* Copy rest of the vstate except datap */ + memset(&ptrace_vstate, 0, sizeof(ptrace_vstate)); ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ptrace_vstate, 0, sizeof(struct __riscv_v_regset_state)); if (unlikely(ret)) @@ -299,6 +300,9 @@ static int riscv_cfi_get(struct task_struct *target, struct user_cfi_state user_cfi; struct pt_regs *regs; + if (!is_user_lpad_enabled() && !is_user_shstk_enabled()) + return -EINVAL; + memset(&user_cfi, 0, sizeof(user_cfi)); regs = task_pt_regs(target); @@ -337,8 +341,12 @@ static int riscv_cfi_set(struct task_struct *target, struct user_cfi_state user_cfi; struct pt_regs *regs; + if (!is_user_lpad_enabled() && !is_user_shstk_enabled()) + return -EINVAL; + regs = task_pt_regs(target); + memset(&user_cfi, 0, sizeof(user_cfi)); ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &user_cfi, 0, -1); if (ret) return ret; diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index a32344bb220d..d79f99b86033 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -366,6 +366,8 @@ void __init setup_arch(char **cmdline_p) if (!IS_ENABLED(CONFIG_RISCV_ISA_ZBB) || !riscv_isa_extension_available(NULL, ZBB)) static_branch_disable(&efficient_ffs_key); + + riscv_clear_hypervisor_csr(); } bool arch_cpu_is_hotpluggable(int cpu) diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index f6ef57930b50..a9de2dae804c 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -244,6 +244,8 @@ asmlinkage __visible void smp_callin(void) numa_add_cpu(curr_cpuid); + riscv_clear_hypervisor_csr(); + pr_debug("CPU%u: Booted secondary hartid %lu\n", curr_cpuid, cpuid_to_hartid_map(curr_cpuid)); diff --git a/arch/riscv/kernel/suspend.c b/arch/riscv/kernel/suspend.c index 3efbf7874f3b..db220966f782 100644 --- a/arch/riscv/kernel/suspend.c +++ b/arch/riscv/kernel/suspend.c @@ -43,6 +43,8 @@ void suspend_save_csrs(struct suspend_context *context) void suspend_restore_csrs(struct suspend_context *context) { + riscv_clear_hypervisor_csr(); + csr_write(CSR_SCRATCH, 0); if (riscv_has_extension_unlikely(RISCV_ISA_EXT_XLINUXENVCFG)) csr_write(CSR_ENVCFG, context->envcfg); diff --git a/arch/riscv/lib/strnlen.S b/arch/riscv/lib/strnlen.S index a8911605c248..5b3bbf0a3098 100644 --- a/arch/riscv/lib/strnlen.S +++ b/arch/riscv/lib/strnlen.S @@ -87,9 +87,25 @@ strnlen_zbb: * Aligned boundary. Use the address of the last valid byte * (s + count - 1) to avoid loading a word past the count * boundary in the loop below. count == 0 is handled above. + * + * Saturate the boundary when s + count would wrap around (very + * large counts, e.g. SIZE_MAX passed by FORTIFY strcat/strlcat + * with a destination whose size is unknown at compile time). + * Without this, the wrapped boundary lands before s and the + * pre-loop guard below always exits, returning a truncated + * length. + * + * Clamping the increment to ~s (== SIZE_MAX - s) keeps the + * computation branchless and wrap-free: + * + * s + min(count - 1, ~s) == saturating_add(s, count - 1) + * + * Saturating makes the scan equivalent to strlen(). */ - add t4, a0, a1 - addi t4, t4, -1 + addi t4, a1, -1 /* count - 1 */ + not t1, a0 /* SIZE_MAX - s */ + minu t4, t4, t1 /* clamp, so s + t4 can never wrap */ + add t4, a0, t4 /* saturated s + count - 1 */ andi t4, t4, -SZREG /* Get the first word. */ diff --git a/arch/riscv/mm/extable.c b/arch/riscv/mm/extable.c index dd1530af3ef1..e252eba55ede 100644 --- a/arch/riscv/mm/extable.c +++ b/arch/riscv/mm/extable.c @@ -68,8 +68,8 @@ ex_handler_load_unaligned_zeropad(const struct exception_table_entry *ex, addr = regs_get_gpr(regs, reg_addr * sizeof(unsigned long)); - offset = addr & 0x7UL; - addr &= ~0x7UL; + offset = addr & (sizeof(data) - 1); + addr &= ~(sizeof(data) - 1); data = *(unsigned long *)addr >> (offset * 8);