diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index 4e0e8898c5d4..211119ce7adc 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -202,6 +202,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V3AE | #3312417 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | C1-Pro | #4193714 | ARM64_ERRATUM_4193714 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU-500 | #841119,826419 | ARM_SMMU_MMU_500_CPRE_ERRATA| | | | #562869,1047329 | | +----------------+-----------------+-----------------+-----------------------------+ diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index a0af483c0488..fe60738e5943 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1142,6 +1142,18 @@ config ARM64_ERRATUM_4311569 If unsure, say Y. +config ARM64_ERRATUM_4193714 + bool "C1-Pro: 4193714: SME DVMSync early acknowledgement" + depends on ARM64_SME + default y + help + Enable workaround for C1-Pro acknowledging the DVMSync before + the SME memory accesses are complete. This will cause TLB + maintenance for processes using SME to also issue an IPI to + the affected CPUs. + + If unsure, say Y. + config CAVIUM_ERRATUM_22375 bool "Cavium erratum 22375, 24313" default y diff --git a/arch/arm64/include/asm/Kbuild b/arch/arm64/include/asm/Kbuild index d2ff8f6c3231..31441790b808 100644 --- a/arch/arm64/include/asm/Kbuild +++ b/arch/arm64/include/asm/Kbuild @@ -17,4 +17,5 @@ generic-y += parport.h generic-y += user.h generated-y += cpucap-defs.h +generated-y += kernel-hwcap.h generated-y += sysreg-defs.h diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index 6e3da333442e..d0d3cdd5763c 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -64,6 +64,8 @@ cpucap_is_possible(const unsigned int cap) return IS_ENABLED(CONFIG_ARM64_WORKAROUND_REPEAT_TLBI); case ARM64_WORKAROUND_SPECULATIVE_SSBS: return IS_ENABLED(CONFIG_ARM64_ERRATUM_3194386); + case ARM64_WORKAROUND_4193714: + return IS_ENABLED(CONFIG_ARM64_ERRATUM_4193714); case ARM64_MPAM: /* * KVM MPAM support doesn't rely on the host kernel supporting MPAM. diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 08860d482e60..7b518e81dd15 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -98,6 +98,7 @@ #define ARM_CPU_PART_CORTEX_A725 0xD87 #define ARM_CPU_PART_CORTEX_A720AE 0xD89 #define ARM_CPU_PART_NEOVERSE_N3 0xD8E +#define ARM_CPU_PART_C1_PRO 0xD8B #define APM_CPU_PART_XGENE 0x000 #define APM_CPU_VAR_POTENZA 0x00 @@ -189,6 +190,7 @@ #define MIDR_CORTEX_A725 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A725) #define MIDR_CORTEX_A720AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A720AE) #define MIDR_NEOVERSE_N3 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N3) +#define MIDR_C1_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PRO) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) #define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX) diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 1d2e33559bd5..d9d00b45ab11 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -428,6 +428,24 @@ static inline size_t sme_state_size(struct task_struct const *task) return __sme_state_size(task_get_sme_vl(task)); } +void sme_enable_dvmsync(void); +void sme_set_active(void); +void sme_clear_active(void); + +static inline void sme_enter_from_user_mode(void) +{ + if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714) && + test_thread_flag(TIF_SME)) + sme_clear_active(); +} + +static inline void sme_exit_to_user_mode(void) +{ + if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714) && + test_thread_flag(TIF_SME)) + sme_set_active(); +} + #else static inline void sme_user_disable(void) { BUILD_BUG(); } @@ -456,6 +474,9 @@ static inline size_t sme_state_size(struct task_struct const *task) return 0; } +static inline void sme_enter_from_user_mode(void) { } +static inline void sme_exit_to_user_mode(void) { } + #endif /* ! CONFIG_ARM64_SME */ /* For use by EFI runtime services calls only */ diff --git a/arch/arm64/include/asm/tlbbatch.h b/arch/arm64/include/asm/tlbbatch.h index fedb0b87b8db..6297631532e5 100644 --- a/arch/arm64/include/asm/tlbbatch.h +++ b/arch/arm64/include/asm/tlbbatch.h @@ -2,11 +2,17 @@ #ifndef _ARCH_ARM64_TLBBATCH_H #define _ARCH_ARM64_TLBBATCH_H +#include + struct arch_tlbflush_unmap_batch { +#ifdef CONFIG_ARM64_ERRATUM_4193714 /* - * For arm64, HW can do tlb shootdown, so we don't - * need to record cpumask for sending IPI + * Track CPUs that need SME DVMSync on completion of this batch. + * Otherwise, the arm64 HW can do tlb shootdown, so we don't need to + * record cpumask for sending IPI */ + cpumask_var_t cpumask; +#endif }; #endif /* _ARCH_ARM64_TLBBATCH_H */ diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index 47fa4d39a461..c0bf5b398041 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -80,6 +80,71 @@ static inline unsigned long get_trans_granule(void) } } +#ifdef CONFIG_ARM64_ERRATUM_4193714 + +void sme_do_dvmsync(const struct cpumask *mask); + +static inline void sme_dvmsync(struct mm_struct *mm) +{ + if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) + return; + + sme_do_dvmsync(mm_cpumask(mm)); +} + +static inline void sme_dvmsync_add_pending(struct arch_tlbflush_unmap_batch *batch, + struct mm_struct *mm) +{ + if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) + return; + + /* + * Order the mm_cpumask() read after the hardware DVMSync. + */ + dsb(ish); + if (cpumask_empty(mm_cpumask(mm))) + return; + + /* + * Allocate the batch cpumask on first use. Fall back to an immediate + * IPI for this mm in case of failure. + */ + if (!cpumask_available(batch->cpumask) && + !zalloc_cpumask_var(&batch->cpumask, GFP_ATOMIC)) { + sme_do_dvmsync(mm_cpumask(mm)); + return; + } + + cpumask_or(batch->cpumask, batch->cpumask, mm_cpumask(mm)); +} + +static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch) +{ + if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) + return; + + if (!cpumask_available(batch->cpumask)) + return; + + sme_do_dvmsync(batch->cpumask); + cpumask_clear(batch->cpumask); +} + +#else + +static inline void sme_dvmsync(struct mm_struct *mm) +{ +} +static inline void sme_dvmsync_add_pending(struct arch_tlbflush_unmap_batch *batch, + struct mm_struct *mm) +{ +} +static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch) +{ +} + +#endif /* CONFIG_ARM64_ERRATUM_4193714 */ + /* * Level-based TLBI operations. * @@ -213,7 +278,21 @@ do { \ * Complete broadcast TLB maintenance issued by the host which invalidates * stage 1 information in the host's own translation regime. */ -static inline void __tlbi_sync_s1ish(void) +static inline void __tlbi_sync_s1ish(struct mm_struct *mm) +{ + dsb(ish); + __repeat_tlbi_sync(vale1is, 0); + sme_dvmsync(mm); +} + +static inline void __tlbi_sync_s1ish_batch(struct arch_tlbflush_unmap_batch *batch) +{ + dsb(ish); + __repeat_tlbi_sync(vale1is, 0); + sme_dvmsync_batch(batch); +} + +static inline void __tlbi_sync_s1ish_kernel(void) { dsb(ish); __repeat_tlbi_sync(vale1is, 0); @@ -322,7 +401,7 @@ static inline void flush_tlb_all(void) { dsb(ishst); __tlbi(vmalle1is); - __tlbi_sync_s1ish(); + __tlbi_sync_s1ish_kernel(); isb(); } @@ -334,7 +413,7 @@ static inline void flush_tlb_mm(struct mm_struct *mm) asid = __TLBI_VADDR(0, ASID(mm)); __tlbi(aside1is, asid); __tlbi_user(aside1is, asid); - __tlbi_sync_s1ish(); + __tlbi_sync_s1ish(mm); mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL); } @@ -355,7 +434,7 @@ static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm) */ static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) { - __tlbi_sync_s1ish(); + __tlbi_sync_s1ish_batch(batch); } /* @@ -557,7 +636,7 @@ static __always_inline void __do_flush_tlb_range(struct vm_area_struct *vma, if (!(flags & TLBF_NOSYNC)) { if (!(flags & TLBF_NOBROADCAST)) - __tlbi_sync_s1ish(); + __tlbi_sync_s1ish(mm); else dsb(nsh); } @@ -618,7 +697,7 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end dsb(ishst); __flush_s1_tlb_range_op(vaale1is, start, pages, stride, 0, TLBI_TTL_UNKNOWN); - __tlbi_sync_s1ish(); + __tlbi_sync_s1ish_kernel(); isb(); } @@ -632,7 +711,7 @@ static inline void __flush_tlb_kernel_pgtable(unsigned long kaddr) dsb(ishst); __tlbi(vaae1is, addr); - __tlbi_sync_s1ish(); + __tlbi_sync_s1ish_kernel(); isb(); } @@ -643,6 +722,7 @@ static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *b __flush_tlb_range(&vma, start, end, PAGE_SIZE, 3, TLBF_NOWALKCACHE | TLBF_NOSYNC); + sme_dvmsync_add_pending(batch, mm); } static inline bool __pte_flags_need_flush(ptdesc_t oldval, ptdesc_t newval) diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 5c0ab6bfd44a..5377e4c2eba2 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -575,6 +576,23 @@ static const struct midr_range erratum_spec_ssbs_list[] = { }; #endif +#ifdef CONFIG_ARM64_ERRATUM_4193714 +static bool has_sme_dvmsync_erratum(const struct arm64_cpu_capabilities *entry, + int scope) +{ + if (!id_aa64pfr1_sme(read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1))) + return false; + + return is_affected_midr_range(entry, scope); +} + +static void cpu_enable_sme_dvmsync(const struct arm64_cpu_capabilities *__unused) +{ + if (this_cpu_has_cap(ARM64_WORKAROUND_4193714)) + sme_enable_dvmsync(); +} +#endif + #ifdef CONFIG_AMPERE_ERRATUM_AC03_CPU_38 static const struct midr_range erratum_ac03_cpu_38_list[] = { MIDR_ALL_VERSIONS(MIDR_AMPERE1), @@ -901,6 +919,18 @@ const struct arm64_cpu_capabilities arm64_errata[] = { .matches = need_arm_si_l1_workaround_4311569, }, #endif +#ifdef CONFIG_ARM64_ERRATUM_4193714 + { + .desc = "C1-Pro SME DVMSync early acknowledgement", + .capability = ARM64_WORKAROUND_4193714, + .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, + .matches = has_sme_dvmsync_erratum, + .cpu_enable = cpu_enable_sme_dvmsync, + /* C1-Pro r0p0 - r1p2 (the latter only when REVIDR_EL1[0]==0) */ + .midr_range = MIDR_RANGE(MIDR_C1_PRO, 0, 0, 1, 2), + MIDR_FIXED(MIDR_CPU_VAR_REV(1, 2), BIT(0)), + }, +#endif #ifdef CONFIG_ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD { .desc = "ARM errata 2966298, 3117295", diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index f42ce7b5c67f..cb54335465f6 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -70,6 +71,7 @@ static __always_inline void arm64_enter_from_user_mode(struct pt_regs *regs) { enter_from_user_mode(regs); mte_disable_tco_entry(current); + sme_enter_from_user_mode(); } /* @@ -83,6 +85,7 @@ static __always_inline void arm64_exit_to_user_mode(struct pt_regs *regs) local_irq_disable(); exit_to_user_mode_prepare_legacy(regs); local_daif_mask(); + sme_exit_to_user_mode(); mte_check_tfsr_exit(); exit_to_user_mode(); } diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 9de1d8a604cb..60a45d600b46 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -1358,6 +1360,83 @@ void do_sve_acc(unsigned long esr, struct pt_regs *regs) put_cpu_fpsimd_context(); } +#ifdef CONFIG_ARM64_ERRATUM_4193714 + +/* + * SME/CME erratum handling. + */ +static cpumask_t sme_dvmsync_cpus; + +/* + * These helpers are only called from non-preemptible contexts, so + * smp_processor_id() is safe here. + */ +void sme_set_active(void) +{ + unsigned int cpu = smp_processor_id(); + + if (!cpumask_test_cpu(cpu, &sme_dvmsync_cpus)) + return; + + cpumask_set_cpu(cpu, mm_cpumask(current->mm)); + + /* + * A subsequent (post ERET) SME access may use a stale address + * translation. On C1-Pro, a TLBI+DSB on a different CPU will wait for + * the completion of cpumask_set_cpu() above as it appears in program + * order before the SME access. The post-TLBI+DSB read of mm_cpumask() + * will lead to the IPI being issued. + * + * https://lore.kernel.org/r/ablEXwhfKyJW1i7l@J2N7QTR9R3 + */ +} + +void sme_clear_active(void) +{ + unsigned int cpu = smp_processor_id(); + + if (!cpumask_test_cpu(cpu, &sme_dvmsync_cpus)) + return; + + /* + * With SCTLR_EL1.IESB enabled, the SME memory transactions are + * completed on entering EL1. + */ + cpumask_clear_cpu(cpu, mm_cpumask(current->mm)); +} + +static void sme_dvmsync_ipi(void *unused) +{ + /* + * With SCTLR_EL1.IESB on, taking an exception is sufficient to ensure + * the completion of the SME memory accesses, so no need for an + * explicit DSB. + */ +} + +void sme_do_dvmsync(const struct cpumask *mask) +{ + /* + * This is called from the TLB maintenance functions after the DSB ISH + * to send the hardware DVMSync message. If this CPU sees the mask as + * empty, the remote CPU executing sme_set_active() would have seen + * the DVMSync and no IPI required. + */ + if (cpumask_empty(mask)) + return; + + preempt_disable(); + smp_call_function_many(mask, sme_dvmsync_ipi, NULL, true); + preempt_enable(); +} + +void sme_enable_dvmsync(void) +{ + cpumask_set_cpu(smp_processor_id(), &sme_dvmsync_cpus); +} + +#endif /* CONFIG_ARM64_ERRATUM_4193714 */ + /* * Trapped SME access * diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index c0bf1f46cdc6..033643cd4e5e 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -340,8 +341,41 @@ void flush_thread(void) flush_gcs(); } +#ifdef CONFIG_ARM64_ERRATUM_4193714 + +static void arch_dup_tlbbatch_mask(struct task_struct *dst) +{ + /* + * Clear the inherited cpumask with memset() to cover both cases where + * cpumask_var_t is a pointer or an array. It will be allocated lazily + * in sme_dvmsync_add_pending() if CPUMASK_OFFSTACK=y. + */ + if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) + memset(&dst->tlb_ubc.arch.cpumask, 0, + sizeof(dst->tlb_ubc.arch.cpumask)); +} + +static void arch_release_tlbbatch_mask(struct task_struct *tsk) +{ + if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) + free_cpumask_var(tsk->tlb_ubc.arch.cpumask); +} + +#else + +static void arch_dup_tlbbatch_mask(struct task_struct *dst) +{ +} + +static void arch_release_tlbbatch_mask(struct task_struct *tsk) +{ +} + +#endif /* CONFIG_ARM64_ERRATUM_4193714 */ + void arch_release_task_struct(struct task_struct *tsk) { + arch_release_tlbbatch_mask(tsk); fpsimd_release_task(tsk); } @@ -357,6 +391,8 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) *dst = *src; + arch_dup_tlbbatch_mask(dst); + /* * Drop stale reference to src's sve_state and convert dst to * non-streaming FPSIMD mode. diff --git a/arch/arm64/kernel/sys_compat.c b/arch/arm64/kernel/sys_compat.c index 7e9860143add..0451f96c2c3f 100644 --- a/arch/arm64/kernel/sys_compat.c +++ b/arch/arm64/kernel/sys_compat.c @@ -37,7 +37,7 @@ __do_compat_cache_op(unsigned long start, unsigned long end) * We pick the reserved-ASID to minimise the impact. */ __tlbi(aside1is, 0UL); - __tlbi_sync_s1ish(); + __tlbi_sync_s1ish(current->mm); } ret = caches_clean_inval_user_pou(start, start + chunk); diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index b7286d977788..811c2479e82d 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -106,6 +106,7 @@ WORKAROUND_2077057 WORKAROUND_2457168 WORKAROUND_2645198 WORKAROUND_2658417 +WORKAROUND_4193714 WORKAROUND_4311569 WORKAROUND_AMPERE_AC03_CPU_38 WORKAROUND_AMPERE_AC04_CPU_23 diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index a9938006d0e6..226ff6f532fa 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -22,7 +22,7 @@ #include "mpam_internal.h" -DECLARE_WAIT_QUEUE_HEAD(resctrl_mon_ctx_waiters); +static DECLARE_WAIT_QUEUE_HEAD(resctrl_mon_ctx_waiters); /* * The classes we've picked to map to resctrl resources, wrapped @@ -220,10 +220,18 @@ int resctrl_arch_set_cdp_enabled(enum resctrl_res_level rid, bool enable) if (cdp_enabled && !mpam_resctrl_controls[RDT_RESOURCE_MBA].cdp_enabled) mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = false; + /* + * If resctrl has attempted to enable CDP on MBA, re-enable MBA as two + * configurations will be provided so there is no aliasing problem. + */ if (mpam_resctrl_controls[RDT_RESOURCE_MBA].cdp_enabled && mpam_resctrl_controls[RDT_RESOURCE_MBA].class) mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = true; + /* On unmount when CDP is disabled, re-enable MBA */ + if (!cdp_enabled && mpam_resctrl_controls[RDT_RESOURCE_MBA].class) + mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = true; + if (enable) { if (mpam_partid_max < 1) return -EINVAL; @@ -1399,7 +1407,7 @@ mpam_resctrl_alloc_domain(unsigned int cpu, struct mpam_resctrl_res *res) } if (r->mon_capable) { - struct mpam_component *any_mon_comp; + struct mpam_component *any_mon_comp = NULL; struct mpam_resctrl_mon *mon; enum resctrl_event_id eventid; diff --git a/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c b/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c index 0c9ea24a200c..66d00b6ceb78 100644 --- a/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c +++ b/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c @@ -157,7 +157,8 @@ static int arm_cca_report_new(struct tsm_report *report, void *data) } while (info.result == RSI_INCOMPLETE && info.offset < RSI_GRANULE_SIZE); - if (info.result != RSI_SUCCESS) { + /* Break out in case of failure */ + if (info.result != RSI_SUCCESS && info.result != RSI_INCOMPLETE) { ret = -ENXIO; token_size = 0; goto exit_free_granule_page;