From c5ac64f10a60667f41dfb44e8e7229a7690a826f Mon Sep 17 00:00:00 2001 From: "Christoph Lameter (Ampere)" Date: Wed, 6 Dec 2023 20:11:49 -0800 Subject: [PATCH] ARM64: Convert TLB page flushing Move the functions related to individual page flushing and convert them to use the new infrastructure. Signed-off-by: Christoph Lameter (Ampere) --- arch/arm64/include/asm/tlbflush.h | 33 +++++++++++------------ arch/arm64/mm/context.c | 45 +++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 18 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index 8f0d4b48758b..8b06c7ec1fe5 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -250,20 +250,21 @@ static inline void flush_tlb_all(void) _count_vm_tlb_event(NR_TLB_FLUSH_ALL); } -extern void flush_tlb_mm(struct mm_struct *mm); +void flush_tlb_mm(struct mm_struct *mm); + +void __flush_tlb_page(struct mm_struct *mm, + unsigned long uaddr, bool sync); + +static inline void flush_tlb_page(struct vm_area_struct *vma, + unsigned long uaddr) +{ + __flush_tlb_page(vma->vm_mm, uaddr, true); +} static inline void __flush_tlb_page_nosync(struct mm_struct *mm, unsigned long uaddr) { - unsigned long addr; - - dsb(ishst); - addr = __TLBI_VADDR(uaddr, ASID(mm)); - __tlbi(vale1is, addr); - __tlbi_user(vale1is, addr); - mmu_notifier_arch_invalidate_secondary_tlbs(mm, uaddr & PAGE_MASK, - (uaddr & PAGE_MASK) + PAGE_SIZE); - _count_vm_tlb_event(NR_TLB_FLUSH_ONE); + __flush_tlb_page(mm, uaddr, false); } static inline void flush_tlb_page_nosync(struct vm_area_struct *vma, @@ -272,12 +273,6 @@ static inline void flush_tlb_page_nosync(struct vm_area_struct *vma, return __flush_tlb_page_nosync(vma->vm_mm, uaddr); } -static inline void flush_tlb_page(struct vm_area_struct *vma, - unsigned long uaddr) -{ - flush_tlb_page_nosync(vma, uaddr); - dsb(ish); -} static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm) { @@ -293,6 +288,8 @@ static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm) return true; } +void __tlbbatch_flush(void); + static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch, struct mm_struct *mm, unsigned long uaddr) @@ -307,7 +304,7 @@ static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *b */ static inline void arch_flush_tlb_batched_pending(struct mm_struct *mm) { - dsb(ish); + __tlbbatch_flush(); } /* @@ -322,7 +319,7 @@ static inline void arch_flush_tlb_batched_pending(struct mm_struct *mm) */ static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) { - dsb(ish); + __tlbbatch_flush(); } /* diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index d11213f82d8e..418d9931ae53 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -598,6 +598,51 @@ void flush_tlb_mm(struct mm_struct *mm) mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL); } +struct ipi_flush_tlb_page_param { + unsigned long uaddr; + struct mm_struct *mm; +}; + +static inline void ipi_flush_tlb_page(void *p) +{ + struct ipi_flush_tlb_page_param *i = p; + + flush_tlb_pre(TLB_LOCAL); + flush_tlb_addr(TLB_LOCAL, i->mm, i->uaddr); + flush_tlb_post(TLB_LOCAL); + count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED); +} + +void __flush_tlb_page(struct mm_struct *mm, + unsigned long uaddr, bool sync) +{ + struct ipi_flush_tlb_page_param i = { uaddr, mm }; + enum tlb_state ts = tlbstat_mm(i.mm); + + if (ts == TLB_IPI) { + + on_each_cpu_mask(mm_cpumask(i.mm), ipi_flush_tlb_page, &i, true); + count_vm_tlb_event(NR_TLB_REMOTE_FLUSH); + + } else { + + flush_tlb_pre(ts); + flush_tlb_addr(ts, i.mm, uaddr); + + if (sync) + flush_tlb_post(ts); + + } + + mmu_notifier_arch_invalidate_secondary_tlbs(i.mm, uaddr & PAGE_MASK, + (uaddr & PAGE_MASK) + PAGE_SIZE); +} + +void __tlbbatch_flush(void) +{ + flush_tlb_post(TLB_BROADCAST); +} + static ssize_t tlb_mode_read_file(struct file *file, char __user *user_buf, size_t count, loff_t *ppos) { -- 2.47.3