2005-04-17 06:20:36 +08:00
|
|
|
#ifndef __ASM_SH_TLB_H
|
|
|
|
#define __ASM_SH_TLB_H
|
|
|
|
|
2007-11-10 19:35:53 +08:00
|
|
|
#ifdef CONFIG_SUPERH64
|
2012-10-03 01:01:25 +08:00
|
|
|
# include <asm/tlb_64.h>
|
2007-11-10 19:35:53 +08:00
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifndef __ASSEMBLY__
|
2009-03-17 20:19:49 +08:00
|
|
|
#include <linux/pagemap.h>
|
|
|
|
|
|
|
|
#ifdef CONFIG_MMU
|
2011-05-31 12:27:41 +08:00
|
|
|
#include <linux/swap.h>
|
2009-03-17 20:19:49 +08:00
|
|
|
#include <asm/pgalloc.h>
|
|
|
|
#include <asm/tlbflush.h>
|
2009-11-20 05:11:05 +08:00
|
|
|
#include <asm/mmu_context.h>
|
2009-03-17 20:19:49 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* TLB handling. This allows us to remove pages from the page
|
|
|
|
* tables, and efficiently handle the TLB issues.
|
|
|
|
*/
|
|
|
|
struct mmu_gather {
|
|
|
|
struct mm_struct *mm;
|
|
|
|
unsigned int fullmm;
|
|
|
|
unsigned long start, end;
|
|
|
|
};
|
2007-11-10 19:35:53 +08:00
|
|
|
|
2009-03-17 20:19:49 +08:00
|
|
|
static inline void init_tlb_gather(struct mmu_gather *tlb)
|
|
|
|
{
|
|
|
|
tlb->start = TASK_SIZE;
|
|
|
|
tlb->end = 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2009-03-17 20:19:49 +08:00
|
|
|
if (tlb->fullmm) {
|
|
|
|
tlb->start = 0;
|
|
|
|
tlb->end = TASK_SIZE;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-05-25 08:11:54 +08:00
|
|
|
static inline void
|
Fix TLB gather virtual address range invalidation corner cases
Ben Tebulin reported:
"Since v3.7.2 on two independent machines a very specific Git
repository fails in 9/10 cases on git-fsck due to an SHA1/memory
failures. This only occurs on a very specific repository and can be
reproduced stably on two independent laptops. Git mailing list ran
out of ideas and for me this looks like some very exotic kernel issue"
and bisected the failure to the backport of commit 53a59fc67f97 ("mm:
limit mmu_gather batching to fix soft lockups on !CONFIG_PREEMPT").
That commit itself is not actually buggy, but what it does is to make it
much more likely to hit the partial TLB invalidation case, since it
introduces a new case in tlb_next_batch() that previously only ever
happened when running out of memory.
The real bug is that the TLB gather virtual memory range setup is subtly
buggered. It was introduced in commit 597e1c3580b7 ("mm/mmu_gather:
enable tlb flush range in generic mmu_gather"), and the range handling
was already fixed at least once in commit e6c495a96ce0 ("mm: fix the TLB
range flushed when __tlb_remove_page() runs out of slots"), but that fix
was not complete.
The problem with the TLB gather virtual address range is that it isn't
set up by the initial tlb_gather_mmu() initialization (which didn't get
the TLB range information), but it is set up ad-hoc later by the
functions that actually flush the TLB. And so any such case that forgot
to update the TLB range entries would potentially miss TLB invalidates.
Rather than try to figure out exactly which particular ad-hoc range
setup was missing (I personally suspect it's the hugetlb case in
zap_huge_pmd(), which didn't have the same logic as zap_pte_range()
did), this patch just gets rid of the problem at the source: make the
TLB range information available to tlb_gather_mmu(), and initialize it
when initializing all the other tlb gather fields.
This makes the patch larger, but conceptually much simpler. And the end
result is much more understandable; even if you want to play games with
partial ranges when invalidating the TLB contents in chunks, now the
range information is always there, and anybody who doesn't want to
bother with it won't introduce subtle bugs.
Ben verified that this fixes his problem.
Reported-bisected-and-tested-by: Ben Tebulin <tebulin@googlemail.com>
Build-testing-by: Stephen Rothwell <sfr@canb.auug.org.au>
Build-testing-by: Richard Weinberger <richard.weinberger@gmail.com>
Reviewed-by: Michal Hocko <mhocko@suse.cz>
Acked-by: Peter Zijlstra <peterz@infradead.org>
Cc: stable@vger.kernel.org
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-08-16 02:42:25 +08:00
|
|
|
tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end)
|
2009-03-17 20:19:49 +08:00
|
|
|
{
|
|
|
|
tlb->mm = mm;
|
Fix TLB gather virtual address range invalidation corner cases
Ben Tebulin reported:
"Since v3.7.2 on two independent machines a very specific Git
repository fails in 9/10 cases on git-fsck due to an SHA1/memory
failures. This only occurs on a very specific repository and can be
reproduced stably on two independent laptops. Git mailing list ran
out of ideas and for me this looks like some very exotic kernel issue"
and bisected the failure to the backport of commit 53a59fc67f97 ("mm:
limit mmu_gather batching to fix soft lockups on !CONFIG_PREEMPT").
That commit itself is not actually buggy, but what it does is to make it
much more likely to hit the partial TLB invalidation case, since it
introduces a new case in tlb_next_batch() that previously only ever
happened when running out of memory.
The real bug is that the TLB gather virtual memory range setup is subtly
buggered. It was introduced in commit 597e1c3580b7 ("mm/mmu_gather:
enable tlb flush range in generic mmu_gather"), and the range handling
was already fixed at least once in commit e6c495a96ce0 ("mm: fix the TLB
range flushed when __tlb_remove_page() runs out of slots"), but that fix
was not complete.
The problem with the TLB gather virtual address range is that it isn't
set up by the initial tlb_gather_mmu() initialization (which didn't get
the TLB range information), but it is set up ad-hoc later by the
functions that actually flush the TLB. And so any such case that forgot
to update the TLB range entries would potentially miss TLB invalidates.
Rather than try to figure out exactly which particular ad-hoc range
setup was missing (I personally suspect it's the hugetlb case in
zap_huge_pmd(), which didn't have the same logic as zap_pte_range()
did), this patch just gets rid of the problem at the source: make the
TLB range information available to tlb_gather_mmu(), and initialize it
when initializing all the other tlb gather fields.
This makes the patch larger, but conceptually much simpler. And the end
result is much more understandable; even if you want to play games with
partial ranges when invalidating the TLB contents in chunks, now the
range information is always there, and anybody who doesn't want to
bother with it won't introduce subtle bugs.
Ben verified that this fixes his problem.
Reported-bisected-and-tested-by: Ben Tebulin <tebulin@googlemail.com>
Build-testing-by: Stephen Rothwell <sfr@canb.auug.org.au>
Build-testing-by: Richard Weinberger <richard.weinberger@gmail.com>
Reviewed-by: Michal Hocko <mhocko@suse.cz>
Acked-by: Peter Zijlstra <peterz@infradead.org>
Cc: stable@vger.kernel.org
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-08-16 02:42:25 +08:00
|
|
|
tlb->start = start;
|
|
|
|
tlb->end = end;
|
|
|
|
tlb->fullmm = !(start | (end+1));
|
2009-03-17 20:19:49 +08:00
|
|
|
|
|
|
|
init_tlb_gather(tlb);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
|
|
|
|
{
|
|
|
|
if (tlb->fullmm)
|
|
|
|
flush_tlb_mm(tlb->mm);
|
|
|
|
|
|
|
|
/* keep the page table cache within bounds */
|
|
|
|
check_pgt_cache();
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long address)
|
|
|
|
{
|
|
|
|
if (tlb->start > address)
|
|
|
|
tlb->start = address;
|
|
|
|
if (tlb->end < address + PAGE_SIZE)
|
|
|
|
tlb->end = address + PAGE_SIZE;
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2016-12-13 08:42:37 +08:00
|
|
|
#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \
|
|
|
|
tlb_remove_tlb_entry(tlb, ptep, address)
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
2009-03-17 20:19:49 +08:00
|
|
|
* In the case of tlb vma handling, we can optimise these away in the
|
|
|
|
* case where we're doing a full MM flush. When we're doing a munmap,
|
|
|
|
* the vmas are adjusted to only cover the region to be torn down.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2009-03-17 20:19:49 +08:00
|
|
|
static inline void
|
|
|
|
tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
|
|
|
|
{
|
|
|
|
if (!tlb->fullmm)
|
|
|
|
flush_cache_range(vma, vma->vm_start, vma->vm_end);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
|
|
|
|
{
|
|
|
|
if (!tlb->fullmm && tlb->end) {
|
|
|
|
flush_tlb_range(vma, tlb->start, tlb->end);
|
|
|
|
init_tlb_gather(tlb);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-04-26 07:05:40 +08:00
|
|
|
static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2011-05-25 08:11:54 +08:00
|
|
|
static inline void tlb_flush_mmu(struct mmu_gather *tlb)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
|
|
|
|
{
|
|
|
|
free_page_and_swap_cache(page);
|
2016-07-27 06:24:09 +08:00
|
|
|
return false; /* avoid calling tlb_flush_mmu */
|
2011-05-25 08:11:54 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
|
|
|
|
{
|
|
|
|
__tlb_remove_page(tlb, page);
|
|
|
|
}
|
|
|
|
|
2016-07-27 06:24:12 +08:00
|
|
|
static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
|
|
|
|
struct page *page, int page_size)
|
|
|
|
{
|
|
|
|
return __tlb_remove_page(tlb, page);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void tlb_remove_page_size(struct mmu_gather *tlb,
|
|
|
|
struct page *page, int page_size)
|
|
|
|
{
|
|
|
|
return tlb_remove_page(tlb, page);
|
|
|
|
}
|
|
|
|
|
2016-12-13 08:42:40 +08:00
|
|
|
#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
|
|
|
|
static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
|
|
|
|
unsigned int page_size)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
mm: Pass virtual address to [__]p{te,ud,md}_free_tlb()
Upcoming paches to support the new 64-bit "BookE" powerpc architecture
will need to have the virtual address corresponding to PTE page when
freeing it, due to the way the HW table walker works.
Basically, the TLB can be loaded with "large" pages that cover the whole
virtual space (well, sort-of, half of it actually) represented by a PTE
page, and which contain an "indirect" bit indicating that this TLB entry
RPN points to an array of PTEs from which the TLB can then create direct
entries. Thus, in order to invalidate those when PTE pages are deleted,
we need the virtual address to pass to tlbilx or tlbivax instructions.
The old trick of sticking it somewhere in the PTE page struct page sucks
too much, the address is almost readily available in all call sites and
almost everybody implemets these as macros, so we may as well add the
argument everywhere. I added it to the pmd and pud variants for consistency.
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Acked-by: David Howells <dhowells@redhat.com> [MN10300 & FRV]
Acked-by: Nick Piggin <npiggin@suse.de>
Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com> [s390]
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-07-22 13:44:28 +08:00
|
|
|
#define pte_free_tlb(tlb, ptep, addr) pte_free((tlb)->mm, ptep)
|
|
|
|
#define pmd_free_tlb(tlb, pmdp, addr) pmd_free((tlb)->mm, pmdp)
|
|
|
|
#define pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp)
|
2009-03-17 20:19:49 +08:00
|
|
|
|
|
|
|
#define tlb_migrate_finish(mm) do { } while (0)
|
|
|
|
|
2010-01-19 14:20:35 +08:00
|
|
|
#if defined(CONFIG_CPU_SH4) || defined(CONFIG_SUPERH64)
|
2009-11-18 05:05:31 +08:00
|
|
|
extern void tlb_wire_entry(struct vm_area_struct *, unsigned long, pte_t);
|
|
|
|
extern void tlb_unwire_entry(void);
|
|
|
|
#else
|
|
|
|
static inline void tlb_wire_entry(struct vm_area_struct *vma ,
|
|
|
|
unsigned long addr, pte_t pte)
|
|
|
|
{
|
|
|
|
BUG();
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void tlb_unwire_entry(void)
|
|
|
|
{
|
|
|
|
BUG();
|
|
|
|
}
|
2010-01-19 14:20:35 +08:00
|
|
|
#endif
|
2009-11-18 05:05:31 +08:00
|
|
|
|
2009-03-17 20:19:49 +08:00
|
|
|
#else /* CONFIG_MMU */
|
|
|
|
|
|
|
|
#define tlb_start_vma(tlb, vma) do { } while (0)
|
|
|
|
#define tlb_end_vma(tlb, vma) do { } while (0)
|
|
|
|
#define __tlb_remove_tlb_entry(tlb, pte, address) do { } while (0)
|
|
|
|
#define tlb_flush(tlb) do { } while (0)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
#include <asm-generic/tlb.h>
|
2007-11-10 19:35:53 +08:00
|
|
|
|
2009-03-17 20:19:49 +08:00
|
|
|
#endif /* CONFIG_MMU */
|
2007-11-10 19:35:53 +08:00
|
|
|
#endif /* __ASSEMBLY__ */
|
|
|
|
#endif /* __ASM_SH_TLB_H */
|