2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
|
* Low-level SLB routines
|
|
|
|
|
*
|
|
|
|
|
* Copyright (C) 2004 David Gibson <dwg@au.ibm.com>, IBM
|
|
|
|
|
*
|
|
|
|
|
* Based on earlier C version:
|
|
|
|
|
* Dave Engebretsen and Mike Corrigan {engebret|mikejc}@us.ibm.com
|
|
|
|
|
* Copyright (c) 2001 Dave Engebretsen
|
|
|
|
|
* Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM
|
|
|
|
|
*
|
|
|
|
|
* This program is free software; you can redistribute it and/or
|
|
|
|
|
* modify it under the terms of the GNU General Public License
|
|
|
|
|
* as published by the Free Software Foundation; either version
|
|
|
|
|
* 2 of the License, or (at your option) any later version.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#include <linux/config.h>
|
|
|
|
|
#include <asm/processor.h>
|
|
|
|
|
#include <asm/ppc_asm.h>
|
2005-09-10 02:57:26 +08:00
|
|
|
#include <asm/asm-offsets.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <asm/cputable.h>
|
2005-11-07 08:06:55 +08:00
|
|
|
#include <asm/page.h>
|
|
|
|
|
#include <asm/mmu.h>
|
|
|
|
|
#include <asm/pgtable.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
/* void slb_allocate_realmode(unsigned long ea);
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
|
|
|
|
* Create an SLB entry for the given EA (user or kernel).
|
|
|
|
|
* r3 = faulting address, r13 = PACA
|
|
|
|
|
* r9, r10, r11 are clobbered by this function
|
|
|
|
|
* No other registers are examined or changed.
|
|
|
|
|
*/
|
2005-11-07 08:06:55 +08:00
|
|
|
_GLOBAL(slb_allocate_realmode)
|
|
|
|
|
/* r3 = faulting address */
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
srdi r9,r3,60 /* get region */
|
2005-11-07 08:06:55 +08:00
|
|
|
srdi r10,r3,28 /* get esid */
|
2005-12-06 00:24:33 +08:00
|
|
|
cmpldi cr7,r9,0xc /* cmp PAGE_OFFSET for later use */
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-12-06 00:24:33 +08:00
|
|
|
/* r3 = address, r10 = esid, cr7 = <> PAGE_OFFSET */
|
2005-04-17 06:20:36 +08:00
|
|
|
blt cr7,0f /* user or kernel? */
|
|
|
|
|
|
|
|
|
|
/* kernel address: proto-VSID = ESID */
|
|
|
|
|
/* WARNING - MAGIC: we don't use the VSID 0xfffffffff, but
|
|
|
|
|
* this code will generate the protoVSID 0xfffffffff for the
|
|
|
|
|
* top segment. That's ok, the scramble below will translate
|
|
|
|
|
* it to VSID 0, which is reserved as a bad VSID - one which
|
|
|
|
|
* will never have any pages in it. */
|
|
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
/* Check if hitting the linear mapping of the vmalloc/ioremap
|
|
|
|
|
* kernel space
|
|
|
|
|
*/
|
|
|
|
|
bne cr7,1f
|
|
|
|
|
|
|
|
|
|
/* Linear mapping encoding bits, the "li" instruction below will
|
|
|
|
|
* be patched by the kernel at boot
|
|
|
|
|
*/
|
|
|
|
|
_GLOBAL(slb_miss_kernel_load_linear)
|
|
|
|
|
li r11,0
|
|
|
|
|
b slb_finish_load
|
|
|
|
|
|
|
|
|
|
1: /* vmalloc/ioremap mapping encoding bits, the "li" instruction below
|
|
|
|
|
* will be patched by the kernel at boot
|
|
|
|
|
*/
|
|
|
|
|
_GLOBAL(slb_miss_kernel_load_virtual)
|
|
|
|
|
li r11,0
|
|
|
|
|
b slb_finish_load
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
0: /* user address: proto-VSID = context << 15 | ESID. First check
|
|
|
|
|
* if the address is within the boundaries of the user region
|
|
|
|
|
*/
|
|
|
|
|
srdi. r9,r10,USER_ESID_BITS
|
2005-04-17 06:20:36 +08:00
|
|
|
bne- 8f /* invalid ea bits set */
|
|
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
/* Figure out if the segment contains huge pages */
|
2005-04-17 06:20:36 +08:00
|
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
|
|
|
|
BEGIN_FTR_SECTION
|
2005-11-07 08:06:55 +08:00
|
|
|
b 1f
|
|
|
|
|
END_FTR_SECTION_IFCLR(CPU_FTR_16M_PAGE)
|
[PATCH] ppc64: Fix bug in SLB miss handler for hugepages
This patch, however, should be applied on top of the 64k-page-size patch to
fix some problems with hugepage (some pre-existing, another introduced by
this patch).
The patch fixes a bug in the SLB miss handler for hugepages on ppc64
introduced by the dynamic hugepage patch (commit id
c594adad5653491813959277fb87a2fef54c4e05) due to a misunderstanding of the
srd instruction's behaviour (mea culpa). The problem arises when a 64-bit
process maps some hugepages in the low 4GB of the address space (unusual).
In this case, as well as the 256M segment in question being marked for
hugepages, other segments at 32G intervals will be incorrectly marked for
hugepages.
In the process, this patch tweaks the semantics of the hugepage bitmaps to
be more sensible. Previously, an address below 4G was marked for hugepages
if the appropriate segment bit in the "low areas" bitmask was set *or* if
the low bit in the "high areas" bitmap was set (which would mark all
addresses below 1TB for hugepage). With this patch, any given address is
governed by a single bitmap. Addresses below 4GB are marked for hugepage
if and only if their bit is set in the "low areas" bitmap (256M
granularity). Addresses between 4GB and 1TB are marked for hugepage iff
the low bit in the "high areas" bitmap is set. Higher addresses are marked
for hugepage iff their bit in the "high areas" bitmap is set (1TB
granularity).
To avoid conflicts, this patch must be applied on top of BenH's pending
patch for 64k base page size [0]. As such, this patch also addresses a
hugepage problem introduced by that patch. That patch allows hugepages of
1MB in size on hardware which supports it, however, that won't work when
using 4k pages (4 level pagetable), because in that case hugepage PTEs are
stored at the PMD level, and each PMD entry maps 2MB. This patch simply
disallows hugepages in that case (we can do something cleverer to re-enable
them some other day).
Built, booted, and a handful of hugepage related tests passed on POWER5
LPAR (both ARCH=powerpc and ARCH=ppc64).
[0] http://gate.crashing.org/~benh/ppc64-64k-pages.diff
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-11-07 16:57:52 +08:00
|
|
|
cmpldi r10,16
|
|
|
|
|
|
|
|
|
|
lhz r9,PACALOWHTLBAREAS(r13)
|
|
|
|
|
mr r11,r10
|
|
|
|
|
blt 5f
|
|
|
|
|
|
2005-08-11 14:55:21 +08:00
|
|
|
lhz r9,PACAHIGHHTLBAREAS(r13)
|
2005-11-07 08:06:55 +08:00
|
|
|
srdi r11,r10,(HTLB_AREA_SHIFT-SID_SHIFT)
|
[PATCH] ppc64: Fix bug in SLB miss handler for hugepages
This patch, however, should be applied on top of the 64k-page-size patch to
fix some problems with hugepage (some pre-existing, another introduced by
this patch).
The patch fixes a bug in the SLB miss handler for hugepages on ppc64
introduced by the dynamic hugepage patch (commit id
c594adad5653491813959277fb87a2fef54c4e05) due to a misunderstanding of the
srd instruction's behaviour (mea culpa). The problem arises when a 64-bit
process maps some hugepages in the low 4GB of the address space (unusual).
In this case, as well as the 256M segment in question being marked for
hugepages, other segments at 32G intervals will be incorrectly marked for
hugepages.
In the process, this patch tweaks the semantics of the hugepage bitmaps to
be more sensible. Previously, an address below 4G was marked for hugepages
if the appropriate segment bit in the "low areas" bitmask was set *or* if
the low bit in the "high areas" bitmap was set (which would mark all
addresses below 1TB for hugepage). With this patch, any given address is
governed by a single bitmap. Addresses below 4GB are marked for hugepage
if and only if their bit is set in the "low areas" bitmap (256M
granularity). Addresses between 4GB and 1TB are marked for hugepage iff
the low bit in the "high areas" bitmap is set. Higher addresses are marked
for hugepage iff their bit in the "high areas" bitmap is set (1TB
granularity).
To avoid conflicts, this patch must be applied on top of BenH's pending
patch for 64k base page size [0]. As such, this patch also addresses a
hugepage problem introduced by that patch. That patch allows hugepages of
1MB in size on hardware which supports it, however, that won't work when
using 4k pages (4 level pagetable), because in that case hugepage PTEs are
stored at the PMD level, and each PMD entry maps 2MB. This patch simply
disallows hugepages in that case (we can do something cleverer to re-enable
them some other day).
Built, booted, and a handful of hugepage related tests passed on POWER5
LPAR (both ARCH=powerpc and ARCH=ppc64).
[0] http://gate.crashing.org/~benh/ppc64-64k-pages.diff
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-11-07 16:57:52 +08:00
|
|
|
|
|
|
|
|
5: srd r9,r9,r11
|
|
|
|
|
andi. r9,r9,1
|
2005-11-07 08:06:55 +08:00
|
|
|
beq 1f
|
|
|
|
|
_GLOBAL(slb_miss_user_load_huge)
|
|
|
|
|
li r11,0
|
|
|
|
|
b 2f
|
|
|
|
|
1:
|
2005-08-31 12:34:05 +08:00
|
|
|
#endif /* CONFIG_HUGETLB_PAGE */
|
2005-08-11 14:55:21 +08:00
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
_GLOBAL(slb_miss_user_load_normal)
|
|
|
|
|
li r11,0
|
2005-08-11 14:55:21 +08:00
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
2:
|
|
|
|
|
ld r9,PACACONTEXTID(r13)
|
|
|
|
|
rldimi r10,r9,USER_ESID_BITS,0
|
|
|
|
|
b slb_finish_load
|
|
|
|
|
|
|
|
|
|
8: /* invalid EA */
|
|
|
|
|
li r10,0 /* BAD_VSID */
|
|
|
|
|
li r11,SLB_VSID_USER /* flags don't much matter */
|
|
|
|
|
b slb_finish_load
|
|
|
|
|
|
|
|
|
|
#ifdef __DISABLED__
|
|
|
|
|
|
|
|
|
|
/* void slb_allocate_user(unsigned long ea);
|
|
|
|
|
*
|
|
|
|
|
* Create an SLB entry for the given EA (user or kernel).
|
|
|
|
|
* r3 = faulting address, r13 = PACA
|
|
|
|
|
* r9, r10, r11 are clobbered by this function
|
|
|
|
|
* No other registers are examined or changed.
|
|
|
|
|
*
|
|
|
|
|
* It is called with translation enabled in order to be able to walk the
|
|
|
|
|
* page tables. This is not currently used.
|
|
|
|
|
*/
|
|
|
|
|
_GLOBAL(slb_allocate_user)
|
|
|
|
|
/* r3 = faulting address */
|
|
|
|
|
srdi r10,r3,28 /* get esid */
|
|
|
|
|
|
|
|
|
|
crset 4*cr7+lt /* set "user" flag for later */
|
|
|
|
|
|
|
|
|
|
/* check if we fit in the range covered by the pagetables*/
|
|
|
|
|
srdi. r9,r3,PGTABLE_EADDR_SIZE
|
|
|
|
|
crnot 4*cr0+eq,4*cr0+eq
|
|
|
|
|
beqlr
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
/* now we need to get to the page tables in order to get the page
|
|
|
|
|
* size encoding from the PMD. In the future, we'll be able to deal
|
|
|
|
|
* with 1T segments too by getting the encoding from the PGD instead
|
|
|
|
|
*/
|
|
|
|
|
ld r9,PACAPGDIR(r13)
|
|
|
|
|
cmpldi cr0,r9,0
|
|
|
|
|
beqlr
|
|
|
|
|
rlwinm r11,r10,8,25,28
|
|
|
|
|
ldx r9,r9,r11 /* get pgd_t */
|
|
|
|
|
cmpldi cr0,r9,0
|
|
|
|
|
beqlr
|
|
|
|
|
rlwinm r11,r10,3,17,28
|
|
|
|
|
ldx r9,r9,r11 /* get pmd_t */
|
|
|
|
|
cmpldi cr0,r9,0
|
|
|
|
|
beqlr
|
|
|
|
|
|
|
|
|
|
/* build vsid flags */
|
|
|
|
|
andi. r11,r9,SLB_VSID_LLP
|
|
|
|
|
ori r11,r11,SLB_VSID_USER
|
|
|
|
|
|
|
|
|
|
/* get context to calculate proto-VSID */
|
2005-08-31 12:34:05 +08:00
|
|
|
ld r9,PACACONTEXTID(r13)
|
2005-11-07 08:06:55 +08:00
|
|
|
rldimi r10,r9,USER_ESID_BITS,0
|
|
|
|
|
|
|
|
|
|
/* fall through slb_finish_load */
|
|
|
|
|
|
|
|
|
|
#endif /* __DISABLED__ */
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
/*
|
|
|
|
|
* Finish loading of an SLB entry and return
|
|
|
|
|
*
|
2005-12-06 00:24:33 +08:00
|
|
|
* r3 = EA, r10 = proto-VSID, r11 = flags, clobbers r9, cr7 = <> PAGE_OFFSET
|
2005-11-07 08:06:55 +08:00
|
|
|
*/
|
|
|
|
|
slb_finish_load:
|
|
|
|
|
ASM_VSID_SCRAMBLE(r10,r9)
|
|
|
|
|
rldimi r11,r10,SLB_VSID_SHIFT,16 /* combine VSID and flags */
|
|
|
|
|
|
|
|
|
|
/* r3 = EA, r11 = VSID data */
|
|
|
|
|
/*
|
|
|
|
|
* Find a slot, round robin. Previously we tried to find a
|
|
|
|
|
* free slot first but that took too long. Unfortunately we
|
|
|
|
|
* dont have any LRU information to help us choose a slot.
|
|
|
|
|
*/
|
|
|
|
|
#ifdef CONFIG_PPC_ISERIES
|
|
|
|
|
/*
|
|
|
|
|
* On iSeries, the "bolted" stack segment can be cast out on
|
|
|
|
|
* shared processor switch so we need to check for a miss on
|
|
|
|
|
* it and restore it to the right slot.
|
|
|
|
|
*/
|
|
|
|
|
ld r9,PACAKSAVE(r13)
|
|
|
|
|
clrrdi r9,r9,28
|
|
|
|
|
clrrdi r3,r3,28
|
|
|
|
|
li r10,SLB_NUM_BOLTED-1 /* Stack goes in last bolted slot */
|
|
|
|
|
cmpld r9,r3
|
|
|
|
|
beq 3f
|
|
|
|
|
#endif /* CONFIG_PPC_ISERIES */
|
|
|
|
|
|
|
|
|
|
ld r10,PACASTABRR(r13)
|
|
|
|
|
addi r10,r10,1
|
|
|
|
|
/* use a cpu feature mask if we ever change our slb size */
|
|
|
|
|
cmpldi r10,SLB_NUM_ENTRIES
|
|
|
|
|
|
|
|
|
|
blt+ 4f
|
|
|
|
|
li r10,SLB_NUM_BOLTED
|
|
|
|
|
|
|
|
|
|
4:
|
|
|
|
|
std r10,PACASTABRR(r13)
|
|
|
|
|
|
|
|
|
|
3:
|
|
|
|
|
rldimi r3,r10,0,36 /* r3= EA[0:35] | entry */
|
|
|
|
|
oris r10,r3,SLB_ESID_V@h /* r3 |= SLB_ESID_V */
|
|
|
|
|
|
|
|
|
|
/* r3 = ESID data, r11 = VSID data */
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* No need for an isync before or after this slbmte. The exception
|
|
|
|
|
* we enter with and the rfid we exit with are context synchronizing.
|
|
|
|
|
*/
|
|
|
|
|
slbmte r11,r10
|
|
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
/* we're done for kernel addresses */
|
|
|
|
|
crclr 4*cr0+eq /* set result to "success" */
|
|
|
|
|
bgelr cr7
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
/* Update the slb cache */
|
|
|
|
|
lhz r3,PACASLBCACHEPTR(r13) /* offset = paca->slb_cache_ptr */
|
|
|
|
|
cmpldi r3,SLB_CACHE_ENTRIES
|
|
|
|
|
bge 1f
|
|
|
|
|
|
|
|
|
|
/* still room in the slb cache */
|
|
|
|
|
sldi r11,r3,1 /* r11 = offset * sizeof(u16) */
|
|
|
|
|
rldicl r10,r10,36,28 /* get low 16 bits of the ESID */
|
|
|
|
|
add r11,r11,r13 /* r11 = (u16 *)paca + offset */
|
|
|
|
|
sth r10,PACASLBCACHE(r11) /* paca->slb_cache[offset] = esid */
|
|
|
|
|
addi r3,r3,1 /* offset++ */
|
|
|
|
|
b 2f
|
|
|
|
|
1: /* offset >= SLB_CACHE_ENTRIES */
|
|
|
|
|
li r3,SLB_CACHE_ENTRIES+1
|
|
|
|
|
2:
|
|
|
|
|
sth r3,PACASLBCACHEPTR(r13) /* paca->slb_cache_ptr = offset */
|
2005-11-07 08:06:55 +08:00
|
|
|
crclr 4*cr0+eq /* set result to "success" */
|
2005-04-17 06:20:36 +08:00
|
|
|
blr
|
|
|
|
|
|