void hash_preload(struct mm_struct *mm, unsigned long ea,
unsigned long access, unsigned long trap)
{
unsigned long vsid;
void *pgdir;
pte_t *ptep;
cpumask_t mask;
unsigned long flags;
int local = 0;
/* We don't want huge pages prefaulted for now
*/
if (unlikely(in_hugepage_area(mm->context, ea)))
return;
DBG_LOW("hash_preload(mm=%p, mm->pgdir=%p, ea=%016lx, access=%lx,"
" trap=%lx\n", mm, mm->pgd, ea, access, trap);
/* Get PTE, VSID, access mask */
pgdir = mm->pgd;
if (pgdir == NULL)
return;
ptep = find_linux_pte(pgdir, ea);
if (!ptep)
return;
vsid = get_vsid(mm->context.id, ea);
/* Hash it in */
local_irq_save(flags);
mask = cpumask_of_cpu(smp_processor_id());
if (cpus_equal(mm->cpu_vm_mask, mask))
local = 1;
#ifndef CONFIG_PPC_64K_PAGES
__hash_page_4K(ea, access, vsid, ptep, trap, local);
#else
if (mmu_ci_restrictions) {
/* If this PTE is non-cacheable, switch to 4k */
if (mm->context.user_psize == MMU_PAGE_64K &&
(pte_val(*ptep) & _PAGE_NO_CACHE)) {
mm->context.user_psize = MMU_PAGE_4K;
mm->context.sllp = SLB_VSID_USER |
mmu_psize_defs[MMU_PAGE_4K].sllp;
get_paca()->context = mm->context;
slb_flush_and_rebolt();
}
}
if (mm->context.user_psize == MMU_PAGE_64K)
__hash_page_64K(ea, access, vsid, ptep, trap, local);
else
__hash_page_4K(ea, access, vsid, ptep, trap, local);
#endif /* CONFIG_PPC_64K_PAGES */
local_irq_restore(flags);
}
static hugepte_t *hugepte_alloc(struct mm_struct *mm, unsigned long addr)
{
pgd_t *pgd;
pmd_t *pmd = NULL;
BUG_ON(!in_hugepage_area(mm->context, addr));
pgd = pgd_offset(mm, addr);
pmd = pmd_alloc(mm, pgd, addr);
/* We shouldn't find a (normal) PTE page pointer here */
BUG_ON(!pmd_none(*pmd) && !pmd_hugepage(*pmd));
return (hugepte_t *)pmd;
}
static int __spu_trap_data_seg(struct spu *spu, unsigned long ea)
{
struct spu_priv2 __iomem *priv2 = spu->priv2;
struct mm_struct *mm = spu->mm;
u64 esid, vsid;
pr_debug("%s\n", __FUNCTION__);
if (test_bit(SPU_CONTEXT_SWITCH_ACTIVE, &spu->flags)) {
/* SLBs are pre-loaded for context switch, so
* we should never get here!
*/
printk("%s: invalid access during switch!\n", __func__);
return 1;
}
if (!mm || (REGION_ID(ea) != USER_REGION_ID)) {
/* Future: support kernel segments so that drivers
* can use SPUs.
*/
pr_debug("invalid region access at %016lx\n", ea);
return 1;
}
esid = (ea & ESID_MASK) | SLB_ESID_V;
vsid = (get_vsid(mm->context.id, ea) << SLB_VSID_SHIFT) | SLB_VSID_USER;
if (in_hugepage_area(mm->context, ea))
vsid |= SLB_VSID_L;
out_be64(&priv2->slb_index_W, spu->slb_replace);
out_be64(&priv2->slb_vsid_RW, vsid);
out_be64(&priv2->slb_esid_RW, esid);
spu->slb_replace++;
if (spu->slb_replace >= 8)
spu->slb_replace = 0;
spu_restart_dma(spu);
return 0;
}
/* Result code is:
* 0 - handled
* 1 - normal page fault
* -1 - critical hash insertion error
*/
int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
{
void *pgdir;
unsigned long vsid;
struct mm_struct *mm;
pte_t *ptep;
cpumask_t tmp;
int rc, user_region = 0, local = 0;
int psize;
DBG_LOW("hash_page(ea=%016lx, access=%lx, trap=%lx\n",
ea, access, trap);
if ((ea & ~REGION_MASK) >= PGTABLE_RANGE) {
DBG_LOW(" out of pgtable range !\n");
return 1;
}
/* Get region & vsid */
switch (REGION_ID(ea)) {
case USER_REGION_ID:
user_region = 1;
mm = current->mm;
if (! mm) {
DBG_LOW(" user region with no mm !\n");
return 1;
}
vsid = get_vsid(mm->context.id, ea);
psize = mm->context.user_psize;
break;
case VMALLOC_REGION_ID:
mm = &init_mm;
vsid = get_kernel_vsid(ea);
if (ea < VMALLOC_END)
psize = mmu_vmalloc_psize;
else
psize = mmu_io_psize;
break;
default:
/* Not a valid range
* Send the problem up to do_page_fault
*/
return 1;
}
DBG_LOW(" mm=%p, mm->pgdir=%p, vsid=%016lx\n", mm, mm->pgd, vsid);
/* Get pgdir */
pgdir = mm->pgd;
if (pgdir == NULL)
return 1;
/* Check CPU locality */
tmp = cpumask_of_cpu(smp_processor_id());
if (user_region && cpus_equal(mm->cpu_vm_mask, tmp))
local = 1;
/* Handle hugepage regions */
if (unlikely(in_hugepage_area(mm->context, ea))) {
DBG_LOW(" -> huge page !\n");
return hash_huge_page(mm, access, ea, vsid, local, trap);
}
/* Get PTE and page size from page tables */
ptep = find_linux_pte(pgdir, ea);
if (ptep == NULL || !pte_present(*ptep)) {
DBG_LOW(" no PTE !\n");
return 1;
}
#ifndef CONFIG_PPC_64K_PAGES
DBG_LOW(" i-pte: %016lx\n", pte_val(*ptep));
#else
DBG_LOW(" i-pte: %016lx %016lx\n", pte_val(*ptep),
pte_val(*(ptep + PTRS_PER_PTE)));
#endif
/* Pre-check access permissions (will be re-checked atomically
* in __hash_page_XX but this pre-check is a fast path
*/
if (access & ~pte_val(*ptep)) {
DBG_LOW(" no access !\n");
return 1;
}
/* Do actual hashing */
#ifndef CONFIG_PPC_64K_PAGES
rc = __hash_page_4K(ea, access, vsid, ptep, trap, local);
#else
if (mmu_ci_restrictions) {
/* If this PTE is non-cacheable, switch to 4k */
if (psize == MMU_PAGE_64K &&
(pte_val(*ptep) & _PAGE_NO_CACHE)) {
if (user_region) {
psize = MMU_PAGE_4K;
mm->context.user_psize = MMU_PAGE_4K;
mm->context.sllp = SLB_VSID_USER |
mmu_psize_defs[MMU_PAGE_4K].sllp;
} else if (ea < VMALLOC_END) {
/*
* some driver did a non-cacheable mapping
* in vmalloc space, so switch vmalloc
* to 4k pages
*/
printk(KERN_ALERT "Reducing vmalloc segment "
"to 4kB pages because of "
"non-cacheable mapping\n");
psize = mmu_vmalloc_psize = MMU_PAGE_4K;
}
}
if (user_region) {
if (psize != get_paca()->context.user_psize) {
get_paca()->context = mm->context;
slb_flush_and_rebolt();
}
} else if (get_paca()->vmalloc_sllp !=
mmu_psize_defs[mmu_vmalloc_psize].sllp) {
get_paca()->vmalloc_sllp =
mmu_psize_defs[mmu_vmalloc_psize].sllp;
slb_flush_and_rebolt();
}
}
if (psize == MMU_PAGE_64K)
rc = __hash_page_64K(ea, access, vsid, ptep, trap, local);
else
rc = __hash_page_4K(ea, access, vsid, ptep, trap, local);
#endif /* CONFIG_PPC_64K_PAGES */
#ifndef CONFIG_PPC_64K_PAGES
DBG_LOW(" o-pte: %016lx\n", pte_val(*ptep));
#else
DBG_LOW(" o-pte: %016lx %016lx\n", pte_val(*ptep),
pte_val(*(ptep + PTRS_PER_PTE)));
#endif
DBG_LOW(" -> rc=%d\n", rc);
return rc;
}
