/* * Handle cache congruency of kernel and userspace mappings of page when kernel * writes-to/reads-from * * The idea is to defer flushing of kernel mapping after a WRITE, possible if: * -dcache is NOT aliasing, hence any U/K-mappings of page are congruent * -U-mapping doesn't exist yet for page (finalised in update_mmu_cache) * -In SMP, if hardware caches are coherent * * There's a corollary case, where kernel READs from a userspace mapped page. * If the U-mapping is not congruent to to K-mapping, former needs flushing. */ void flush_dcache_page(struct page *page) { struct address_space *mapping; if (!cache_is_vipt_aliasing()) { clear_bit(PG_dc_clean, &page->flags); return; } /* don't handle anon pages here */ mapping = page_mapping(page); if (!mapping) return; /* * pagecache page, file not yet mapped to userspace * Make a note that K-mapping is dirty */ if (!mapping_mapped(mapping)) { clear_bit(PG_dc_clean, &page->flags); } else if (page_mapped(page)) { /* kernel reading from page with U-mapping */ void *paddr = page_address(page); unsigned long vaddr = page->index << PAGE_CACHE_SHIFT; if (addr_not_cache_congruent(paddr, vaddr)) __flush_dcache_page(paddr, vaddr); } }
/* * 1. Validate the Cache Geomtery (compile time config matches hardware) * 2. If I-cache suffers from aliasing, setup work arounds (difft flush rtn) * (aliasing D-cache configurations are not supported YET) * 3. Enable the Caches, setup default flush mode for D-Cache * 3. Calculate the SHMLBA used by user space */ void arc_cache_init(void) { unsigned int __maybe_unused cpu = smp_processor_id(); struct cpuinfo_arc_cache __maybe_unused *ic, __maybe_unused *dc; char str[256]; printk(arc_cache_mumbojumbo(0, str, sizeof(str))); #ifdef CONFIG_ARC_HAS_ICACHE ic = &cpuinfo_arc700[cpu].icache; if (ic->ver) { if (ic->line_len != L1_CACHE_BYTES) panic("ICache line [%d] != kernel Config [%d]", ic->line_len, L1_CACHE_BYTES); if (ic->ver != CONFIG_ARC_MMU_VER) panic("Cache ver [%d] doesn't match MMU ver [%d]\n", ic->ver, CONFIG_ARC_MMU_VER); } #endif #ifdef CONFIG_ARC_HAS_DCACHE dc = &cpuinfo_arc700[cpu].dcache; if (dc->ver) { unsigned int dcache_does_alias; if (dc->line_len != L1_CACHE_BYTES) panic("DCache line [%d] != kernel Config [%d]", dc->line_len, L1_CACHE_BYTES); /* check for D-Cache aliasing */ dcache_does_alias = (dc->sz / dc->assoc) > PAGE_SIZE; if (dcache_does_alias && !cache_is_vipt_aliasing()) panic("Enable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); else if (!dcache_does_alias && cache_is_vipt_aliasing()) panic("Don't need CONFIG_ARC_CACHE_VIPT_ALIASING\n"); } #endif }
/* * We need to ensure that shared mappings are correctly aligned to * avoid aliasing issues with VIPT caches. We need to ensure that * a specific page of an object is always mapped at a multiple of * SHMLBA bytes. * * We unconditionally provide this function for all cases, however * in the VIVT case, we optimise out the alignment rules. */ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma; int do_align = 0; int aliasing = cache_is_vipt_aliasing(); struct vm_unmapped_area_info info; /* * We only need to do colour alignment if either the I or D * caches alias. */ if (aliasing) do_align = filp || (flags & MAP_SHARED); /* * We enforce the MAP_FIXED case. */ if (flags & MAP_FIXED) { if (aliasing && flags & MAP_SHARED && (addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1)) return -EINVAL; return addr; } if (len > TASK_SIZE) return -ENOMEM; if (addr) { if (do_align) addr = COLOUR_ALIGN(addr, pgoff); else addr = PAGE_ALIGN(addr); vma = find_vma(mm, addr); if (TASK_SIZE - len >= addr && (!vma || addr + len <= vma->vm_start)) return addr; } info.flags = 0; info.length = len; info.low_limit = mm->mmap_base; info.high_limit = TASK_SIZE; info.align_mask = do_align ? (PAGE_MASK & (SHMLBA - 1)) : 0; info.align_offset = pgoff << PAGE_SHIFT; return vm_unmapped_area(&info); }
static int __init v6_userpage_init(void) { if (cache_is_vipt_aliasing()) { pgd_t *pgd; pmd_t *pmd; pgd = pgd_offset_k(from_address); pmd = pmd_alloc(&init_mm, pgd, from_address); if (!pmd) BUG(); from_pte = pte_alloc_kernel(&init_mm, pmd, from_address); if (!from_pte) BUG(); to_pte = pte_alloc_kernel(&init_mm, pmd, to_address); if (!to_pte) BUG(); cpu_user.cpu_clear_user_page = v6_clear_user_page_aliasing; cpu_user.cpu_copy_user_page = v6_copy_user_page_aliasing; } return 0; }
/* * We need to ensure that shared mappings are correctly aligned to * avoid aliasing issues with VIPT caches. We need to ensure that * a specific page of an object is always mapped at a multiple of * SHMLBA bytes. * * We unconditionally provide this function for all cases, however * in the VIVT case, we optimise out the alignment rules. */ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma; unsigned long start_addr = addr; int do_align = 0; int aliasing = cache_is_vipt_aliasing(); /* * We only need to do colour alignment if either the I or D * caches alias. */ if (aliasing) do_align = filp || (flags & MAP_SHARED); #ifdef CONFIG_ARM_FCSE start_addr = addr; #endif /* CONFIG_ARM_FCSE */ /* * We enforce the MAP_FIXED case. */ if (flags & MAP_FIXED) { if (aliasing && flags & MAP_SHARED && (addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1)) return -EINVAL; goto found_addr; } if (len > TASK_SIZE) return -ENOMEM; if (addr) { if (do_align) addr = COLOUR_ALIGN(addr, pgoff); else addr = PAGE_ALIGN(addr); vma = find_vma(mm, addr); if (TASK_SIZE - len >= addr && (!vma || addr + len <= vma->vm_start)) goto found_addr; } if (len > mm->cached_hole_size) { start_addr = addr = mm->free_area_cache; } else { start_addr = addr = TASK_UNMAPPED_BASE; mm->cached_hole_size = 0; } /* 8 bits of randomness in 20 address space bits */ if ((current->flags & PF_RANDOMIZE) && !(current->personality & ADDR_NO_RANDOMIZE)) addr += (get_random_int() % (1 << 8)) << PAGE_SHIFT; full_search: if (do_align) addr = COLOUR_ALIGN(addr, pgoff); else addr = PAGE_ALIGN(addr); for (vma = find_vma(mm, addr); ; vma = vma->vm_next) { /* At this point: (!vma || addr < vma->vm_end). */ if (TASK_SIZE - len < addr) { /* * Start a new search - just in case we missed * some holes. */ if (start_addr != TASK_UNMAPPED_BASE) { start_addr = addr = TASK_UNMAPPED_BASE; mm->cached_hole_size = 0; goto full_search; } return -ENOMEM; } if (!vma || addr + len <= vma->vm_start) { /* * Remember the place where we stopped the search: */ mm->free_area_cache = addr + len; goto found_addr; } if (addr + mm->cached_hole_size < vma->vm_start) mm->cached_hole_size = vma->vm_start - addr; addr = vma->vm_end; if (do_align) addr = COLOUR_ALIGN(addr, pgoff); } found_addr: #ifdef CONFIG_ARM_FCSE { unsigned long new_addr = fcse_check_mmap_addr(mm, start_addr, addr, len, flags); if (new_addr != addr) { addr = new_addr; if (!(addr & ~PAGE_MASK)) { start_addr = TASK_UNMAPPED_BASE; mm->cached_hole_size = 0; goto full_search; } } } #endif /* CONFIG_ARM_FCSE */ return addr; }
/* * We need to ensure that shared mappings are correctly aligned to * avoid aliasing issues with VIPT caches. We need to ensure that * a specific page of an object is always mapped at a multiple of * SHMLBA bytes. * * We unconditionally provide this function for all cases, however * in the VIVT case, we optimise out the alignment rules. */ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma; unsigned long start_addr; int do_align = 0; int aliasing = cache_is_vipt_aliasing(); /* * We only need to do colour alignment if either the I or D * caches alias. */ if (aliasing) do_align = filp || (flags & MAP_SHARED); /* * We enforce the MAP_FIXED case. */ if (flags & MAP_FIXED) { if (aliasing && flags & MAP_SHARED && (addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1)) return -EINVAL; return addr; } if (len > TASK_SIZE) return -ENOMEM; if (addr) { if (do_align) addr = COLOUR_ALIGN(addr, pgoff); else addr = PAGE_ALIGN(addr); vma = find_vma(mm, addr); if (TASK_SIZE - len >= addr && (!vma || addr + len <= vma->vm_start)) return addr; } if (len > mm->cached_hole_size) { start_addr = addr = mm->free_area_cache; } else { start_addr = addr = mm->mmap_base; mm->cached_hole_size = 0; } full_search: if (do_align) addr = COLOUR_ALIGN(addr, pgoff); else addr = PAGE_ALIGN(addr); for (vma = find_vma(mm, addr); ; vma = vma->vm_next) { /* At this point: (!vma || addr < vma->vm_end). */ if (TASK_SIZE - len < addr) { /* * Start a new search - just in case we missed * some holes. */ if (start_addr != TASK_UNMAPPED_BASE) { start_addr = addr = TASK_UNMAPPED_BASE; mm->cached_hole_size = 0; goto full_search; } return -ENOMEM; } if (!vma || addr + len <= vma->vm_start) { /* * Remember the place where we stopped the search: */ mm->free_area_cache = addr + len; return addr; } if (addr + mm->cached_hole_size < vma->vm_start) mm->cached_hole_size = vma->vm_start - addr; addr = vma->vm_end; if (do_align) addr = COLOUR_ALIGN(addr, pgoff); } }
unsigned long arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, const unsigned long len, const unsigned long pgoff, const unsigned long flags) { struct vm_area_struct *vma; struct mm_struct *mm = current->mm; unsigned long addr = addr0; int do_align = 0; int aliasing = cache_is_vipt_aliasing(); /* * We only need to do colour alignment if either the I or D * caches alias. */ if (aliasing) do_align = filp || (flags & MAP_SHARED); /* requested length too big for entire address space */ if (len > TASK_SIZE) return -ENOMEM; if (flags & MAP_FIXED) { if (aliasing && flags & MAP_SHARED && (addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1)) return -EINVAL; return addr; } /* requesting a specific address */ if (addr) { if (do_align) addr = COLOUR_ALIGN(addr, pgoff); else addr = PAGE_ALIGN(addr); vma = find_vma(mm, addr); if (TASK_SIZE - len >= addr && (!vma || addr + len <= vma->vm_start)) return addr; } /* check if free_area_cache is useful for us */ if (len <= mm->cached_hole_size) { mm->cached_hole_size = 0; mm->free_area_cache = mm->mmap_base; } /* either no address requested or can't fit in requested address hole */ addr = mm->free_area_cache; if (do_align) { unsigned long base = COLOUR_ALIGN_DOWN(addr - len, pgoff); addr = base + len; } /* make sure it can fit in the remaining address space */ if (addr > len) { vma = find_vma(mm, addr-len); if (!vma || addr <= vma->vm_start) /* remember the address as a hint for next time */ return (mm->free_area_cache = addr-len); } if (mm->mmap_base < len) goto bottomup; addr = mm->mmap_base - len; if (do_align) addr = COLOUR_ALIGN_DOWN(addr, pgoff); do { /* * Lookup failure means no vma is above this address, * else if new region fits below vma->vm_start, * return with success: */ vma = find_vma(mm, addr); if (!vma || addr+len <= vma->vm_start) /* remember the address as a hint for next time */ return (mm->free_area_cache = addr); /* remember the largest hole we saw so far */ if (addr + mm->cached_hole_size < vma->vm_start) mm->cached_hole_size = vma->vm_start - addr; /* try just below the current vma->vm_start */ addr = vma->vm_start - len; if (do_align) addr = COLOUR_ALIGN_DOWN(addr, pgoff); } while (len < vma->vm_start); bottomup: /* * A failed mmap() very likely causes application failure, * so fall back to the bottom-up function here. This scenario * can happen with large stack limits and large mmap() * allocations. */ mm->cached_hole_size = ~0UL; mm->free_area_cache = TASK_UNMAPPED_BASE; addr = arch_get_unmapped_area(filp, addr0, len, pgoff, flags); /* * Restore the topdown base: */ mm->free_area_cache = mm->mmap_base; mm->cached_hole_size = ~0UL; return addr; }
unsigned long arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, const unsigned long len, const unsigned long pgoff, const unsigned long flags) { struct vm_area_struct *vma; struct mm_struct *mm = current->mm; unsigned long addr = addr0; int do_align = 0; int aliasing = cache_is_vipt_aliasing(); struct vm_unmapped_area_info info; /* * We only need to do colour alignment if either the I or D * caches alias. */ if (aliasing) do_align = filp || (flags & MAP_SHARED); /* requested length too big for entire address space */ if (len > TASK_SIZE) return -ENOMEM; if (flags & MAP_FIXED) { if (aliasing && flags & MAP_SHARED && (addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1)) return -EINVAL; return addr; } /* requesting a specific address */ if (addr) { if (do_align) addr = COLOUR_ALIGN(addr, pgoff); else addr = PAGE_ALIGN(addr); vma = find_vma(mm, addr); if (TASK_SIZE - len >= addr && (!vma || addr + len <= vma->vm_start)) return addr; } info.flags = VM_UNMAPPED_AREA_TOPDOWN; info.length = len; info.low_limit = FIRST_USER_ADDRESS; info.high_limit = mm->mmap_base; info.align_mask = do_align ? (PAGE_MASK & (SHMLBA - 1)) : 0; info.align_offset = pgoff << PAGE_SHIFT; addr = vm_unmapped_area(&info); /* * A failed mmap() very likely causes application failure, * so fall back to the bottom-up function here. This scenario * can happen with large stack limits and large mmap() * allocations. */ if (addr & ~PAGE_MASK) { VM_BUG_ON(addr != -ENOMEM); info.flags = 0; info.low_limit = mm->mmap_base; info.high_limit = TASK_SIZE; addr = vm_unmapped_area(&info); } return addr; }
/* * 1. Validate the Cache Geomtery (compile time config matches hardware) * 2. If I-cache suffers from aliasing, setup work arounds (difft flush rtn) * (aliasing D-cache configurations are not supported YET) * 3. Enable the Caches, setup default flush mode for D-Cache * 3. Calculate the SHMLBA used by user space */ void arc_cache_init(void) { unsigned int cpu = smp_processor_id(); struct cpuinfo_arc_cache *ic = &cpuinfo_arc700[cpu].icache; struct cpuinfo_arc_cache *dc = &cpuinfo_arc700[cpu].dcache; unsigned int dcache_does_alias, temp; char str[256]; printk(arc_cache_mumbojumbo(0, str, sizeof(str))); if (!ic->ver) goto chk_dc; #ifdef CONFIG_ARC_HAS_ICACHE /* 1. Confirm some of I-cache params which Linux assumes */ if (ic->line_len != ARC_ICACHE_LINE_LEN) panic("Cache H/W doesn't match kernel Config"); if (ic->ver != CONFIG_ARC_MMU_VER) panic("Cache ver doesn't match MMU ver\n"); #endif /* Enable/disable I-Cache */ temp = read_aux_reg(ARC_REG_IC_CTRL); #ifdef CONFIG_ARC_HAS_ICACHE temp &= ~IC_CTRL_CACHE_DISABLE; #else temp |= IC_CTRL_CACHE_DISABLE; #endif write_aux_reg(ARC_REG_IC_CTRL, temp); chk_dc: if (!dc->ver) return; #ifdef CONFIG_ARC_HAS_DCACHE if (dc->line_len != ARC_DCACHE_LINE_LEN) panic("Cache H/W doesn't match kernel Config"); /* check for D-Cache aliasing */ dcache_does_alias = (dc->sz / dc->assoc) > PAGE_SIZE; if (dcache_does_alias && !cache_is_vipt_aliasing()) panic("Enable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); else if (!dcache_does_alias && cache_is_vipt_aliasing()) panic("Don't need CONFIG_ARC_CACHE_VIPT_ALIASING\n"); #endif /* Set the default Invalidate Mode to "simpy discard dirty lines" * as this is more frequent then flush before invalidate * Ofcourse we toggle this default behviour when desired */ temp = read_aux_reg(ARC_REG_DC_CTRL); temp &= ~DC_CTRL_INV_MODE_FLUSH; #ifdef CONFIG_ARC_HAS_DCACHE /* Enable D-Cache: Clear Bit 0 */ write_aux_reg(ARC_REG_DC_CTRL, temp & ~IC_CTRL_CACHE_DISABLE); #else /* Flush D cache */ write_aux_reg(ARC_REG_DC_FLSH, 0x1); /* Disable D cache */ write_aux_reg(ARC_REG_DC_CTRL, temp | IC_CTRL_CACHE_DISABLE); #endif return; }