bt_paddr_t bt_mmu_extract(bt_pgd_t pgd_h, bt_vaddr_t virt, BT_u32 size) { bt_pte_t pte; bt_vaddr_t start, end, pg; bt_paddr_t pa; bt_pgd_t pgd = GET_PGD(pgd_h); start = BT_PAGE_TRUNC(virt); end = BT_PAGE_TRUNC(virt+size-1); // Check all pages exist. for(pg = start; pg <= end; pg += BT_PAGE_SIZE) { if(!pte_present(pgd, pg)) { return 0; } pte = virt_to_pte(pgd, pg); if(!page_present(pte, pg)) { return 0; } } pte = virt_to_pte(pgd, start); pa = (bt_paddr_t) pte_to_phys(pte, start); return pa + (bt_paddr_t) (virt - start); }
void arch_exit_mmap(struct mm_struct *mm) { pte_t *pte; pte = virt_to_pte(mm, STUB_CODE); if (pte != NULL) pte_clear(mm, STUB_CODE, pte); pte = virt_to_pte(mm, STUB_DATA); if (pte == NULL) return; pte_clear(mm, STUB_DATA, pte); }
static pte_t *maybe_map(unsigned long virt, int is_write) { pte_t *pte = virt_to_pte(current->mm, virt); int err, dummy_code; if ((pte == NULL) || !pte_present(*pte) || (is_write && !pte_write(*pte))) { err = handle_page_fault(virt, 0, is_write, 1, &dummy_code); if (err) return NULL; pte = virt_to_pte(current->mm, virt); } if (!pte_present(*pte)) pte = NULL; return pte; }
int page_home(struct page *page) { if (PageHighMem(page)) { return initial_page_home(); } else { unsigned long kva = (unsigned long)page_address(page); return pte_to_home(*virt_to_pte(NULL, kva)); } }
/* * The __w1data area holds data that is only written during initialization, * and is read-only and thus freely cacheable thereafter. Fix the page * table entries that cover that region accordingly. */ static void mark_w1data_ro(void) { /* Loop over page table entries */ unsigned long addr = (unsigned long)__w1data_begin; BUG_ON((addr & (PAGE_SIZE-1)) != 0); for (; addr <= (unsigned long)__w1data_end - 1; addr += PAGE_SIZE) { unsigned long pfn = kaddr_to_pfn((void *)addr); pte_t *ptep = virt_to_pte(NULL, addr); BUG_ON(pte_huge(*ptep)); /* not relevant for kdata_huge */ set_pte_at(&init_mm, addr, ptep, pfn_pte(pfn, PAGE_KERNEL_RO)); } }
static void free_init_pages(char *what, unsigned long begin, unsigned long end) { #ifdef CONFIG_HOMECACHE int home = initial_heap_home(); #endif unsigned long addr = (unsigned long) begin; if (kdata_huge && !initfree) { pr_warning("Warning: ignoring initfree=0:" " incompatible with kdata=huge\n"); initfree = 1; } end = (end + PAGE_SIZE - 1) & PAGE_MASK; local_flush_tlb_pages(NULL, begin, PAGE_SIZE, end - begin); for (addr = begin; addr < end; addr += PAGE_SIZE) { /* * Note we just reset the home here directly in the * page table. We know this is safe because our caller * just flushed the caches on all the other cpus, * and they won't be touching any of these pages. */ int pfn = kaddr_to_pfn((void *)addr); struct page *page = pfn_to_page(pfn); pte_t *ptep = virt_to_pte(NULL, addr); if (!initfree) { /* * If debugging page accesses then do not free * this memory but mark them not present - any * buggy init-section access will create a * kernel page fault: */ pte_clear(&init_mm, addr, ptep); continue; } #ifdef CONFIG_HOMECACHE set_page_home(page, home); __clear_bit(PG_homecache_nomigrate, &page->flags); #endif __ClearPageReserved(page); init_page_count(page); if (pte_huge(*ptep)) BUG_ON(!kdata_huge); else set_pte_at(&init_mm, addr, ptep, pfn_pte(pfn, PAGE_KERNEL)); memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE); free_page(addr); totalram_pages++; } pr_info("Freeing %s: %ldk freed\n", what, (end - begin) >> 10); }
// Return true (!= 0) if any referenced bits are set. static int ref_bits_set (int exclude_irqhandler) { void *cur_addr; pte_t *pte; int i; int ret_val = 0; for (i = 0; i < cr_num_drivers; i++) { if (exclude_irqhandler) uprintk ("i %d: ", i); for (cur_addr = cr_base_address[i]; cur_addr < cr_base_address[i] + cr_module_size[i]; cur_addr += PAGE_SIZE) { pte = virt_to_pte (cur_addr); if (pte != NULL) { // See if we're excluding the interrupt handler // from this check. if (exclude_irqhandler && addr_contains_irq_handler (cur_addr)) { pte_unmap(pte); if (exclude_irqhandler) uprintk ("X"); continue; } // See if the page was referenced lately. if (pte_young(*pte) != 0) { // kunmap_atomic (page, KM_IRQ1); pte_unmap(pte); if (exclude_irqhandler) uprintk ("1"); ret_val = 1; continue; } if (exclude_irqhandler) uprintk ("0"); // kunmap_atomic (page, KM_IRQ1); pte_unmap(pte); } } if (exclude_irqhandler) uprintk ("\n"); } return ret_val; }
void homecache_change_page_home(struct page *page, int order, int home) { int i, pages = (1 << order); unsigned long kva; BUG_ON(PageHighMem(page)); BUG_ON(page_count(page) > 1); BUG_ON(page_mapcount(page) != 0); kva = (unsigned long) page_address(page); flush_remote(0, HV_FLUSH_EVICT_L2, &cpu_cacheable_map, kva, pages * PAGE_SIZE, PAGE_SIZE, cpu_online_mask, NULL, 0); for (i = 0; i < pages; ++i, kva += PAGE_SIZE) { pte_t *ptep = virt_to_pte(NULL, kva); pte_t pteval = *ptep; BUG_ON(!pte_present(pteval) || pte_huge(pteval)); *ptep = pte_set_home(pteval, home); } }
// Reset all referenced bits to 0. static void clear_ref_bits (void) { void *cur_addr; pte_t *pte; int i; for (i = 0; i < cr_num_drivers; i++) { for (cur_addr = cr_base_address[i]; cur_addr < cr_base_address[i] + cr_module_size[i]; cur_addr += PAGE_SIZE) { pte = virt_to_pte (cur_addr); if (pte != NULL) { *pte = pte_mkold(*pte); // kunmap_atomic (page, KM_IRQ1); pte_unmap(pte); } } } global_flush_tlb(); }
/* * Walk the kernel page tables and derive the page_home() from * the PTEs, so that set_pte() can properly validate the caching * of all PTEs it sees. */ void __init set_page_homes(void) { #ifdef CONFIG_HOMECACHE struct zone *zone; int home = initial_heap_home(); unsigned long address; /* * First walk the zones and set the pages to all have * the default heap caching. */ for_each_zone(zone) { unsigned long pfn = zone->zone_start_pfn; unsigned long end_pfn = pfn + zone->spanned_pages; struct page *page = pfn_to_page(pfn); for (; pfn < end_pfn; ++pfn, ++page) set_page_home(page, home); } /* * Now walk through the loaded pages, update the page homecache, * and mark all pages as non-migrateable. (Init pages that * are freed back to the heap are unmarked when we free them.) */ for (address = PAGE_OFFSET; address < (unsigned long) _end; address += PAGE_SIZE) { enum { CODE_DELTA = MEM_SV_INTRPT - PAGE_OFFSET }; struct page *pg = virt_to_page((void *)address); pte_t pte = *virt_to_pte(NULL, address); /* Adjust page.home on all loaded pages. */ BUG_ON(!pte_present(pte)); set_page_home(pg, get_page_home(pte)); __SetPageHomecacheNomigrate(pg); } #endif }
/* * Identify large copies from remotely-cached memory, and copy them * via memcpy_multicache() if they look good, otherwise fall back * to the particular kind of copying passed as the memcpy_t function. */ static unsigned long fast_copy(void *dest, const void *source, int len, memcpy_t func) { /* * Check if it's big enough to bother with. We may end up doing a * small copy via TLB manipulation if we're near a page boundary, * but presumably we'll make it up when we hit the second page. */ while (len >= LARGE_COPY_CUTOFF) { int copy_size, bytes_left_on_page; pte_t *src_ptep, *dst_ptep; pte_t src_pte, dst_pte; struct page *src_page, *dst_page; /* Is the source page oloc'ed to a remote cpu? */ retry_source: src_ptep = virt_to_pte(current->mm, (unsigned long)source); if (src_ptep == NULL) break; src_pte = *src_ptep; if (!hv_pte_get_present(src_pte) || !hv_pte_get_readable(src_pte) || hv_pte_get_mode(src_pte) != HV_PTE_MODE_CACHE_TILE_L3) break; if (get_remote_cache_cpu(src_pte) == smp_processor_id()) break; src_page = pfn_to_page(pte_pfn(src_pte)); get_page(src_page); if (pte_val(src_pte) != pte_val(*src_ptep)) { put_page(src_page); goto retry_source; } if (pte_huge(src_pte)) { /* Adjust the PTE to correspond to a small page */ int pfn = pte_pfn(src_pte); pfn += (((unsigned long)source & (HPAGE_SIZE-1)) >> PAGE_SHIFT); src_pte = pfn_pte(pfn, src_pte); src_pte = pte_mksmall(src_pte); } /* Is the destination page writable? */ retry_dest: dst_ptep = virt_to_pte(current->mm, (unsigned long)dest); if (dst_ptep == NULL) { put_page(src_page); break; } dst_pte = *dst_ptep; if (!hv_pte_get_present(dst_pte) || !hv_pte_get_writable(dst_pte)) { put_page(src_page); break; } dst_page = pfn_to_page(pte_pfn(dst_pte)); if (dst_page == src_page) { /* * Source and dest are on the same page; this * potentially exposes us to incoherence if any * part of src and dest overlap on a cache line. * Just give up rather than trying to be precise. */ put_page(src_page); break; } get_page(dst_page); if (pte_val(dst_pte) != pte_val(*dst_ptep)) { put_page(dst_page); goto retry_dest; } if (pte_huge(dst_pte)) { /* Adjust the PTE to correspond to a small page */ int pfn = pte_pfn(dst_pte); pfn += (((unsigned long)dest & (HPAGE_SIZE-1)) >> PAGE_SHIFT); dst_pte = pfn_pte(pfn, dst_pte); dst_pte = pte_mksmall(dst_pte); } /* All looks good: create a cachable PTE and copy from it */ copy_size = len; bytes_left_on_page = PAGE_SIZE - (((int)source) & (PAGE_SIZE-1)); if (copy_size > bytes_left_on_page) copy_size = bytes_left_on_page; bytes_left_on_page = PAGE_SIZE - (((int)dest) & (PAGE_SIZE-1)); if (copy_size > bytes_left_on_page) copy_size = bytes_left_on_page; memcpy_multicache(dest, source, dst_pte, src_pte, copy_size); /* Release the pages */ put_page(dst_page); put_page(src_page); /* Continue on the next page */ dest += copy_size; source += copy_size; len -= copy_size; } return func(dest, source, len); }
static unsigned long fast_copy(void *dest, const void *source, int len, memcpy_t func) { /* */ while (len >= LARGE_COPY_CUTOFF) { int copy_size, bytes_left_on_page; pte_t *src_ptep, *dst_ptep; pte_t src_pte, dst_pte; struct page *src_page, *dst_page; /* */ retry_source: src_ptep = virt_to_pte(current->mm, (unsigned long)source); if (src_ptep == NULL) break; src_pte = *src_ptep; if (!hv_pte_get_present(src_pte) || !hv_pte_get_readable(src_pte) || hv_pte_get_mode(src_pte) != HV_PTE_MODE_CACHE_TILE_L3) break; if (get_remote_cache_cpu(src_pte) == smp_processor_id()) break; src_page = pfn_to_page(hv_pte_get_pfn(src_pte)); get_page(src_page); if (pte_val(src_pte) != pte_val(*src_ptep)) { put_page(src_page); goto retry_source; } if (pte_huge(src_pte)) { /* */ int pfn = hv_pte_get_pfn(src_pte); pfn += (((unsigned long)source & (HPAGE_SIZE-1)) >> PAGE_SHIFT); src_pte = pfn_pte(pfn, src_pte); src_pte = pte_mksmall(src_pte); } /* */ retry_dest: dst_ptep = virt_to_pte(current->mm, (unsigned long)dest); if (dst_ptep == NULL) { put_page(src_page); break; } dst_pte = *dst_ptep; if (!hv_pte_get_present(dst_pte) || !hv_pte_get_writable(dst_pte)) { put_page(src_page); break; } dst_page = pfn_to_page(hv_pte_get_pfn(dst_pte)); if (dst_page == src_page) { /* */ put_page(src_page); break; } get_page(dst_page); if (pte_val(dst_pte) != pte_val(*dst_ptep)) { put_page(dst_page); goto retry_dest; } if (pte_huge(dst_pte)) { /* */ int pfn = hv_pte_get_pfn(dst_pte); pfn += (((unsigned long)dest & (HPAGE_SIZE-1)) >> PAGE_SHIFT); dst_pte = pfn_pte(pfn, dst_pte); dst_pte = pte_mksmall(dst_pte); } /* */ copy_size = len; bytes_left_on_page = PAGE_SIZE - (((int)source) & (PAGE_SIZE-1)); if (copy_size > bytes_left_on_page) copy_size = bytes_left_on_page; bytes_left_on_page = PAGE_SIZE - (((int)dest) & (PAGE_SIZE-1)); if (copy_size > bytes_left_on_page) copy_size = bytes_left_on_page; memcpy_multicache(dest, source, dst_pte, src_pte, copy_size); /* */ put_page(dst_page); put_page(src_page); /* */ dest += copy_size; source += copy_size; len -= copy_size; } return func(dest, source, len); }
int bt_mmu_map(bt_pgd_t pgd_h, bt_paddr_t pa, bt_vaddr_t va, BT_u32 size, int type) { BT_u32 flag = 0; bt_pte_t pte; bt_paddr_t pg; BT_u32 ng = 0; bt_pgd_t pgd = GET_PGD(pgd_h); if((va + size) < 0xC0000000) { ng = MMU_PTE_NG; } pa = BT_PAGE_TRUNC(pa); // Ensure correct alignments. va = BT_PAGE_TRUNC(va); size = BT_PAGE_ALIGN(size); switch(type) { // Build up the ARM MMU flags from BT page types. case BT_PAGE_UNMAP: flag = 0; break; case BT_PAGE_READ: flag = (BT_u32) (MMU_PTE_PRESENT | MMU_PTE_WBUF | MMU_PTE_CACHE | MMU_PTE_USER_RO); break; case BT_PAGE_WRITE: flag = (BT_u32) (MMU_PTE_PRESENT | MMU_PTE_WBUF | MMU_PTE_CACHE | MMU_PTE_USER_RW); break; case BT_PAGE_SYSTEM: flag = (BT_u32) (MMU_PTE_PRESENT | MMU_PTE_WBUF | MMU_PTE_CACHE | MMU_PTE_SYSTEM); break; case BT_PAGE_IOMEM: flag = (BT_u32) (MMU_PTE_PRESENT | MMU_PTE_SYSTEM); break; default: //do_kernel_panic("bt_mmu_map"); return -1; break; } bt_mmu_flush_tlb(); while(size > 0) { if(pte_present(pgd, va)) { pte = virt_to_pte(pgd, va); // Get the page table from PGD. } else { // If its a section or super-section then return an error! - (Kernel coherent pool?). pg = (bt_paddr_t) BT_CacheAlloc(&g_ptCache); if(!pg) { return -1; } memset((void *)pg, 0, MMU_L2TBL_SIZE); pte = (bt_pte_t) pg; pgd[PAGE_DIR(va)] = (BT_u32) bt_virt_to_phys(pte) | MMU_PDE_PRESENT; } pte[PAGE_TABLE(va)] = (BT_u32) pa | flag | ng; pa += BT_PAGE_SIZE; va += BT_PAGE_SIZE; size -= BT_PAGE_SIZE; } bt_mmu_flush_tlb(); return 0; }