/*
* Place a pointer to an L2 page table in a middle page
* directory entry.
*/
static void __init assign_pte(pmd_t *pmd, pte_t *page_table)
{
phys_addr_t pa = __pa(page_table);
unsigned long l2_ptfn = pa >> HV_LOG2_PAGE_TABLE_ALIGN;
pte_t pteval = hv_pte_set_ptfn(__pgprot(_PAGE_TABLE), l2_ptfn);
BUG_ON((pa & (HV_PAGE_TABLE_ALIGN-1)) != 0);
pteval = pte_set_home(pteval, initial_heap_home());
*(pte_t *)pmd = pteval;
if (page_table != (pte_t *)pmd_page_vaddr(*pmd))
BUG();
}
static void free_init_pages(char *what, unsigned long begin, unsigned long end)
{
#ifdef CONFIG_HOMECACHE
int home = initial_heap_home();
#endif
unsigned long addr = (unsigned long) begin;
if (kdata_huge && !initfree) {
pr_warning("Warning: ignoring initfree=0:"
" incompatible with kdata=huge\n");
initfree = 1;
}
end = (end + PAGE_SIZE - 1) & PAGE_MASK;
local_flush_tlb_pages(NULL, begin, PAGE_SIZE, end - begin);
for (addr = begin; addr < end; addr += PAGE_SIZE) {
/*
* Note we just reset the home here directly in the
* page table. We know this is safe because our caller
* just flushed the caches on all the other cpus,
* and they won't be touching any of these pages.
*/
int pfn = kaddr_to_pfn((void *)addr);
struct page *page = pfn_to_page(pfn);
pte_t *ptep = virt_to_pte(NULL, addr);
if (!initfree) {
/*
* If debugging page accesses then do not free
* this memory but mark them not present - any
* buggy init-section access will create a
* kernel page fault:
*/
pte_clear(&init_mm, addr, ptep);
continue;
}
#ifdef CONFIG_HOMECACHE
set_page_home(page, home);
__clear_bit(PG_homecache_nomigrate, &page->flags);
#endif
__ClearPageReserved(page);
init_page_count(page);
if (pte_huge(*ptep))
BUG_ON(!kdata_huge);
else
set_pte_at(&init_mm, addr, ptep,
pfn_pte(pfn, PAGE_KERNEL));
memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
free_page(addr);
totalram_pages++;
}
pr_info("Freeing %s: %ldk freed\n", what, (end - begin) >> 10);
}
/*
* Walk the kernel page tables and derive the page_home() from
* the PTEs, so that set_pte() can properly validate the caching
* of all PTEs it sees.
*/
void __init set_page_homes(void)
{
#ifdef CONFIG_HOMECACHE
struct zone *zone;
int home = initial_heap_home();
unsigned long address;
/*
* First walk the zones and set the pages to all have
* the default heap caching.
*/
for_each_zone(zone) {
unsigned long pfn = zone->zone_start_pfn;
unsigned long end_pfn = pfn + zone->spanned_pages;
struct page *page = pfn_to_page(pfn);
for (; pfn < end_pfn; ++pfn, ++page)
set_page_home(page, home);
}
/*
* Now walk through the loaded pages, update the page homecache,
* and mark all pages as non-migrateable. (Init pages that
* are freed back to the heap are unmarked when we free them.)
*/
for (address = PAGE_OFFSET; address < (unsigned long) _end;
address += PAGE_SIZE) {
enum { CODE_DELTA = MEM_SV_INTRPT - PAGE_OFFSET };
struct page *pg = virt_to_page((void *)address);
pte_t pte = *virt_to_pte(NULL, address);
/* Adjust page.home on all loaded pages. */
BUG_ON(!pte_present(pte));
set_page_home(pg, get_page_home(pte));
__SetPageHomecacheNomigrate(pg);
}
#endif
}
/*
* For a given kernel data VA, how should it be cached?
* We return the complete pgprot_t with caching bits set.
*/
static pgprot_t __init init_pgprot(ulong address)
{
int cpu;
unsigned long page;
enum { CODE_DELTA = MEM_SV_INTRPT - PAGE_OFFSET };
#if CHIP_HAS_CBOX_HOME_MAP()
/* For kdata=huge, everything is just hash-for-home. */
if (kdata_huge)
return construct_pgprot(PAGE_KERNEL, PAGE_HOME_HASH);
#endif
/* We map the aliased pages of permanent text inaccessible. */
if (address < (ulong) _sinittext - CODE_DELTA)
return PAGE_NONE;
/*
* We map read-only data non-coherent for performance. We could
* use neighborhood caching on TILE64, but it's not clear it's a win.
*/
if ((address >= (ulong) __start_rodata &&
address < (ulong) __end_rodata) ||
address == (ulong) empty_zero_page) {
return construct_pgprot(PAGE_KERNEL_RO, PAGE_HOME_IMMUTABLE);
}
#ifndef __tilegx__
#if !ATOMIC_LOCKS_FOUND_VIA_TABLE()
/* Force the atomic_locks[] array page to be hash-for-home. */
if (address == (ulong) atomic_locks)
return construct_pgprot(PAGE_KERNEL, PAGE_HOME_HASH);
#endif
#endif
/*
* Everything else that isn't data or bss is heap, so mark it
* with the initial heap home (hash-for-home, or this cpu). This
* includes any addresses after the loaded image and any address before
* _einitdata, since we already captured the case of text before
* _sinittext, and __pa(einittext) is approximately __pa(sinitdata).
*
* All the LOWMEM pages that we mark this way will get their
* struct page homecache properly marked later, in set_page_homes().
* The HIGHMEM pages we leave with a default zero for their
* homes, but with a zero free_time we don't have to actually
* do a flush action the first time we use them, either.
*/
if (address >= (ulong) _end || address < (ulong) _einitdata)
return construct_pgprot(PAGE_KERNEL, initial_heap_home());
#if CHIP_HAS_CBOX_HOME_MAP()
/* Use hash-for-home if requested for data/bss. */
if (kdata_hash)
return construct_pgprot(PAGE_KERNEL, PAGE_HOME_HASH);
#endif
/*
* Make the w1data homed like heap to start with, to avoid
* making it part of the page-striped data area when we're just
* going to convert it to read-only soon anyway.
*/
if (address >= (ulong)__w1data_begin && address < (ulong)__w1data_end)
return construct_pgprot(PAGE_KERNEL, initial_heap_home());
/*
* Otherwise we just hand out consecutive cpus. To avoid
* requiring this function to hold state, we just walk forward from
* _sdata by PAGE_SIZE, skipping the readonly and init data, to reach
* the requested address, while walking cpu home around kdata_mask.
* This is typically no more than a dozen or so iterations.
*/
page = (((ulong)__w1data_end) + PAGE_SIZE - 1) & PAGE_MASK;
BUG_ON(address < page || address >= (ulong)_end);
cpu = cpumask_first(&kdata_mask);
for (; page < address; page += PAGE_SIZE) {
if (page >= (ulong)&init_thread_union &&
page < (ulong)&init_thread_union + THREAD_SIZE)
continue;
if (page == (ulong)empty_zero_page)
continue;
#ifndef __tilegx__
#if !ATOMIC_LOCKS_FOUND_VIA_TABLE()
if (page == (ulong)atomic_locks)
continue;
#endif
#endif
cpu = cpumask_next(cpu, &kdata_mask);
if (cpu == NR_CPUS)
cpu = cpumask_first(&kdata_mask);
}
return construct_pgprot(PAGE_KERNEL, cpu);
}
/*
* For a given kernel data VA, how should it be cached?
* We return the complete pgprot_t with caching bits set.
*/
static pgprot_t __init init_pgprot(ulong address)
{
int cpu;
unsigned long page;
enum { CODE_DELTA = MEM_SV_START - PAGE_OFFSET };
/* For kdata=huge, everything is just hash-for-home. */
if (kdata_huge)
return construct_pgprot(PAGE_KERNEL, PAGE_HOME_HASH);
/*
* We map the aliased pages of permanent text so we can
* update them if necessary, for ftrace, etc.
*/
if (address < (ulong) _sinittext - CODE_DELTA)
return construct_pgprot(PAGE_KERNEL, PAGE_HOME_HASH);
/* We map read-only data non-coherent for performance. */
if ((address >= (ulong) __start_rodata &&
address < (ulong) __end_rodata) ||
address == (ulong) empty_zero_page) {
return construct_pgprot(PAGE_KERNEL_RO, PAGE_HOME_IMMUTABLE);
}
#ifndef __tilegx__
/* Force the atomic_locks[] array page to be hash-for-home. */
if (address == (ulong) atomic_locks)
return construct_pgprot(PAGE_KERNEL, PAGE_HOME_HASH);
#endif
/*
* Everything else that isn't data or bss is heap, so mark it
* with the initial heap home (hash-for-home, or this cpu). This
* includes any addresses after the loaded image and any address before
* __init_end, since we already captured the case of text before
* _sinittext, and __pa(einittext) is approximately __pa(__init_begin).
*
* All the LOWMEM pages that we mark this way will get their
* struct page homecache properly marked later, in set_page_homes().
* The HIGHMEM pages we leave with a default zero for their
* homes, but with a zero free_time we don't have to actually
* do a flush action the first time we use them, either.
*/
if (address >= (ulong) _end || address < (ulong) __init_end)
return construct_pgprot(PAGE_KERNEL, initial_heap_home());
/* Use hash-for-home if requested for data/bss. */
if (kdata_hash)
return construct_pgprot(PAGE_KERNEL, PAGE_HOME_HASH);
/*
* Otherwise we just hand out consecutive cpus. To avoid
* requiring this function to hold state, we just walk forward from
* __end_rodata by PAGE_SIZE, skipping the readonly and init data, to
* reach the requested address, while walking cpu home around
* kdata_mask. This is typically no more than a dozen or so iterations.
*/
page = (((ulong)__end_rodata) + PAGE_SIZE - 1) & PAGE_MASK;
BUG_ON(address < page || address >= (ulong)_end);
cpu = cpumask_first(&kdata_mask);
for (; page < address; page += PAGE_SIZE) {
if (page >= (ulong)&init_thread_union &&
page < (ulong)&init_thread_union + THREAD_SIZE)
continue;
if (page == (ulong)empty_zero_page)
continue;
#ifndef __tilegx__
if (page == (ulong)atomic_locks)
continue;
#endif
cpu = cpumask_next(cpu, &kdata_mask);
if (cpu == NR_CPUS)
cpu = cpumask_first(&kdata_mask);
}
return construct_pgprot(PAGE_KERNEL, cpu);
}
