static struct page * rmqueue(zone_t *zone, unsigned int order)
{
free_area_t * area = zone->free_area + order;
unsigned int curr_order = order;
struct list_head *head, *curr;
unsigned long flags;
struct page *page;
int i;
spin_lock_irqsave(&zone->lock, flags);
do {
head = &area->free_list;
curr = head->next;
if (curr != head) {
unsigned int index;
page = list_entry(curr, struct page, list);
if (BAD_RANGE(zone,page))
BUG();
list_del(curr);
index = page - zone->zone_mem_map;
if (curr_order != MAX_ORDER-1)
MARK_USED(index, curr_order, area);
zone->free_pages -= 1UL << order;
page = expand(zone, page, index, order, curr_order, area);
spin_unlock_irqrestore(&zone->lock, flags);
set_page_count(page, 1);
if (BAD_RANGE(zone,page))
BUG();
if (PageLRU(page))
BUG();
if (PageActive(page))
BUG();
/*
* we need to reference all the pages for this order,
* otherwise if anyone accesses one of the pages with
* (get/put) it * will be freed :-(
*/
for (i = 1; i < (1 << order); i++)
set_page_count(&page[i], 1);
return page;
}
curr_order++;
area++;
} while (curr_order < MAX_ORDER);
static pte_t *__alloc_for_cache(struct mm_struct *mm, int kernel)
{
void *ret = NULL;
struct page *page;
if (!kernel) {
page = alloc_page(PGALLOC_GFP | __GFP_ACCOUNT);
if (!page)
return NULL;
if (!pgtable_page_ctor(page)) {
__free_page(page);
return NULL;
}
} else {
page = alloc_page(PGALLOC_GFP);
if (!page)
return NULL;
}
ret = page_address(page);
spin_lock(&mm->page_table_lock);
/*
* If we find pgtable_page set, we return
* the allocated page with single fragement
* count.
*/
if (likely(!mm->context.pte_frag)) {
set_page_count(page, PTE_FRAG_NR);
mm->context.pte_frag = ret + PTE_FRAG_SIZE;
}
spin_unlock(&mm->page_table_lock);
return (pte_t *)ret;
}
void __init prom_free_prom_memory (void)
{
unsigned long addr, end;
/*
* Free everything below the kernel itself but leave
* the first page reserved for the exception handlers.
*/
#if defined(CONFIG_DECLANCE) || defined(CONFIG_DECLANCE_MODULE)
/*
* Leave 128 KB reserved for Lance memory for
* IOASIC DECstations.
*
* XXX: save this address for use in dec_lance.c?
*/
if (IOASIC)
end = __pa(&_text) - 0x00020000;
else
#endif
end = __pa(&_text);
addr = PAGE_SIZE;
while (addr < end) {
ClearPageReserved(virt_to_page(__va(addr)));
set_page_count(virt_to_page(__va(addr)), 1);
free_page((unsigned long)__va(addr));
addr += PAGE_SIZE;
}
printk("Freeing unused PROM memory: %ldk freed\n",
(end - PAGE_SIZE) >> 10);
}
void online_page(struct page *page)
{
ClearPageReserved(page);
set_page_count(page, 0);
free_cold_page(page);
totalram_pages++;
num_physpages++;
}
/*
* This function will allocate the requested contiguous pages and
* map them into the kernel's vmalloc() space. This is done so we
* get unique mapping for these pages, outside of the kernel's 1:1
* virtual:physical mapping. This is necessary so we can cover large
* portions of the kernel with single large page TLB entries, and
* still get unique uncached pages for consistent DMA.
*/
void *consistent_alloc(int gfp, size_t size, dma_addr_t *dma_handle)
{
struct vm_struct *area;
unsigned long page, va, pa;
void *ret;
int order, err, i;
if (in_interrupt())
BUG();
/* only allocate page size areas */
size = PAGE_ALIGN(size);
order = get_order(size);
page = __get_free_pages(gfp, order);
if (!page) {
BUG();
return NULL;
}
/* allocate some common virtual space to map the new pages */
area = get_vm_area(size, VM_ALLOC);
if (area == 0) {
free_pages(page, order);
return NULL;
}
va = VMALLOC_VMADDR(area->addr);
ret = (void *) va;
/* this gives us the real physical address of the first page */
*dma_handle = pa = virt_to_bus((void *) page);
/* set refcount=1 on all pages in an order>0 allocation so that vfree() will actually free
* all pages that were allocated.
*/
if (order > 0) {
struct page *rpage = virt_to_page(page);
for (i = 1; i < (1 << order); i++)
set_page_count(rpage + i, 1);
}
err = 0;
for (i = 0; i < size && err == 0; i += PAGE_SIZE)
err = map_page(va + i, pa + i, PAGE_KERNEL_NOCACHE);
if (err) {
vfree((void *) va);
return NULL;
}
/* we need to ensure that there are no cachelines in use, or worse dirty in this area
* - can't do until after virtual address mappings are created
*/
frv_cache_invalidate(va, va + size);
return ret;
}
void release_individual_page_counts(void *buffer, unsigned long order) {
int i;
struct page *spa;
if (order) {
for (i=1;i<(1 << order);i++) {
spa=virt_to_page(buffer+i*PAGE_SIZE);
set_page_count(spa,0);
}
}
}
/* code to manipulate page counters of page blocks */
void add_individual_page_counts(void *buffer, unsigned long order) {
int i,orig;
struct page *spa;
if (order) {
orig=page_count(virt_to_page(buffer)); /* first page setting */
for (i=1;i<(1 << order);i++) {
spa=virt_to_page(buffer+i*PAGE_SIZE);
set_page_count(spa,orig);
}
}
}
void free_initrd_mem(unsigned long start, unsigned long end)
{
int pages = 0;
for (; start < end; start += PAGE_SIZE) {
ClearPageReserved(virt_to_page(start));
set_page_count(virt_to_page(start), 1);
free_page(start);
totalram_pages++;
pages++;
}
printk ("Freeing initrd memory: %dk freed\n", pages);
}
void __init mem_init(void)
{
unsigned long npages = (memory_end - memory_start) >> PAGE_SHIFT;
unsigned long tmp;
#ifdef CONFIG_MMU
unsigned long loop, pfn;
int datapages = 0;
#endif
int codek = 0, datak = 0;
/* this will put all memory onto the freelists */
totalram_pages = free_all_bootmem();
#ifdef CONFIG_MMU
for (loop = 0 ; loop < npages ; loop++)
if (PageReserved(&mem_map[loop]))
datapages++;
#ifdef CONFIG_HIGHMEM
for (pfn = num_physpages - 1; pfn >= num_mappedpages; pfn--) {
struct page *page = &mem_map[pfn];
ClearPageReserved(page);
set_bit(PG_highmem, &page->flags);
set_page_count(page, 1);
__free_page(page);
totalram_pages++;
}
#endif
codek = ((unsigned long) &_etext - (unsigned long) &_stext) >> 10;
datak = datapages << (PAGE_SHIFT - 10);
#else
codek = (_etext - _stext) >> 10;
datak = 0; //(_ebss - _sdata) >> 10;
#endif
tmp = nr_free_pages() << PAGE_SHIFT;
printk("Memory available: %luKiB/%luKiB RAM, %luKiB/%luKiB ROM (%dKiB kernel code, %dKiB data)\n",
tmp >> 10,
npages << (PAGE_SHIFT - 10),
(rom_length > 0) ? ((rom_length >> 10) - codek) : 0,
rom_length >> 10,
codek,
datak
);
} /* end mem_init() */
/* Free up now-unused memory */
static void free_sec(unsigned long start, unsigned long end, const char *name)
{
unsigned long cnt = 0;
while (start < end) {
ClearPageReserved(virt_to_page(start));
set_page_count(virt_to_page(start), 1);
free_page(start);
cnt++;
start += PAGE_SIZE;
}
if (cnt) {
printk(" %ldk %s", cnt << (PAGE_SHIFT - 10), name);
totalram_pages += cnt;
}
}
/*
* free the memory that was only required for initialisation
*/
void __init free_initmem(void)
{
#if defined(CONFIG_RAMKERNEL) && !defined(CONFIG_PROTECT_KERNEL)
unsigned long start, end, addr;
start = PAGE_ALIGN((unsigned long) &__init_begin); /* round up */
end = ((unsigned long) &__init_end) & PAGE_MASK; /* round down */
/* next to check that the page we free is not a partial page */
for (addr = start; addr < end; addr += PAGE_SIZE) {
ClearPageReserved(virt_to_page(addr));
set_page_count(virt_to_page(addr), 1);
free_page(addr);
totalram_pages++;
}
printk("Freeing unused kernel memory: %ldKiB freed (0x%lx - 0x%lx)\n",
(end - start) >> 10, start, end);
#endif
} /* end free_initmem() */
void prom_free_prom_memory (void)
{
struct prom_pmemblock *p;
unsigned long freed = 0;
unsigned long addr;
for (p = pblocks; p->size != 0; p++) {
if (p->type != MEMTYPE_PROM)
continue;
addr = p->base;
while (addr < p->base + p->size) {
ClearPageReserved(virt_to_page(__va(addr)));
set_page_count(virt_to_page(__va(addr)), 1);
free_page(__va(addr));
addr += PAGE_SIZE;
freed += PAGE_SIZE;
}
}
printk("Freeing prom memory: %ldkb freed\n", freed >> 10);
}
static struct page * rmqueue(zone_t *zone, unsigned int order)
{
free_area_t * area = zone->free_area + order;
unsigned int curr_order = order;
struct list_head *head, *curr;
unsigned long flags;
struct page *page;
spin_lock_irqsave(&zone->lock, flags);
do {
head = &area->free_list;
curr = head->next;
if (curr != head) {
unsigned int index;
page = list_entry(curr, struct page, list);
if (BAD_RANGE(zone,page))
BUG();
list_del(curr);
index = page - zone->zone_mem_map;
if (curr_order != MAX_ORDER-1)
MARK_USED(index, curr_order, area);
zone->free_pages -= 1UL << order;
page = expand(zone, page, index, order, curr_order, area);
spin_unlock_irqrestore(&zone->lock, flags);
set_page_count(page, 1);
if (BAD_RANGE(zone,page))
BUG();
if (PageLRU(page))
BUG();
if (PageActive(page))
BUG();
return page;
}
curr_order++;
area++;
} while (curr_order < MAX_ORDER);
void __init
prom_free_prom_memory (void)
{
int i;
unsigned long freed = 0;
unsigned long addr;
for (i = 0; i < boot_mem_map.nr_map; i++) {
if (boot_mem_map.map[i].type != BOOT_MEM_ROM_DATA)
continue;
addr = boot_mem_map.map[i].addr;
while (addr < boot_mem_map.map[i].addr
+ boot_mem_map.map[i].size) {
ClearPageReserved(virt_to_page(__va(addr)));
set_page_count(virt_to_page(__va(addr)), 1);
free_page((unsigned long)__va(addr));
addr += PAGE_SIZE;
freed += PAGE_SIZE;
}
}
printk("Freeing prom memory: %ldkb freed\n", freed >> 10);
}
void
free_initmem()
{
#ifdef CONFIG_RAMKERNEL
unsigned long addr;
extern char __init_begin, __init_end;
/*
* the following code should be cool even if these sections
* are not page aligned.
*/
addr = PAGE_ALIGN((unsigned long)(&__init_begin));
/* next to check that the page we free is not a partial page */
for (; addr + PAGE_SIZE < (unsigned long)(&__init_end); addr +=PAGE_SIZE) {
ClearPageReserved(virt_to_page(addr));
set_page_count(virt_to_page(addr), 1);
free_page(addr);
totalram_pages++;
}
printk("Freeing unused kernel memory: %ldk freed (0x%x - 0x%x)\n",
(addr - PAGE_ALIGN((long) &__init_begin)) >> 10,
(int)(PAGE_ALIGN((unsigned long)(&__init_begin))),
(int)(addr - PAGE_SIZE));
#endif
}
void __init mem_init(void)
{
#ifdef CONFIG_NEED_MULTIPLE_NODES
int nid;
#endif
pg_data_t *pgdat;
unsigned long i;
struct page *page;
unsigned long reservedpages = 0, codesize, initsize, datasize, bsssize;
num_physpages = lmb.memory.size >> PAGE_SHIFT;
high_memory = (void *) __va(max_low_pfn * PAGE_SIZE);
#ifdef CONFIG_NEED_MULTIPLE_NODES
for_each_online_node(nid) {
if (NODE_DATA(nid)->node_spanned_pages != 0) {
printk("freeing bootmem node %x\n", nid);
totalram_pages +=
free_all_bootmem_node(NODE_DATA(nid));
}
}
#else
max_mapnr = max_pfn;
totalram_pages += free_all_bootmem();
#endif
for_each_pgdat(pgdat) {
for (i = 0; i < pgdat->node_spanned_pages; i++) {
if (!pfn_valid(pgdat->node_start_pfn + i))
continue;
page = pgdat_page_nr(pgdat, i);
if (PageReserved(page))
reservedpages++;
}
}
codesize = (unsigned long)&_sdata - (unsigned long)&_stext;
datasize = (unsigned long)&_edata - (unsigned long)&_sdata;
initsize = (unsigned long)&__init_end - (unsigned long)&__init_begin;
bsssize = (unsigned long)&__bss_stop - (unsigned long)&__bss_start;
#ifdef CONFIG_HIGHMEM
{
unsigned long pfn, highmem_mapnr;
highmem_mapnr = total_lowmem >> PAGE_SHIFT;
for (pfn = highmem_mapnr; pfn < max_mapnr; ++pfn) {
struct page *page = pfn_to_page(pfn);
ClearPageReserved(page);
set_page_count(page, 1);
__free_page(page);
totalhigh_pages++;
}
totalram_pages += totalhigh_pages;
printk(KERN_INFO "High memory: %luk\n",
totalhigh_pages << (PAGE_SHIFT-10));
}
#endif /* CONFIG_HIGHMEM */
printk(KERN_INFO "Memory: %luk/%luk available (%luk kernel code, "
"%luk reserved, %luk data, %luk bss, %luk init)\n",
(unsigned long)nr_free_pages() << (PAGE_SHIFT-10),
num_physpages << (PAGE_SHIFT-10),
codesize >> 10,
reservedpages << (PAGE_SHIFT-10),
datasize >> 10,
bsssize >> 10,
initsize >> 10);
mem_init_done = 1;
/* Initialize the vDSO */
vdso_init();
}
static void __free_pages_ok (struct page *page, unsigned int order)
{
unsigned long index, page_idx, mask, flags;
free_area_t *area;
struct page *base;
zone_t *zone;
/*
* Subtle. We do not want to test this in the inlined part of
* __free_page() - it's a rare condition and just increases
* cache footprint unnecesserily. So we do an 'incorrect'
* decrement on page->count for reserved pages, but this part
* makes it safe.
*/
if (PageReserved(page))
return;
/*
* Yes, think what happens when other parts of the kernel take
* a reference to a page in order to pin it for io. -ben
*/
if (PageLRU(page)) {
if (unlikely(in_interrupt())) {
unsigned long flags;
spin_lock_irqsave(&free_pages_ok_no_irq_lock, flags);
page->next_hash = free_pages_ok_no_irq_head;
free_pages_ok_no_irq_head = page;
page->index = order;
spin_unlock_irqrestore(&free_pages_ok_no_irq_lock, flags);
schedule_task(&free_pages_ok_no_irq_task);
return;
}
lru_cache_del(page);
}
if (page->buffers)
BUG();
if (page->mapping)
BUG();
if (!VALID_PAGE(page))
BUG();
if (PageLocked(page))
BUG();
if (PageActive(page))
BUG();
ClearPageReferenced(page);
ClearPageDirty(page);
/* de-reference all the pages for this order */
for (page_idx = 1; page_idx < (1 << order); page_idx++)
set_page_count(&page[page_idx], 0);
if (current->flags & PF_FREE_PAGES)
goto local_freelist;
back_local_freelist:
zone = page_zone(page);
mask = (~0UL) << order;
base = zone->zone_mem_map;
page_idx = page - base;
if (page_idx & ~mask)
BUG();
index = page_idx >> (1 + order);
area = zone->free_area + order;
spin_lock_irqsave(&zone->lock, flags);
zone->free_pages -= mask;
while (mask + (1 << (MAX_ORDER-1))) {
struct page *buddy1, *buddy2;
if (area >= zone->free_area + MAX_ORDER)
BUG();
if (!__test_and_change_bit(index, area->map))
/*
* the buddy page is still allocated.
*/
break;
/*
* Move the buddy up one level.
* This code is taking advantage of the identity:
* -mask = 1+~mask
*/
buddy1 = base + (page_idx ^ -mask);
buddy2 = base + page_idx;
if (BAD_RANGE(zone,buddy1))
BUG();
if (BAD_RANGE(zone,buddy2))
BUG();
list_del(&buddy1->list);
mask <<= 1;
area++;
index >>= 1;
page_idx &= mask;
}
list_add(&(base + page_idx)->list, &area->free_list);
spin_unlock_irqrestore(&zone->lock, flags);
return;
local_freelist:
if (current->nr_local_pages)
goto back_local_freelist;
if (in_interrupt())
goto back_local_freelist;
list_add(&page->list, ¤t->local_pages);
page->index = order;
current->nr_local_pages++;
}
static int increase_reservation(unsigned long nr_pages)
{
unsigned long pfn, i, flags;
struct page *page;
long rc;
struct xen_memory_reservation reservation = {
.address_bits = 0,
.extent_order = 0,
.domid = DOMID_SELF
};
if (nr_pages > ARRAY_SIZE(frame_list))
nr_pages = ARRAY_SIZE(frame_list);
balloon_lock(flags);
page = balloon_first_page();
for (i = 0; i < nr_pages; i++) {
BUG_ON(page == NULL);
frame_list[i] = page_to_pfn(page);;
page = balloon_next_page(page);
}
set_xen_guest_handle(reservation.extent_start, frame_list);
reservation.nr_extents = nr_pages;
rc = HYPERVISOR_memory_op(
XENMEM_populate_physmap, &reservation);
if (rc < nr_pages) {
int ret;
/* We hit the Xen hard limit: reprobe. */
set_xen_guest_handle(reservation.extent_start, frame_list);
reservation.nr_extents = rc;
ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
&reservation);
BUG_ON(ret != rc);
hard_limit = current_pages + rc - driver_pages;
goto out;
}
for (i = 0; i < nr_pages; i++) {
page = balloon_retrieve();
BUG_ON(page == NULL);
pfn = page_to_pfn(page);
BUG_ON(!xen_feature(XENFEAT_auto_translated_physmap) &&
phys_to_machine_mapping_valid(pfn));
/* Update P->M and M->P tables. */
set_phys_to_machine(pfn, frame_list[i]);
#ifdef CONFIG_XEN
xen_machphys_update(frame_list[i], pfn);
/* Link back into the page tables if not highmem. */
if (pfn < max_low_pfn) {
int ret;
ret = HYPERVISOR_update_va_mapping(
(unsigned long)__va(pfn << PAGE_SHIFT),
pfn_pte_ma(frame_list[i], PAGE_KERNEL),
0);
BUG_ON(ret);
}
#endif
/* Relinquish the page back to the allocator. */
ClearPageReserved(page);
set_page_count(page, 1);
__free_page(page);
}
current_pages += nr_pages;
totalram_pages = current_pages;
out:
balloon_unlock(flags);
return 0;
}
static int decrease_reservation(unsigned long nr_pages)
{
unsigned long pfn, i, flags;
struct page *page;
void *v;
int need_sleep = 0;
int ret;
struct xen_memory_reservation reservation = {
.address_bits = 0,
.extent_order = 0,
.domid = DOMID_SELF
};
if (nr_pages > ARRAY_SIZE(frame_list))
nr_pages = ARRAY_SIZE(frame_list);
for (i = 0; i < nr_pages; i++) {
if ((page = alloc_page(GFP_BALLOON)) == NULL) {
nr_pages = i;
need_sleep = 1;
break;
}
pfn = page_to_pfn(page);
frame_list[i] = pfn_to_mfn(pfn);
if (!PageHighMem(page)) {
v = phys_to_virt(pfn << PAGE_SHIFT);
scrub_pages(v, 1);
#ifdef CONFIG_XEN
ret = HYPERVISOR_update_va_mapping(
(unsigned long)v, __pte_ma(0), 0);
BUG_ON(ret);
#endif
}
#ifdef CONFIG_XEN_SCRUB_PAGES
else {
v = kmap(page);
scrub_pages(v, 1);
kunmap(page);
}
#endif
}
#ifdef CONFIG_XEN
/* Ensure that ballooned highmem pages don't have kmaps. */
kmap_flush_unused();
flush_tlb_all();
#endif
balloon_lock(flags);
/* No more mappings: invalidate P2M and add to balloon. */
for (i = 0; i < nr_pages; i++) {
pfn = mfn_to_pfn(frame_list[i]);
set_phys_to_machine(pfn, INVALID_P2M_ENTRY);
balloon_append(pfn_to_page(pfn));
}
set_xen_guest_handle(reservation.extent_start, frame_list);
reservation.nr_extents = nr_pages;
ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation);
BUG_ON(ret != nr_pages);
current_pages -= nr_pages;
totalram_pages = current_pages;
balloon_unlock(flags);
return need_sleep;
}
/*
* We avoid multiple worker processes conflicting via the balloon mutex.
* We may of course race updates of the target counts (which are protected
* by the balloon lock), or with changes to the Xen hard limit, but we will
* recover from these in time.
*/
static void balloon_process(void *unused)
{
int need_sleep = 0;
long credit;
down(&balloon_mutex);
do {
credit = current_target() - current_pages;
if (credit > 0)
need_sleep = (increase_reservation(credit) != 0);
if (credit < 0)
need_sleep = (decrease_reservation(-credit) != 0);
#ifndef CONFIG_PREEMPT
if (need_resched())
schedule();
#endif
} while ((credit != 0) && !need_sleep);
/* Schedule more work if there is some still to be done. */
if (current_target() != current_pages)
mod_timer(&balloon_timer, jiffies + HZ);
up(&balloon_mutex);
}
unsigned long __init prom_free_prom_memory(void)
{
unsigned long freed = 0;
unsigned long addr;
int i;
#ifdef CONFIG_REALTEK_RECLAIM_BOOT_MEM
unsigned long dest;
struct page *page;
int count;
#endif
for (i = 0; i < boot_mem_map.nr_map; i++) {
if (boot_mem_map.map[i].type != BOOT_MEM_ROM_DATA)
continue;
addr = boot_mem_map.map[i].addr;
while (addr < boot_mem_map.map[i].addr
+ boot_mem_map.map[i].size) {
ClearPageReserved(virt_to_page(__va(addr)));
set_page_count(virt_to_page(__va(addr)), 1);
free_page((unsigned long)__va(addr));
addr += PAGE_SIZE;
freed += PAGE_SIZE;
}
}
printk("Freeing prom memory: %ldkb freed\n", freed >> 10);
#ifdef CONFIG_REALTEK_RECLAIM_BOOT_MEM
if (!is_mars_cpu()) {
// venus or neptune
addr = F_ADDR1;
if (debug_flag)
dest = T_ADDR1;
else
dest = T_ADDR2;
} else {
// mars
addr = F_ADDR2;
if (debug_flag)
dest = T_ADDR1;
else
dest = T_ADDR3;
}
printk("Reclaim bootloader memory from %x to %x\n", addr, dest);
count = 0;
while (addr < dest) {
page = virt_to_page(addr);
/*
printk("mem_map: %x, page: %x, size: %d \n", (int)mem_map, (int)page, sizeof(struct page));
if (PageReserved(page) != 1)
BUG();
if (page->_count.counter != -1)
BUG();
*/
count++;
__ClearPageReserved(page);
set_page_count(page, 1);
__free_page(page);
addr += 0x1000; // 4KB
}
totalram_pages += count;
#endif
return freed;
}
/*
* Turn a non-refcounted page (->_count == 0) into refcounted with
* a count of one.
*/
static inline void set_page_refcounted(struct page *page)
{
VM_BUG_ON(PageTail(page));
VM_BUG_ON(atomic_read(&page->_count));
set_page_count(page, 1);
}
void gnttab_reset_grant_page(struct page *page)
{
set_page_count(page, 1);
reset_page_mapcount(page);
}