/* IAMROOT-12AB: * ------------- * sparse 메모리 모델이 아닌 경우 page로 node_page_ext를 * 찾은 후 page_ext를 찾아 반환한다. */ struct page_ext *lookup_page_ext(struct page *page) { unsigned long pfn = page_to_pfn(page); unsigned long offset; struct page_ext *base; base = NODE_DATA(page_to_nid(page))->node_page_ext; #ifdef CONFIG_DEBUG_VM /* * The sanity checks the page allocator does upon freeing a * page can reach here before the page_ext arrays are * allocated when feeding a range of pages to the allocator * for the first time during bootup or memory hotplug. */ if (unlikely(!base)) return NULL; #endif /* IAMROOT-12AB: * ------------- * offset은 mem_map에서 구한 후 함수에서 반환 시에 base(page_ext *)에 * offset을 더해 반환하여 page_ext 포인터 주소를 알려준다. */ offset = pfn - round_down(node_start_pfn(page_to_nid(page)), MAX_ORDER_NR_PAGES); return base + offset; }
struct page_cgroup *lookup_page_cgroup(struct page *page) { unsigned long pfn = page_to_pfn(page); unsigned long offset; struct page_cgroup *base; base = NODE_DATA(page_to_nid(page))->node_page_cgroup; if (unlikely(!base)) return NULL; offset = pfn - NODE_DATA(page_to_nid(page))->node_start_pfn; return base + offset; }
static void count_pmd_pages(struct mm_struct * mm, struct vm_area_struct * vma, pmd_t *dir, unsigned long address, unsigned long end, signed char *data_buf, int node_map) { pte_t * pte; unsigned long pmd_end; struct page *page; unsigned long pfn; int val, index; if (pmd_none(*dir)) return; pmd_end = (address + PMD_SIZE) & PMD_MASK; if (end > pmd_end) end = pmd_end; index = 0; do { pte = pte_offset_map(dir, address); if (!pte_none(pte) && pte_present(*pte)) { pfn = pte_pfn(*pte); if (pfn_valid(pfn)) { page = pfn_to_page(pfn); val = node_map ? page_to_nid(page) : page_count(page); val = (val > 99) ? 99 : val; data_buf[index] = val; } } address += PAGE_SIZE; pte++; index++; } while (address && (address < end)); return; }
static void enqueue_huge_page(struct page *page) { int nid = page_to_nid(page); list_add(&page->lru, &hugepage_freelists[nid]); free_huge_pages++; free_huge_pages_node[nid]++; }
/* Ensure all existing pages follow the policy. */ static int verify_pages(unsigned long addr, unsigned long end, unsigned long *nodes) { while (addr < end) { struct page *p; pte_t *pte; pmd_t *pmd; pgd_t *pgd = pgd_offset_k(addr); if (pgd_none(*pgd)) { addr = (addr + PGDIR_SIZE) & PGDIR_MASK; continue; } pmd = pmd_offset(pgd, addr); if (pmd_none(*pmd)) { addr = (addr + PMD_SIZE) & PMD_MASK; continue; } p = NULL; pte = pte_offset_map(pmd, addr); if (pte_present(*pte)) p = pte_page(*pte); pte_unmap(pte); if (p) { unsigned nid = page_to_nid(p); if (!test_bit(nid, nodes)) return -EIO; } addr += PAGE_SIZE; } return 0; }
struct page_ext *lookup_page_ext(const struct page *page) { unsigned long pfn = page_to_pfn(page); unsigned long index; struct page_ext *base; base = NODE_DATA(page_to_nid(page))->node_page_ext; /* * The sanity checks the page allocator does upon freeing a * page can reach here before the page_ext arrays are * allocated when feeding a range of pages to the allocator * for the first time during bootup or memory hotplug. */ if (unlikely(!base)) return NULL; index = pfn - round_down(node_start_pfn(page_to_nid(page)), MAX_ORDER_NR_PAGES); return get_entry(base, index); }
struct page_cgroup *lookup_page_cgroup(struct page *page) { unsigned long pfn = page_to_pfn(page); unsigned long offset; struct page_cgroup *base; base = NODE_DATA(page_to_nid(page))->node_page_cgroup; #ifdef CONFIG_DEBUG_VM /* * The sanity checks the page allocator does upon freeing a * page can reach here before the page_cgroup arrays are * allocated when feeding a range of pages to the allocator * for the first time during bootup or memory hotplug. */ if (unlikely(!base)) return NULL; #endif offset = pfn - NODE_DATA(page_to_nid(page))->node_start_pfn; return base + offset; }
/* * slob_alloc: entry point into the slob allocator. * * We modified the slob_alloc function. We defined new local variables: * 1.) temp_amt_free will accumulate all the free bytes on each page * (used for the system call sys_get_slob_amt_free); * 2.) best_sp points to the page with the "best fit"; * 3.) best_fit/ current_fit contain a number in which the smaller * the number, the better the fit; best_fit is the overall best number * and current_fit is the current page's number * * When iterating through the free page list, our slob_alloc will: * 1.) collect the free units of the page and store them into * temp_amt_free; * 2.) call a helper function slob_page_best_fit_check which * returns a number put into current_fit. * 3.) This current_fit number is checked against a number of * cases. First case is if current_fit is equal to 0. This means, * a perfect fit, so we break out of the loop and allocate there. * Second case is -1, meaning there is no block fit so we continue * the loop. Last case is some positive number so we check to see if * this number is less than the best_fit. Or we check to see if * best_fit has been set yet. If this case is reached, then the new * best page is the current page in the loop. * * Once the loop is over, we try to allocate on that page if best_fit is * some positive number. Otherwise, we don't have enough space and must * allocate a new page. This part of the slob_alloc function was mostly * unchanged except for updating the values of the array lists for the * system calls. When we have to allocate a new page, is when we set the * amt_claimed, at position of the counter, to the size of the request (in * bytes). We also set the amt_free, at the position of the counter, to * the total accumulated units from the list, converting it to bytes. Then * we increment the counter. * */ static void *slob_alloc(size_t size, gfp_t gfp, int align, int node) { struct slob_page *sp; struct list_head *prev; struct list_head *slob_list; slob_t *b = NULL; unsigned long flags; // Lab 3 - Statistics long temp_amt_free = 0; struct slob_page *best_sp = NULL; int best_fit = -1; if (size < SLOB_BREAK1) slob_list = &free_slob_small; else if (size < SLOB_BREAK2) slob_list = &free_slob_medium; else slob_list = &free_slob_large; spin_lock_irqsave(&slob_lock, flags); /* Iterate through each partially free page, try to find room */ list_for_each_entry(sp, slob_list, list) { int current_fit = -1; // Lab 3 - Statistics temp_amt_free = temp_amt_free + sp->units; #ifdef CONFIG_NUMA /* * If there's a node specification, search for a partial * page with a matching node id in the freelist. */ if (node != -1 && page_to_nid(&sp->page) != node) continue; #endif /* Enough room on this page? */ if (sp->units < SLOB_UNITS(size)) continue; #ifdef SLOB_BEST_FIT_ALG current_fit = slob_page_best_fit_check(sp, size, align); if(current_fit == 0) { best_sp = sp; best_fit = current_fit; break; } else if(current_fit > 0 && (best_fit == -1 || current_fit < best_fit) ) { best_sp = sp; best_fit = current_fit; } continue; }
/* __alloc_bootmem...() is protected by !slab_available() */ int __init_refok init_section_page_cgroup(unsigned long pfn) { struct mem_section *section; struct page_cgroup *base, *pc; unsigned long table_size; int nid, index; section = __pfn_to_section(pfn); if (!section->page_cgroup) { nid = page_to_nid(pfn_to_page(pfn)); table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION; if (slab_is_available()) { base = kmalloc_node(table_size, GFP_KERNEL, nid); if (!base) base = vmalloc_node(table_size, nid); } else { base = __alloc_bootmem_node_nopanic(NODE_DATA(nid), table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); } } else { /* * We don't have to allocate page_cgroup again, but * address of memmap may be changed. So, we have to initialize * again. */ base = section->page_cgroup + pfn; table_size = 0; /* check address of memmap is changed or not. */ if (base->page == pfn_to_page(pfn)) return 0; } if (!base) { printk(KERN_ERR "page cgroup allocation failure\n"); return -ENOMEM; } for (index = 0; index < PAGES_PER_SECTION; index++) { pc = base + index; __init_page_cgroup(pc, pfn + index); } section = __pfn_to_section(pfn); section->page_cgroup = base - pfn; total_usage += table_size; return 0; }
struct page_ext *lookup_page_ext(struct page *page) { unsigned long pfn = page_to_pfn(page); unsigned long index; struct page_ext *base; base = NODE_DATA(page_to_nid(page))->node_page_ext; #if defined(CONFIG_DEBUG_VM) || defined(CONFIG_PAGE_POISONING) /* * The sanity checks the page allocator does upon freeing a * page can reach here before the page_ext arrays are * allocated when feeding a range of pages to the allocator * for the first time during bootup or memory hotplug. * * This check is also necessary for ensuring page poisoning * works as expected when enabled */ if (unlikely(!base)) return NULL; #endif index = pfn - round_down(node_start_pfn(page_to_nid(page)), MAX_ORDER_NR_PAGES); return get_entry(base, index); }
/* __alloc_bootmem...() is protected by !slab_available() */ static int __init_refok init_section_page_cgroup(unsigned long pfn) { struct mem_section *section = __pfn_to_section(pfn); struct page_cgroup *base, *pc; unsigned long table_size; int nid, index; if (!section->page_cgroup) { nid = page_to_nid(pfn_to_page(pfn)); table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION; VM_BUG_ON(!slab_is_available()); if (node_state(nid, N_HIGH_MEMORY)) { base = kmalloc_node(table_size, GFP_KERNEL | __GFP_NOWARN, nid); if (!base) base = vmalloc_node(table_size, nid); } else { base = kmalloc(table_size, GFP_KERNEL | __GFP_NOWARN); if (!base) base = vmalloc(table_size); } } else { /* * We don't have to allocate page_cgroup again, but * address of memmap may be changed. So, we have to initialize * again. */ base = section->page_cgroup + pfn; table_size = 0; /* check address of memmap is changed or not. */ if (base->page == pfn_to_page(pfn)) return 0; } if (!base) { printk(KERN_ERR "page cgroup allocation failure\n"); return -ENOMEM; } for (index = 0; index < PAGES_PER_SECTION; index++) { pc = base + index; __init_page_cgroup(pc, pfn + index); } section->page_cgroup = base - pfn; total_usage += table_size; return 0; }
struct page *alloc_migrate_target(struct page *page, unsigned long nid, int **resultp) { /* * hugeTLB: allocate a destination page from a nearest neighbor node, * accordance with memory policy of the user process if possible. For * now as a simple work-around, we use the next node for destination. * Normal page: use prefer mempolicy for destination if called by * hotplug, use default mempolicy for destination if called by cma. */ if (PageHuge(page)) return alloc_huge_page_node(page_hstate(compound_head(page)), next_node_in(page_to_nid(page), node_online_map)); else return alloc_pages_node(nid, GFP_HIGHUSER_MOVABLE, 0); }
/* * slob_alloc: entry point into the slob allocator. */ static void *slob_alloc(size_t size, gfp_t gfp, int align, int node) { struct slob_page *sp; struct list_head *slob_list; slob_t *b = NULL; unsigned long flags; int i; unsigned long int count_free; #ifdef BESTFIT_PAGE slobidx_t min,prev_min; struct slob_page *curr; int flag_if; int check; #else struct list_head *prev; #endif /*arxikopoioyme ta total_alloc kai ta total_free*/ if(flag_mem==0){ total_alloc = 0; total_free = 0; flag_mem = 1; } if (size < SLOB_BREAK1) slob_list = &free_slob_small; else if (size < SLOB_BREAK2) slob_list = &free_slob_medium; else slob_list = &free_slob_large; spin_lock_irqsave(&slob_lock, flags); /* Iterate through each partially free page, try to find room */ counter_print++; #ifdef BESTFIT_PAGE flag_if = 0; curr = NULL; min = 0; prev_min = SLOB_UNITS(size); /*diatexoume th lista mexri na broyme thn kalyterh selida*/ list_for_each_entry(sp, slob_list, list) { #ifdef CONFIG_NUMA /* * If there's a node specification, search for a partial * page with a matching node id in the freelist. */ if (node != -1 && page_to_nid(&sp->page) != node) continue; #endif /* Enough room on this page? */ if (sp->units < prev_min){ continue; } if(flag_if==0){ check = check_block(sp,size,align); if(check){ min = sp->units; curr = sp; flag_if = 1; } } else{ if(sp->units <= min){ check = check_block(sp,size,align); if(check){ min = sp->units; curr = sp; } } } } //kaloyme thn slob_page_alloc if(curr!=NULL){ b = slob_page_alloc(curr, size, align); } else{ b = NULL; } #else list_for_each_entry(sp, slob_list, list) { #ifdef CONFIG_NUMA /* * If there's a node specification, search for a partial * page with a matching node id in the freelist. */ if (node != -1 && page_to_nid(&sp->page) != node) continue; #endif /* Enough room on this page? */ if (sp->units < SLOB_UNITS(size)) continue; /* Attempt to alloc */ prev = sp->list.prev; b = slob_page_alloc(sp, size, align); if (!b) continue; /* Improve fragment distribution and reduce our average * search time by starting our next search here. (see * Knuth vol 1, sec 2.5, pg 449) */ if (prev != slob_list->prev && slob_list->next != prev->next) list_move_tail(slob_list, prev->next); break; } #endif spin_unlock_irqrestore(&slob_lock, flags); /* Not enough space: must allocate a new page */ if (!b) { b = slob_new_pages(gfp & ~__GFP_ZERO, 0, node); if (!b) return NULL; sp = slob_page(b); set_slob_page(sp); spin_lock_irqsave(&slob_lock, flags); sp->units = SLOB_UNITS(PAGE_SIZE); sp->free = b; INIT_LIST_HEAD(&sp->list); set_slob(b, SLOB_UNITS(PAGE_SIZE), b + SLOB_UNITS(PAGE_SIZE)); set_slob_page_free(sp, slob_list); b = slob_page_alloc(sp, size, align); BUG_ON(!b); spin_unlock_irqrestore(&slob_lock, flags); } if (unlikely((gfp & __GFP_ZERO) && b)) memset(b, 0, size); /*diatrexoyme kai tis treis listes megethon gia na metrhsoyme ta synolika free olwn twn selidwn*/ count_free = 0; for(i=0;i<3;i++){ if (i==0){ slob_list = &free_slob_small; } else if (i==1){ slob_list = &free_slob_medium; } else{ slob_list = &free_slob_large; } list_for_each_entry(sp, slob_list, list) { count_free = count_free + sp->units; } }
static void *slob_alloc(size_t size, gfp_t gfp, int align, int node) { struct page *sp; struct list_head *prev; struct list_head *slob_list; slob_t *b = NULL; unsigned long flags; if (size < SLOB_BREAK1) slob_list = &free_slob_small; else if (size < SLOB_BREAK2) slob_list = &free_slob_medium; else slob_list = &free_slob_large; spin_lock_irqsave(&slob_lock, flags); list_for_each_entry(sp, slob_list, list) { #ifdef CONFIG_NUMA /* * If there's a node specification, search for a partial * page with a matching node id in the freelist. */ if (node != NUMA_NO_NODE && page_to_nid(sp) != node) continue; #endif if (sp->units < SLOB_UNITS(size)) continue; prev = sp->list.prev; b = slob_page_alloc(sp, size, align); if (!b) continue; if (prev != slob_list->prev && slob_list->next != prev->next) list_move_tail(slob_list, prev->next); break; } spin_unlock_irqrestore(&slob_lock, flags); if (!b) { b = slob_new_pages(gfp & ~__GFP_ZERO, 0, node); if (!b) return NULL; sp = slob_page(b); set_slob_page(sp); spin_lock_irqsave(&slob_lock, flags); sp->units = SLOB_UNITS(PAGE_SIZE); sp->freelist = b; INIT_LIST_HEAD(&sp->list); set_slob(b, SLOB_UNITS(PAGE_SIZE), b + SLOB_UNITS(PAGE_SIZE)); set_slob_page_free(sp, slob_list); b = slob_page_alloc(sp, size, align); BUG_ON(!b); spin_unlock_irqrestore(&slob_lock, flags); } if (unlikely((gfp & __GFP_ZERO) && b)) memset(b, 0, size); return b; }
int online_pages(unsigned long pfn, unsigned long nr_pages) { unsigned long flags; unsigned long onlined_pages = 0; struct zone *zone; int need_zonelists_rebuild = 0; int nid; int ret; struct memory_notify arg; arg.start_pfn = pfn; arg.nr_pages = nr_pages; arg.status_change_nid = -1; nid = page_to_nid(pfn_to_page(pfn)); if (node_present_pages(nid) == 0) arg.status_change_nid = nid; ret = memory_notify(MEM_GOING_ONLINE, &arg); ret = notifier_to_errno(ret); if (ret) { memory_notify(MEM_CANCEL_ONLINE, &arg); return ret; } /* * This doesn't need a lock to do pfn_to_page(). * The section can't be removed here because of the * memory_block->state_sem. */ zone = page_zone(pfn_to_page(pfn)); pgdat_resize_lock(zone->zone_pgdat, &flags); grow_zone_span(zone, pfn, pfn + nr_pages); grow_pgdat_span(zone->zone_pgdat, pfn, pfn + nr_pages); pgdat_resize_unlock(zone->zone_pgdat, &flags); /* * If this zone is not populated, then it is not in zonelist. * This means the page allocator ignores this zone. * So, zonelist must be updated after online. */ if (!populated_zone(zone)) need_zonelists_rebuild = 1; walk_memory_resource(pfn, nr_pages, &onlined_pages, online_pages_range); zone->present_pages += onlined_pages; zone->zone_pgdat->node_present_pages += onlined_pages; setup_per_zone_pages_min(); if (onlined_pages) { kswapd_run(zone_to_nid(zone)); node_set_state(zone_to_nid(zone), N_HIGH_MEMORY); } if (need_zonelists_rebuild) build_all_zonelists(); vm_total_pages = nr_free_pagecache_pages(); writeback_set_ratelimit(); if (onlined_pages) memory_notify(MEM_ONLINE, &arg); return 0; }
/* * slob_alloc: entry point into the slob allocator. */ static void *slob_alloc(size_t size, gfp_t gfp, int align, int node) { struct slob_page *sp = NULL; struct slob_page *sp_t; struct list_head *slob_list; slob_t *b = NULL; unsigned long flags; if (size < SLOB_BREAK1) slob_list = &free_slob_small; else if (size < SLOB_BREAK2) slob_list = &free_slob_medium; else slob_list = &free_slob_large; spin_lock_irqsave(&slob_lock, flags); /* Iterate through each partially free page, try to find room */ list_for_each_entry(sp_t, slob_list, list) { #ifdef CONFIG_NUMA /* * If there's a node specification, search for a partial * page with a matching node id in the freelist. */ if (node != -1 && page_to_nid(&sp_t->page) != node) continue; #endif /* Enough room on this page? */ if (sp_t->units < SLOB_UNITS(size)) /* Not enough room */ continue; if (sp == NULL) sp = sp_t; if (sp_t->units < sp->units) /* Get the smallest slob_page that * is large enough for our needs */ sp = sp_t; } /* Attempt to alloc */ if(sp != NULL) { b = slob_page_alloc(sp, size, align); } spin_unlock_irqrestore(&slob_lock, flags); /* Not enough space: must allocate a new page */ if (!b) { b = slob_new_pages(gfp & ~__GFP_ZERO, 0, node); if (!b) return NULL; sp = slob_page(b); set_slob_page(sp); /* We allocatted a new page, increment the count */ slob_page_count++; spin_lock_irqsave(&slob_lock, flags); sp->units = SLOB_UNITS(PAGE_SIZE); sp->free = b; INIT_LIST_HEAD(&sp->list); set_slob(b, SLOB_UNITS(PAGE_SIZE), b + SLOB_UNITS(PAGE_SIZE)); set_slob_page_free(sp, slob_list); b = slob_page_alloc(sp, size, align); BUG_ON(!b); spin_unlock_irqrestore(&slob_lock, flags); } if (unlikely((gfp & __GFP_ZERO) && b)) memset(b, 0, size); return b; }
/* * slob_alloc: entry point into the slob allocator. */ static void *slob_alloc(size_t size, gfp_t gfp, int align, int node) { struct page *sp; struct list_head *prev; struct list_head *slob_list; slob_t *b = NULL; unsigned long flags; if (size < SLOB_BREAK1) slob_list = &free_slob_small; else if (size < SLOB_BREAK2) slob_list = &free_slob_medium; else slob_list = &free_slob_large; spin_lock_irqsave(&slob_lock, flags); /* Iterate through each partially free page, try to find room */ list_for_each_entry(sp, slob_list, lru) { #ifdef CONFIG_NUMA /* * If there's a node specification, search for a partial * page with a matching node id in the freelist. */ if (node != NUMA_NO_NODE && page_to_nid(sp) != node) continue; #endif /* Enough room on this page? */ if (sp->units < SLOB_UNITS(size)) continue; /* Attempt to alloc */ prev = sp->lru.prev; b = slob_page_alloc(sp, size, align); if (!b) continue; /* Improve fragment distribution and reduce our average * search time by starting our next search here. (see * Knuth vol 1, sec 2.5, pg 449) */ if (prev != slob_list->prev && slob_list->next != prev->next) list_move_tail(slob_list, prev->next); break; } spin_unlock_irqrestore(&slob_lock, flags); /* Not enough space: must allocate a new page */ if (!b) { b = slob_new_pages(gfp & ~__GFP_ZERO, 0, node); if (!b) return NULL; sp = virt_to_page(b); __SetPageSlab(sp); spin_lock_irqsave(&slob_lock, flags); sp->units = SLOB_UNITS(PAGE_SIZE); sp->freelist = b; INIT_LIST_HEAD(&sp->lru); set_slob(b, SLOB_UNITS(PAGE_SIZE), b + SLOB_UNITS(PAGE_SIZE)); set_slob_page_free(sp, slob_list); b = slob_page_alloc(sp, size, align); BUG_ON(!b); spin_unlock_irqrestore(&slob_lock, flags); } if (unlikely((gfp & __GFP_ZERO) && b)) memset(b, 0, size); return b; }