static int __init_refok init_section_page_cgroup(unsigned long pfn, int nid) { struct page_cgroup *base, *pc; struct mem_section *section; unsigned long table_size; unsigned long nr; int index; nr = pfn_to_section_nr(pfn); section = __nr_to_section(nr); if (section->page_cgroup) return 0; table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION; VM_BUG_ON(!slab_is_available()); base = alloc_page_cgroup(table_size, nid); if (!base) { printk(KERN_ERR "page cgroup allocation failure\n"); return -ENOMEM; } for (index = 0; index < PAGES_PER_SECTION; index++) { pc = base + index; init_page_cgroup(pc, nr); } /* * The passed "pfn" may not be aligned to SECTION. For the calculation * we need to apply a mask. */ pfn &= PAGE_SECTION_MASK; section->page_cgroup = base - pfn; total_usage += table_size; return 0; }
static void * __ref alloc_p2m_page(void) { if (unlikely(!slab_is_available())) return alloc_bootmem_align(PAGE_SIZE, PAGE_SIZE); return (void *)__get_free_page(GFP_KERNEL | __GFP_REPEAT); }
/** * __alloc_bootmem_node - allocate boot memory from a specific node * @pgdat: node to allocate from * @size: size of the request in bytes * @align: alignment of the region * @goal: preferred starting address of the region * * The goal is dropped if it can not be satisfied and the allocation will * fall back to memory below @goal. * * Allocation may fall back to any node in the system if the specified node * can not hold the requested memory. * * The function panics if the request can not be satisfied. */ void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal) { BUG_ON(slab_is_available()); return ___alloc_bootmem_node(pgdat, size, align, goal, 0); }
int __ref msi_bitmap_alloc(struct msi_bitmap *bmp, unsigned int irq_count, struct device_node *of_node) { int size; if (!irq_count) return -EINVAL; size = BITS_TO_LONGS(irq_count) * sizeof(long); pr_debug("msi_bitmap: allocator bitmap size is 0x%x bytes\n", size); bmp->bitmap_from_slab = slab_is_available(); if (bmp->bitmap_from_slab) bmp->bitmap = kzalloc(size, GFP_KERNEL); else { bmp->bitmap = memblock_virt_alloc(size, 0); /* the bitmap won't be freed from memblock allocator */ kmemleak_not_leak(bmp->bitmap); } if (!bmp->bitmap) { pr_debug("msi_bitmap: ENOMEM allocating allocator bitmap!\n"); return -ENOMEM; } /* We zalloc'ed the bitmap, so all irqs are free by default */ spin_lock_init(&bmp->lock); bmp->of_node = of_node_get(of_node); bmp->irq_count = irq_count; return 0; }
/** * __alloc_bootmem_node - allocate boot memory from a specific node * @pgdat: node to allocate from * @size: size of the request in bytes * @align: alignment of the region * @goal: preferred starting address of the region * * The goal is dropped if it can not be satisfied and the allocation will * fall back to memory below @goal. * * Allocation may fall back to any node in the system if the specified node * can not hold the requested memory. * * The function panics if the request can not be satisfied. */ void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal) { if (WARN_ON_ONCE(slab_is_available())) return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); return ___alloc_bootmem_node(pgdat, size, align, goal, 0); }
void __mf_check(void *ptr, unsigned int sz, int type, const char *location) { if (!slab_is_available()) return; if (verify_ptr((unsigned long)ptr)) return; if (type) /* write */ slab_check_write(ptr, sz, location); }
static void __ref free_p2m_page(void *p) { if (unlikely(!slab_is_available())) { free_bootmem((unsigned long)p, PAGE_SIZE); return; } free_page((unsigned long)p); }
static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot, phys_addr_t (*pgtable_alloc)(void)) { pmd_t *pmd; unsigned long next; /* * Check for initial section mappings in the pgd/pud and remove them. */ if (pud_none(*pud) || pud_sect(*pud)) { phys_addr_t pmd_phys; BUG_ON(!pgtable_alloc); pmd_phys = pgtable_alloc(); pmd = pmd_set_fixmap(pmd_phys); if (pud_sect(*pud)) { /* * need to have the 1G of mappings continue to be * present */ split_pud(pud, pmd); } __pud_populate(pud, pmd_phys, PUD_TYPE_TABLE); flush_tlb_all(); pmd_clear_fixmap(); } BUG_ON(pud_bad(*pud)); pmd = pmd_set_fixmap_offset(pud, addr); do { next = pmd_addr_end(addr, end); /* try section mapping first */ if (((addr | next | phys) & ~SECTION_MASK) == 0 && block_mappings_allowed(pgtable_alloc)) { pmd_t old_pmd =*pmd; pmd_set_huge(pmd, phys, prot); /* * Check for previous table entries created during * boot (__create_page_tables) and flush them. */ if (!pmd_none(old_pmd)) { flush_tlb_all(); if (pmd_table(old_pmd)) { phys_addr_t table = pmd_page_paddr(old_pmd); if (!WARN_ON_ONCE(slab_is_available())) memblock_free(table, PAGE_SIZE); } } } else { alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys), prot, pgtable_alloc); } phys += next - addr; } while (pmd++, addr = next, addr != end); pmd_clear_fixmap(); }
void * __ref zalloc_maybe_bootmem(size_t size, gfp_t mask) { void *p; if (slab_is_available()) p = kzalloc(size, mask); else { p = memblock_virt_alloc(size, 0); } return p; }
static void alloc_init_pmd(struct mm_struct *mm, pud_t *pud, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot, void *(*alloc)(unsigned long size)) { pmd_t *pmd; unsigned long next; /* * Check for initial section mappings in the pgd/pud and remove them. */ if (pud_none(*pud) || pud_sect(*pud)) { pmd = alloc(PTRS_PER_PMD * sizeof(pmd_t)); if (pud_sect(*pud)) { /* * need to have the 1G of mappings continue to be * present */ split_pud(pud, pmd); } pud_populate(mm, pud, pmd); flush_tlb_all(); } BUG_ON(pud_bad(*pud)); pmd = pmd_offset(pud, addr); do { next = pmd_addr_end(addr, end); /* try section mapping first */ if (((addr | next | phys) & ~SECTION_MASK) == 0) { pmd_t old_pmd =*pmd; set_pmd(pmd, __pmd(phys | pgprot_val(mk_sect_prot(prot)))); /* * Check for previous table entries created during * boot (__create_page_tables) and flush them. */ if (!pmd_none(old_pmd)) { flush_tlb_all(); if (pmd_table(old_pmd)) { phys_addr_t table = __pa(pte_offset_map(&old_pmd, 0)); if (!WARN_ON_ONCE(slab_is_available())) memblock_free(table, PAGE_SIZE); } } } else { alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys), prot, alloc); } phys += next - addr; } while (pmd++, addr = next, addr != end); }
void * __meminit vmemmap_alloc_block(unsigned long size, int node) { /* If the main allocator is up use that, fallback to bootmem. */ if (slab_is_available()) { struct page *page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, get_order(size)); if (page) return page_address(page); return NULL; } else return __earlyonly_bootmem_alloc(node, size, size, __pa(MAX_DMA_ADDRESS)); }
__ref pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { pte_t *pte; if (slab_is_available()) { pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_ZERO); } else { pte = __va(memblock_alloc(PAGE_SIZE, PAGE_SIZE)); if (pte) clear_page(pte); } return pte; }
static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot, phys_addr_t (*pgtable_alloc)(void)) { pud_t *pud; unsigned long next; if (pgd_none(*pgd)) { phys_addr_t pud_phys; BUG_ON(!pgtable_alloc); pud_phys = pgtable_alloc(); __pgd_populate(pgd, pud_phys, PUD_TYPE_TABLE); } BUG_ON(pgd_bad(*pgd)); pud = pud_set_fixmap_offset(pgd, addr); do { next = pud_addr_end(addr, end); /* * For 4K granule only, attempt to put down a 1GB block */ if (use_1G_block(addr, next, phys) && block_mappings_allowed(pgtable_alloc)) { pud_t old_pud = *pud; pud_set_huge(pud, phys, prot); /* * If we have an old value for a pud, it will * be pointing to a pmd table that we no longer * need (from swapper_pg_dir). * * Look up the old pmd table and free it. */ if (!pud_none(old_pud)) { flush_tlb_all(); if (pud_table(old_pud)) { phys_addr_t table = pud_page_paddr(old_pud); if (!WARN_ON_ONCE(slab_is_available())) memblock_free(table, PAGE_SIZE); } } } else { alloc_init_pmd(pud, addr, next, phys, prot, pgtable_alloc); } phys += next - addr; } while (pud++, addr = next, addr != end); pud_clear_fixmap(); }
static __init_refok void *sh64_get_page(void) { void *page; if (slab_is_available()) page = (void *)get_zeroed_page(GFP_KERNEL); else page = alloc_bootmem_pages(PAGE_SIZE); if (!page || ((unsigned long)page & ~PAGE_MASK)) panic("sh64_get_page: Out of memory already?\n"); return page; }
static pte_t __ref *vmem_pte_alloc(void) { pte_t *pte; if (slab_is_available()) pte = (pte_t *) page_table_alloc(&init_mm); else pte = alloc_bootmem(PTRS_PER_PTE * sizeof(pte_t)); if (!pte) return NULL; clear_table((unsigned long *) pte, _PAGE_TYPE_EMPTY, PTRS_PER_PTE * sizeof(pte_t)); return pte; }
/* __alloc_bootmem...() is protected by !slab_available() */ int __init_refok init_section_page_cgroup(unsigned long pfn) { struct mem_section *section; struct page_cgroup *base, *pc; unsigned long table_size; int nid, index; section = __pfn_to_section(pfn); if (!section->page_cgroup) { nid = page_to_nid(pfn_to_page(pfn)); table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION; if (slab_is_available()) { base = kmalloc_node(table_size, GFP_KERNEL, nid); if (!base) base = vmalloc_node(table_size, nid); } else { base = __alloc_bootmem_node_nopanic(NODE_DATA(nid), table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); } } else { /* * We don't have to allocate page_cgroup again, but * address of memmap may be changed. So, we have to initialize * again. */ base = section->page_cgroup + pfn; table_size = 0; /* check address of memmap is changed or not. */ if (base->page == pfn_to_page(pfn)) return 0; } if (!base) { printk(KERN_ERR "page cgroup allocation failure\n"); return -ENOMEM; } for (index = 0; index < PAGES_PER_SECTION; index++) { pc = base + index; __init_page_cgroup(pc, pfn + index); } section = __pfn_to_section(pfn); section->page_cgroup = base - pfn; total_usage += table_size; return 0; }
/* * nid, region_start, and region_end are hints to try to place the page * table memory in the same node or region. */ static int __map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t flags, unsigned int map_page_size, int nid, unsigned long region_start, unsigned long region_end) { unsigned long pfn = pa >> PAGE_SHIFT; pgd_t *pgdp; pud_t *pudp; pmd_t *pmdp; pte_t *ptep; /* * Make sure task size is correct as per the max adddr */ BUILD_BUG_ON(TASK_SIZE_USER64 > RADIX_PGTABLE_RANGE); if (unlikely(!slab_is_available())) return early_map_kernel_page(ea, pa, flags, map_page_size, nid, region_start, region_end); /* * Should make page table allocation functions be able to take a * node, so we can place kernel page tables on the right nodes after * boot. */ pgdp = pgd_offset_k(ea); pudp = pud_alloc(&init_mm, pgdp, ea); if (!pudp) return -ENOMEM; if (map_page_size == PUD_SIZE) { ptep = (pte_t *)pudp; goto set_the_pte; } pmdp = pmd_alloc(&init_mm, pudp, ea); if (!pmdp) return -ENOMEM; if (map_page_size == PMD_SIZE) { ptep = pmdp_ptep(pmdp); goto set_the_pte; } ptep = pte_alloc_kernel(pmdp, ea); if (!ptep) return -ENOMEM; set_the_pte: set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags)); smp_wmb(); return 0; }
void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal) { void *ptr; if (WARN_ON_ONCE(slab_is_available())) return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); ptr = __alloc_memory_core_early(pgdat->node_id, size, align, goal, -1ULL); if (ptr) return ptr; return __alloc_bootmem_nopanic(size, align, goal); }
/* __alloc_bootmem...() is protected by !slab_available() */ static int __init_refok init_section_page_cgroup(unsigned long pfn) { struct mem_section *section = __pfn_to_section(pfn); struct page_cgroup *base, *pc; unsigned long table_size; int nid, index; if (!section->page_cgroup) { nid = page_to_nid(pfn_to_page(pfn)); table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION; VM_BUG_ON(!slab_is_available()); if (node_state(nid, N_HIGH_MEMORY)) { base = kmalloc_node(table_size, GFP_KERNEL | __GFP_NOWARN, nid); if (!base) base = vmalloc_node(table_size, nid); } else { base = kmalloc(table_size, GFP_KERNEL | __GFP_NOWARN); if (!base) base = vmalloc(table_size); } } else { /* * We don't have to allocate page_cgroup again, but * address of memmap may be changed. So, we have to initialize * again. */ base = section->page_cgroup + pfn; table_size = 0; /* check address of memmap is changed or not. */ if (base->page == pfn_to_page(pfn)) return 0; } if (!base) { printk(KERN_ERR "page cgroup allocation failure\n"); return -ENOMEM; } for (index = 0; index < PAGES_PER_SECTION; index++) { pc = base + index; __init_page_cgroup(pc, pfn + index); } section->page_cgroup = base - pfn; total_usage += table_size; return 0; }
int rtas_call(int token, int nargs, int nret, int *outputs, ...) { va_list list; int i; unsigned long s; struct rtas_args *rtas_args; char *buff_copy = NULL; int ret; if (!rtas.entry || token == RTAS_UNKNOWN_SERVICE) return -1; s = lock_rtas(); rtas_args = &rtas.args; rtas_args->token = cpu_to_be32(token); rtas_args->nargs = cpu_to_be32(nargs); rtas_args->nret = cpu_to_be32(nret); rtas_args->rets = &(rtas_args->args[nargs]); va_start(list, outputs); for (i = 0; i < nargs; ++i) rtas_args->args[i] = cpu_to_be32(va_arg(list, __u32)); va_end(list); for (i = 0; i < nret; ++i) rtas_args->rets[i] = 0; enter_rtas(__pa(rtas_args)); /* A -1 return code indicates that the last command couldn't be completed due to a hardware error. */ if (be32_to_cpu(rtas_args->rets[0]) == -1) buff_copy = __fetch_rtas_last_error(NULL); if (nret > 1 && outputs != NULL) for (i = 0; i < nret-1; ++i) outputs[i] = be32_to_cpu(rtas_args->rets[i+1]); ret = (nret > 0)? be32_to_cpu(rtas_args->rets[0]): 0; unlock_rtas(s); if (buff_copy) { log_error(buff_copy, ERR_TYPE_RTAS_LOG, 0); if (slab_is_available()) kfree(buff_copy); } return ret; }
/** * __alloc_bootmem_low_node - allocate low boot memory from a specific node * @pgdat: node to allocate from * @size: size of the request in bytes * @align: alignment of the region * @goal: preferred starting address of the region * * The goal is dropped if it can not be satisfied and the allocation will * fall back to memory below @goal. * * Allocation may fall back to any node in the system if the specified node * can not hold the requested memory. * * The function panics if the request can not be satisfied. */ void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal) { void *ptr; if (WARN_ON_ONCE(slab_is_available())) return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); ptr = __alloc_memory_core_early(pgdat->node_id, size, align, goal, ARCH_LOW_ADDRESS_LIMIT); if (ptr) return ptr; return __alloc_memory_core_early(MAX_NUMNODES, size, align, goal, ARCH_LOW_ADDRESS_LIMIT); }
static struct mem_section *sparse_index_alloc(int nid) { struct mem_section *section = NULL; unsigned long array_size = SECTIONS_PER_ROOT * sizeof(struct mem_section); if (slab_is_available()) section = kmalloc_node(array_size, GFP_KERNEL, nid); else section = alloc_bootmem_node(NODE_DATA(nid), array_size); if (section) memset(section, 0, array_size); return section; }
/* * Unmap an IO region and remove it from imalloc'd list. * Access to IO memory should be serialized by driver. */ void __iounmap(volatile void __iomem *token) { void *addr; if (!slab_is_available()) return; addr = (void *) ((unsigned long __force) PCI_FIX_ADDR(token) & PAGE_MASK); if ((unsigned long)addr < ioremap_bot) { printk(KERN_WARNING "Attempt to iounmap early bolted mapping" " at 0x%p\n", addr); return; } vunmap(addr); }
static void alloc_init_pud(struct mm_struct *mm, pgd_t *pgd, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot, void *(*alloc)(unsigned long size)) { pud_t *pud; unsigned long next; if (pgd_none(*pgd)) { pud = alloc(PTRS_PER_PUD * sizeof(pud_t)); pgd_populate(mm, pgd, pud); } BUG_ON(pgd_bad(*pgd)); pud = pud_offset(pgd, addr); do { next = pud_addr_end(addr, end); /* * For 4K granule only, attempt to put down a 1GB block */ if (use_1G_block(addr, next, phys)) { pud_t old_pud = *pud; set_pud(pud, __pud(phys | pgprot_val(mk_sect_prot(prot)))); /* * If we have an old value for a pud, it will * be pointing to a pmd table that we no longer * need (from swapper_pg_dir). * * Look up the old pmd table and free it. */ if (!pud_none(old_pud)) { flush_tlb_all(); if (pud_table(old_pud)) { phys_addr_t table = __pa(pmd_offset(&old_pud, 0)); if (!WARN_ON_ONCE(slab_is_available())) memblock_free(table, PAGE_SIZE); } } } else { alloc_init_pmd(mm, pud, addr, next, phys, prot, alloc); } phys += next - addr; } while (pud++, addr = next, addr != end); }
/* 특정 node에 bootmem을 할당. 메모리 할당을 못하면 panic 발생. * goal은 시작 주소의 목표이지만, 할당이 제대로 안된 경우는 무시될 * 수 있다. */ void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal) { void *ptr; if (WARN_ON_ONCE(slab_is_available())) return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); ptr = __alloc_memory_core_early(pgdat->node_id, size, align, goal, -1ULL); if (ptr) return ptr; /* 특정 node 에서 할당이 실패한 경우, MAX_NUMNODES에 할당 */ return __alloc_memory_core_early(MAX_NUMNODES, size, align, goal, -1ULL); }
/* * map_kernel_page currently only called by __ioremap * map_kernel_page adds an entry to the ioremap page table * and adds an entry to the HPT, possibly bolting it */ int __ref map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot) { pgd_t *pgdp; pud_t *pudp; pmd_t *pmdp; pte_t *ptep; BUILD_BUG_ON(TASK_SIZE_USER64 > PGTABLE_RANGE); if (slab_is_available()) { pgdp = pgd_offset_k(ea); pudp = pud_alloc(&init_mm, pgdp, ea); if (!pudp) return -ENOMEM; pmdp = pmd_alloc(&init_mm, pudp, ea); if (!pmdp) return -ENOMEM; ptep = pte_alloc_kernel(pmdp, ea); if (!ptep) return -ENOMEM; } else { pgdp = pgd_offset_k(ea); #ifndef __PAGETABLE_PUD_FOLDED if (pgd_none(*pgdp)) { pudp = early_alloc_pgtable(PUD_TABLE_SIZE); pgd_populate(&init_mm, pgdp, pudp); } #endif /* !__PAGETABLE_PUD_FOLDED */ pudp = pud_offset(pgdp, ea); if (pud_none(*pudp)) { pmdp = early_alloc_pgtable(PMD_TABLE_SIZE); pud_populate(&init_mm, pudp, pmdp); } pmdp = pmd_offset(pudp, ea); if (!pmd_present(*pmdp)) { ptep = early_alloc_pgtable(PAGE_SIZE); pmd_populate_kernel(&init_mm, pmdp, ptep); } ptep = pte_offset_kernel(pmdp, ea); } set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, prot)); smp_wmb(); return 0; }
void __iomem * __ioremap_caller(phys_addr_t addr, unsigned long size, unsigned long flags, void *caller) { phys_addr_t paligned; void __iomem *ret; /* * Choose an address to map it to. * Once the imalloc system is running, we use it. * Before that, we map using addresses going * up from ioremap_bot. imalloc will use * the addresses from ioremap_bot through * IMALLOC_END * */ paligned = addr & PAGE_MASK; size = PAGE_ALIGN(addr + size) - paligned; if ((size == 0) || (paligned == 0)) return NULL; if (slab_is_available()) { struct vm_struct *area; area = __get_vm_area_caller(size, VM_IOREMAP, ioremap_bot, IOREMAP_END, caller); if (area == NULL) return NULL; area->phys_addr = paligned; ret = __ioremap_at(paligned, area->addr, size, flags); if (!ret) vunmap(area->addr); } else { ret = __ioremap_at(paligned, (void *)ioremap_bot, size, flags); if (ret) ioremap_bot += size; } if (ret) ret += addr & ~PAGE_MASK; return ret; }
void * __meminit vmemmap_alloc_block(unsigned long size, int node) { if (slab_is_available()) { struct page *page; if (node_state(node, N_HIGH_MEMORY)) page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, get_order(size)); else page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(size)); if (page) return page_address(page); return NULL; } else return __earlyonly_bootmem_alloc(node, size, size, __pa(MAX_DMA_ADDRESS)); }
void __ref init_kstat_irqs(struct irq_desc *desc, int node, int nr) { void *ptr; if (slab_is_available()) ptr = kzalloc_node(nr * sizeof(*desc->kstat_irqs), GFP_ATOMIC, node); else ptr = alloc_bootmem_node(NODE_DATA(node), nr * sizeof(*desc->kstat_irqs)); /* * don't overwite if can not get new one * init_copy_kstat_irqs() could still use old one */ if (ptr) { printk(KERN_DEBUG " alloc kstat_irqs on node %d\n", node); desc->kstat_irqs = ptr; } }
void __init reserve_real_mode(void) { phys_addr_t mem; size_t size = real_mode_size_needed(); if (!size) return; WARN_ON(slab_is_available()); /* Has to be under 1M so we can execute real-mode AP code. */ mem = memblock_find_in_range(0, 1<<20, size, PAGE_SIZE); if (!mem) { pr_info("No sub-1M memory is available for the trampoline\n"); return; } memblock_reserve(mem, size); set_real_mode_mem(mem, size); }