/* * Initialise the sparsemem vmemmap using huge-pages at the PMD level. */ int __meminit vmemmap_populate(struct page *start_page, unsigned long size, int node) { unsigned long addr = (unsigned long)start_page; unsigned long end = (unsigned long)(start_page + size); unsigned long next; pgd_t *pgd; pud_t *pud; pmd_t *pmd; for (; addr < end; addr = next) { next = pmd_addr_end(addr, end); pgd = vmemmap_pgd_populate(addr, node); if (!pgd) return -ENOMEM; pud = vmemmap_pud_populate(pgd, addr, node); if (!pud) return -ENOMEM; pmd = pmd_offset(pud, addr); if (pmd_none(*pmd)) { pte_t entry; void *p = vmemmap_alloc_block(PMD_SIZE, node); if (!p) return -ENOMEM; entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); mk_pte_huge(entry); set_pmd(pmd, __pmd(pte_val(entry))); printk(KERN_DEBUG " [%lx-%lx] PMD ->%p on node %d\n", addr, addr + PMD_SIZE - 1, p, node); } else
static __meminit struct vmemmap_backing * vmemmap_list_alloc(int node) { struct vmemmap_backing *vmem_back; /* get from freed entries first */ if (num_freed) { num_freed--; vmem_back = next; next = next->list; return vmem_back; } /* allocate a page when required and hand out chunks */ if (!num_left) { next = vmemmap_alloc_block(PAGE_SIZE, node); if (unlikely(!next)) { WARN_ON(1); return NULL; } num_left = PAGE_SIZE / sizeof(struct vmemmap_backing); } num_left--; return next++; }
/* need to make sure size is all the same during early stage */ void * __meminit vmemmap_alloc_block_buf(unsigned long size, int node) { void *ptr; if (!vmemmap_buf) return vmemmap_alloc_block(size, node); /* take the from buf */ ptr = (void *)ALIGN((unsigned long)vmemmap_buf, size); if (ptr + size > vmemmap_buf_end) return vmemmap_alloc_block(size, node); vmemmap_buf = ptr + size; return ptr; }
pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node) { pte_t *pte = pte_offset_kernel(pmd, addr); if (pte_none(*pte)) { pte_t entry; void *p = vmemmap_alloc_block(PAGE_SIZE, node); if (!p) return NULL; entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); set_pte_at(&init_mm, addr, pte, entry); }
int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) { unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift; /* Align to the page size of the linear mapping. */ start = _ALIGN_DOWN(start, page_size); pr_debug("vmemmap_populate %lx..%lx, node %d\n", start, end, node); for (; start < end; start += page_size) { void *p; int rc; if (vmemmap_populated(start, page_size)) continue; p = vmemmap_alloc_block(page_size, node); if (!p) return -ENOMEM; vmemmap_list_populate(__pa(p), start, node); pr_debug(" * %016lx..%016lx allocated at %p\n", start, start + page_size, p); rc = vmemmap_create_mapping(start, page_size, __pa(p)); if (rc < 0) { pr_warning( "vmemmap_populate: Unable to create vmemmap mapping: %d\n", rc); return -EFAULT; } } return 0; }