static int sel_mmap_handle_status(struct file *filp,
struct vm_area_struct *vma)
{
struct page *status = filp->private_data;
unsigned long size = vma->vm_end - vma->vm_start;
BUG_ON(!status);
/* only allows one page from the head */
if (vma->vm_pgoff > 0 || size != PAGE_SIZE)
return -EIO;
/* disallow writable mapping */
if (vma->vm_flags & VM_WRITE)
return -EPERM;
/* disallow mprotect() turns it into writable */
vma->vm_flags &= ~VM_MAYWRITE;
return remap_pfn_range(vma, vma->vm_start,
page_to_pfn(status),
size, vma->vm_page_prot);
}
struct page_ext *lookup_page_ext(struct page *page)
{
unsigned long pfn = page_to_pfn(page);
unsigned long offset;
struct page_ext *base;
base = NODE_DATA(page_to_nid(page))->node_page_ext;
#ifdef CONFIG_DEBUG_VM
/*
* The sanity checks the page allocator does upon freeing a
* page can reach here before the page_ext arrays are
* allocated when feeding a range of pages to the allocator
* for the first time during bootup or memory hotplug.
*/
if (unlikely(!base))
return NULL;
#endif
offset = pfn - round_down(node_start_pfn(page_to_nid(page)),
MAX_ORDER_NR_PAGES);
return base + offset;
}
/*
* PVH: we need three things: virtual address, pfns, and mfns. The pfns
* are allocated via ballooning, then we call arch_gnttab_map_shared to
* allocate the VA and put pfn's in the pte's for the VA. The mfn's are
* finally allocated in gnttab_map() by xen which also populates the P2M.
*/
static int xlated_setup_gnttab_pages(unsigned long numpages, void **addr)
{
int i, rc;
unsigned long pfns[numpages];
struct page *pages[numpages];
rc = alloc_xenballooned_pages(numpages, pages, 0);
if (rc != 0) {
pr_warn("%s Couldn't balloon alloc %ld pfns rc:%d\n", __func__,
numpages, rc);
return rc;
}
for (i = 0; i < numpages; i++)
pfns[i] = page_to_pfn(pages[i]);
rc = arch_gnttab_map_shared(pfns, numpages, numpages, addr);
if (rc != 0)
free_xenballooned_pages(numpages, pages);
return rc;
}
static int sel_mmap_handle_status(struct file *filp,
struct vm_area_struct *vma)
{
struct page *status = filp->private_data;
unsigned long size = vma->vm_end - vma->vm_start;
BUG_ON(!status);
if (vma->vm_pgoff > 0 || size != PAGE_SIZE)
return -EIO;
if (vma->vm_flags & VM_WRITE)
return -EPERM;
vma->vm_flags &= ~VM_MAYWRITE;
return remap_pfn_range(vma, vma->vm_start,
page_to_pfn(status),
size, vma->vm_page_prot);
}
static struct page *__kmalloc_section_memmap(unsigned long nr_pages)
{
struct page *page, *ret;
unsigned long memmap_size = sizeof(struct page) * nr_pages;
page = alloc_pages(GFP_KERNEL|__GFP_NOWARN, get_order(memmap_size));
if (page)
goto got_map_page;
ret = vmalloc(memmap_size);
if (ret)
goto got_map_ptr;
return NULL;
got_map_page:
ret = (struct page *)pfn_to_kaddr(page_to_pfn(page));
got_map_ptr:
memset(ret, 0, memmap_size);
return ret;
}
void machine_kexec(struct kimage *image)
{
unsigned long page_list;
unsigned long reboot_code_buffer_phys;
void *reboot_code_buffer;
page_list = image->head & PAGE_MASK;
/* we need both effective and real address here */
reboot_code_buffer_phys =
page_to_pfn(image->control_code_page) << PAGE_SHIFT;
reboot_code_buffer = page_address(image->control_code_page);
/* Prepare parameters for reboot_code_buffer*/
kexec_start_address = image->start;
kexec_indirection_page = page_list;
kexec_mach_type = machine_arch_type;
kexec_boot_atags = image->start - KEXEC_ARM_ZIMAGE_OFFSET + KEXEC_ARM_ATAGS_OFFSET;
/* copy our kernel relocation code to the control code page */
memcpy(reboot_code_buffer,
relocate_new_kernel, relocate_new_kernel_size);
flush_icache_range((unsigned long) reboot_code_buffer,
(unsigned long) reboot_code_buffer + KEXEC_CONTROL_PAGE_SIZE);
printk(KERN_INFO "Bye!\n");
local_irq_disable();
local_fiq_disable();
setup_mm_for_reboot(0); /* mode is not used, so just pass 0*/
flush_cache_all();
outer_flush_all();
outer_disable();
cpu_proc_fin();
outer_inv_all();
flush_cache_all();
__virt_to_phys(cpu_reset)(reboot_code_buffer_phys);
}
void machine_kexec(struct kimage *image)
{
unsigned long page_list;
unsigned long reboot_code_buffer_phys;
void *reboot_code_buffer;
if (num_online_cpus() > 1) {
pr_err("kexec: error: multiple CPUs still online\n");
return;
}
page_list = image->head & PAGE_MASK;
/* we need both effective and real address here */
reboot_code_buffer_phys =
page_to_pfn(image->control_code_page) << PAGE_SHIFT;
reboot_code_buffer = page_address(image->control_code_page);
/* Prepare parameters for reboot_code_buffer*/
kexec_start_address = image->start;
kexec_indirection_page = page_list;
kexec_mach_type = machine_arch_type;
if (!kexec_boot_atags)
kexec_boot_atags = image->start - KEXEC_ARM_ZIMAGE_OFFSET + KEXEC_ARM_ATAGS_OFFSET;
/* copy our kernel relocation code to the control code page */
memcpy(reboot_code_buffer,
relocate_new_kernel, relocate_new_kernel_size);
flush_icache_range((unsigned long) reboot_code_buffer,
(unsigned long) reboot_code_buffer + KEXEC_CONTROL_PAGE_SIZE);
printk(KERN_INFO "Bye!\n");
if (kexec_reinit)
kexec_reinit();
soft_restart(reboot_code_buffer_phys);
}
static void *ion_page_pool_alloc_pages(struct ion_page_pool *pool)
{
struct page *page = alloc_pages(pool->gfp_mask, pool->order);
if (!page)
return NULL;
/* this is only being used to flush the page for dma,
this api is not really suitable for calling from a driver
but no better way to flush a page for dma exist at this time */
#ifdef CONFIG_64BIT
dma_sync_single_for_device(NULL, (dma_addr_t)page_to_phys(page),
PAGE_SIZE << pool->order,
DMA_BIDIRECTIONAL);
#else
arm_dma_ops.sync_single_for_device(NULL,
pfn_to_dma(NULL, page_to_pfn(page)),
PAGE_SIZE << pool->order,
DMA_BIDIRECTIONAL);
#endif
return page;
}
static void kvm_mmu_write(void *dest, u64 val)
{
__u64 pte_phys;
struct kvm_mmu_op_write_pte wpte;
#ifdef CONFIG_HIGHPTE
struct page *page;
unsigned long dst = (unsigned long) dest;
page = kmap_atomic_to_page(dest);
pte_phys = page_to_pfn(page);
pte_phys <<= PAGE_SHIFT;
pte_phys += (dst & ~(PAGE_MASK));
#else
pte_phys = (unsigned long)__pa(dest);
#endif
wpte.header.op = KVM_MMU_OP_WRITE_PTE;
wpte.pte_val = val;
wpte.pte_phys = pte_phys;
kvm_deferred_mmu_op(&wpte, sizeof wpte);
}
int xen_unmap_domain_mfn_range(struct vm_area_struct *vma,
int nr, struct page **pages)
{
int i;
for (i = 0; i < nr; i++) {
struct xen_remove_from_physmap xrp;
unsigned long rc, pfn;
pfn = page_to_pfn(pages[i]);
xrp.domid = DOMID_SELF;
xrp.gpfn = pfn;
rc = HYPERVISOR_memory_op(XENMEM_remove_from_physmap, &xrp);
if (rc) {
pr_warn("Failed to unmap pfn:%lx rc:%ld\n",
pfn, rc);
return rc;
}
}
return 0;
}
/*
* Check that @page is mapped at @address into @mm.
*
* If @sync is false, page_check_address may perform a racy check to avoid
* the page table lock when the pte is not present (helpful when reclaiming
* highly shared pages).
*
* On success returns with pte mapped and locked.
*/
static pte_t *mr__page_check_address(struct page *page, struct mm_struct *mm,
unsigned long address, spinlock_t **ptlp, int sync)
{
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
spinlock_t *ptl;
pgd = pgd_offset(mm, address);
if (!pgd_present(*pgd))
return NULL;
pud = pud_offset(pgd, address);
if (!pud_present(*pud))
return NULL;
pmd = pmd_offset(pud, address);
if (!pmd_present(*pmd))
return NULL;
if (pmd_trans_huge(*pmd))
return NULL;
pte = pte_offset_map(pmd, address);
/* Make a quick check before getting the lock */
if (!sync && !pte_present(*pte)) {
pte_unmap(pte);
return NULL;
}
ptl = pte_lockptr(mm, pmd);
spin_lock(ptl);
if (pte_present(*pte) && page_to_pfn(page) == pte_pfn(*pte)) {
*ptlp = ptl;
return pte;
}
pte_unmap_unlock(pte, ptl);
return NULL;
}
/* Map the first page in an SG segment: common for multiple and single block IO */
static void *usdhi6_sg_map(struct usdhi6_host *host)
{
struct mmc_data *data = host->mrq->data;
struct scatterlist *sg = data->sg_len > 1 ? host->sg : data->sg;
size_t head = PAGE_SIZE - sg->offset;
size_t blk_head = head % data->blksz;
WARN(host->pg.page, "%p not properly unmapped!\n", host->pg.page);
if (WARN(sg_dma_len(sg) % data->blksz,
"SG size %u isn't a multiple of block size %u\n",
sg_dma_len(sg), data->blksz))
return NULL;
host->pg.page = sg_page(sg);
host->pg.mapped = kmap(host->pg.page);
host->offset = sg->offset;
/*
* Block size must be a power of 2 for multi-block transfers,
* therefore blk_head is equal for all pages in this SG
*/
host->head_len = blk_head;
if (head < data->blksz)
/*
* The first block in the SG crosses a page boundary.
* Max blksz = 512, so blocks can only span 2 pages
*/
usdhi6_blk_bounce(host, sg);
else
host->blk_page = host->pg.mapped;
dev_dbg(mmc_dev(host->mmc), "Mapped %p (%lx) at %p + %u for CMD%u @ 0x%p\n",
host->pg.page, page_to_pfn(host->pg.page), host->pg.mapped,
sg->offset, host->mrq->cmd->opcode, host->mrq);
return host->blk_page + host->offset;
}
/**
* dma_release_from_contiguous() - release allocated pages
* @dev: Pointer to device for which the pages were allocated.
* @pages: Allocated pages.
* @count: Number of allocated pages.
*
* This function releases memory allocated by dma_alloc_from_contiguous().
* It returns false when provided pages do not belong to contiguous area and
* true otherwise.
*/
bool dma_release_from_contiguous(struct device *dev, struct page *pages,
int count)
{
struct cma *cma = dev_get_cma_area(dev);
unsigned long pfn;
if (!cma || !pages)
return false;
pr_debug("%s(page %p)\n", __func__, (void *)pages);
pfn = page_to_pfn(pages);
if (pfn < cma->base_pfn || pfn >= cma->base_pfn + cma->count)
return false;
VM_BUG_ON(pfn + count > cma->base_pfn + cma->count);
free_contig_range(pfn, count);
clear_cma_bitmap(cma, pfn, count);
return true;
}
void machine_kexec(struct kimage *image)
{
unsigned long page_list;
unsigned long reboot_code_buffer_phys;
void *reboot_code_buffer;
arch_kexec();
page_list = image->head & PAGE_MASK;
reboot_code_buffer_phys =
page_to_pfn(image->control_code_page) << PAGE_SHIFT;
reboot_code_buffer = page_address(image->control_code_page);
kexec_start_address = image->start;
kexec_indirection_page = page_list;
kexec_mach_type = machine_arch_type;
kexec_boot_atags = image->start - KEXEC_ARM_ZIMAGE_OFFSET + KEXEC_ARM_ATAGS_OFFSET;
memcpy(reboot_code_buffer,
relocate_new_kernel, relocate_new_kernel_size);
flush_icache_range((unsigned long) reboot_code_buffer,
(unsigned long) reboot_code_buffer + KEXEC_CONTROL_PAGE_SIZE);
printk(KERN_INFO "Bye!\n");
cpu_proc_fin();
__virt_to_phys(cpu_reset)(reboot_code_buffer_phys);
}
static void __dma_clear_buffer(struct page *page, size_t size)
{
if (!PageHighMem(page)) {
void *ptr = page_address(page);
if (ptr) {
memset(ptr, 0, size);
dmac_flush_range(ptr, ptr + size);
outer_flush_range(__pa(ptr), __pa(ptr) + size);
}
} else {
phys_addr_t base = __pfn_to_phys(page_to_pfn(page));
phys_addr_t end = base + size;
while (size > 0) {
void *ptr = kmap_atomic(page);
memset(ptr, 0, PAGE_SIZE);
dmac_flush_range(ptr, ptr + PAGE_SIZE);
kunmap_atomic(ptr);
page++;
size -= PAGE_SIZE;
}
outer_flush_range(base, end);
}
}
static int my_init(void)
{
char* virt = (char *) __get_free_page(GFP_KERNEL);
strcpy(virt, "ahoj");
phys_addr_t phys = virt_to_phys(virt);
struct page* page = virt_to_page(virt);
SetPageReserved(page);
u32 *map = ioremap(phys, PAGE_SIZE);
unsigned long pfn = page_to_pfn(page);
printk(KERN_INFO "pb173: virt: %p", virt);
printk(KERN_INFO "pb173: phys: %llx", phys);
printk(KERN_INFO "pb173: page: %p", page);
printk(KERN_INFO "pb173: map: %p", map);
printk(KERN_INFO "pb173: page_to_pfn: %lx", pfn);
printk(KERN_INFO "pb173: obsah virt: %s", virt);
printk(KERN_INFO "pb173: obsah map: %s", map);
iounmap(virt);
ClearPageReserved(page);
free_page((unsigned long)virt);
return -EIO;
}
long pmem_direct_access(struct block_device *bdev, sector_t sector,
void __pmem **kaddr, pfn_t *pfn, long size)
{
struct pmem_device *pmem = bdev->bd_queue->queuedata;
resource_size_t offset = sector * 512 + pmem->data_offset;
if (unlikely(is_bad_pmem(&pmem->bb, sector, size)))
return -EIO;
/*
* Limit dax to a single page at a time given vmalloc()-backed
* in the nfit_test case.
*/
if (get_nfit_res(pmem->phys_addr + offset)) {
struct page *page;
*kaddr = pmem->virt_addr + offset;
page = vmalloc_to_page(pmem->virt_addr + offset);
*pfn = page_to_pfn_t(page);
dev_dbg_ratelimited(disk_to_dev(bdev->bd_disk)->parent,
"%s: sector: %#llx pfn: %#lx\n", __func__,
(unsigned long long) sector, page_to_pfn(page));
return PAGE_SIZE;
}
*kaddr = pmem->virt_addr + offset;
*pfn = phys_to_pfn_t(pmem->phys_addr + offset, pmem->pfn_flags);
/*
* If badblocks are present, limit known good range to the
* requested range.
*/
if (unlikely(pmem->bb.count))
return size;
return pmem->size - pmem->pfn_pad - offset;
}
/**
* dma_map_sg - map a set of SG buffers for streaming mode DMA
* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
* @sg: list of buffers
* @nents: number of buffers to map
* @dir: DMA transfer direction
*
* Map a set of buffers described by scatterlist in streaming mode for DMA.
* This is the scatter-gather version of the dma_map_single interface.
* Here the scatter gather list elements are each tagged with the
* appropriate dma address and length. They are obtained via
* sg_dma_{address,length}.
*
* Device ownership issues as mentioned for dma_map_single are the same
* here.
*/
int dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
enum dma_data_direction dir)
{
dma_addr_t dma_address;
struct scatterlist *s;
int i, j;
BUG_ON(!valid_dma_direction(dir));
for_each_sg(sg, s, nents, i) {
dma_address = __dma_map_page(dev, sg_page(s), s->offset,
s->length, dir);
/* When the page doesn't have a valid PFN, we assume that
* dma_address is already present. */
if (pfn_valid(page_to_pfn(sg_page(s))))
s->dma_address = dma_address;
#ifdef CONFIG_NEED_SG_DMA_LENGTH
s->dma_length = s->length;
#endif
if (dma_mapping_error(dev, s->dma_address))
goto bad_mapping;
}
static void *
__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp,
pgprot_t prot)
{
struct page *page;
void *addr;
*handle = ~0;
size = PAGE_ALIGN(size);
page = __dma_alloc_buffer(dev, size, gfp);
if (!page)
return NULL;
if (!arch_is_coherent())
addr = __dma_alloc_remap(page, size, gfp, prot);
else
addr = page_address(page);
if (addr)
*handle = pfn_to_dma(dev, page_to_pfn(page));
return addr;
}
struct page_ext *lookup_page_ext(struct page *page)
{
unsigned long pfn = page_to_pfn(page);
unsigned long index;
struct page_ext *base;
base = NODE_DATA(page_to_nid(page))->node_page_ext;
#if defined(CONFIG_DEBUG_VM) || defined(CONFIG_PAGE_POISONING)
/*
* The sanity checks the page allocator does upon freeing a
* page can reach here before the page_ext arrays are
* allocated when feeding a range of pages to the allocator
* for the first time during bootup or memory hotplug.
*
* This check is also necessary for ensuring page poisoning
* works as expected when enabled
*/
if (unlikely(!base))
return NULL;
#endif
index = pfn - round_down(node_start_pfn(page_to_nid(page)),
MAX_ORDER_NR_PAGES);
return get_entry(base, index);
}
static void *
__dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
const void *caller)
{
struct vm_struct *area;
unsigned long addr;
/*
* DMA allocation can be mapped to user space, so lets
* set VM_USERMAP flags too.
*/
area = get_vm_area_caller(size, VM_ARM_DMA_CONSISTENT | VM_USERMAP,
caller);
if (!area)
return NULL;
addr = (unsigned long)area->addr;
area->phys_addr = __pfn_to_phys(page_to_pfn(page));
if (ioremap_page_range(addr, addr + size, area->phys_addr, prot)) {
vunmap((void *)addr);
return NULL;
}
return (void *)addr;
}
static void *
__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp,
pgprot_t prot)
{
struct page *page;
void *addr;
/*
* FIXME:DMA memory, split_page, BUG_ON(PageCompound()) google it pls :P c58721
*/
/* Google result:
* Following is a work-around (a.k.a. hack) to prevent pages
* with __GFP_COMP being passed to split_page() which cannot
* handle them. The real problem is that this flag probably
* should be 0 on ARM as it is not supported on this
* platform--see CONFIG_HUGETLB_PAGE. */
gfp &= ~(__GFP_COMP);
*handle = ~0;
size = PAGE_ALIGN(size);
page = __dma_alloc_buffer(dev, size, gfp);
if (!page)
return NULL;
if (!arch_is_coherent())
addr = __dma_alloc_remap(page, size, gfp, prot);
else
addr = page_address(page);
if (addr)
*handle = pfn_to_dma(dev, page_to_pfn(page));
else
__dma_free_buffer(page, size);
return addr;
}
void *huge_vmap(struct page *page, pgprot_t prot)
{
int i;
unsigned long flags, addr;
pgd_t *pgd;
pmd_t *pmd;
spin_lock_irqsave(&map_lock, flags);
for (i = 0; i < CONFIG_NR_HUGE_VMAPS; ++i) {
if (map[i] == NULL) {
map[i] = page;
break;
}
}
spin_unlock_irqrestore(&map_lock, flags);
if (i == CONFIG_NR_HUGE_VMAPS)
return NULL;
addr = HUGE_VMAP_BASE + (i * HPAGE_SIZE);
pgd = swapper_pg_dir + pgd_index(addr);
pmd = pmd_offset(pud_offset(pgd, addr), addr);
set_pte((pte_t *)pmd, pte_mkhuge(pfn_pte(page_to_pfn(page), prot)));
return (void *) addr;
}
static int exynos_drm_ump_add_buffer(void *obj,
unsigned int *handle, unsigned int *id)
{
struct exynos_drm_gem_obj *gem_obj = obj;
struct exynos_drm_gem_buf *buf = gem_obj->buffer;
ump_dd_physical_block *ump_mem_desc;
unsigned int nblocks;
DRM_DEBUG_KMS("%s\n", __FILE__);
if (IS_NONCONTIG_BUFFER(gem_obj->flags)) {
unsigned int i = 0;
if (!buf->pages)
return -EFAULT;
nblocks = gem_obj->size >> PAGE_SHIFT;
ump_mem_desc = kzalloc(sizeof(*ump_mem_desc) * nblocks,
GFP_KERNEL);
if (!ump_mem_desc) {
DRM_ERROR("failed to alloc ump_mem_desc.\n");
return -ENOMEM;
}
/*
* if EXYNOS_BO_NONCONTIG type, gem object would already
* have pages allocated by gem creation so contain page
* frame numbers of all pages into ump descriptors.
*/
while (i < nblocks) {
ump_mem_desc[i].addr =
page_to_pfn(buf->pages[i]) << PAGE_SHIFT;
ump_mem_desc[i].size = PAGE_SIZE;
i++;
}
} else {
static inline unsigned long vaddr(struct pending_req *req, int seg)
{
unsigned long pfn = page_to_pfn(blkbk->pending_page(req, seg));
return (unsigned long)pfn_to_kaddr(pfn);
}
void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
{
pgtable_page_dtor(pte);
paravirt_release_pte(page_to_pfn(pte));
tlb_remove_page(tlb, pte);
}
asmlinkage void __init start_kernel(void)
{
char * command_line;
extern struct kernel_param __start___param[], __stop___param[];
smp_setup_processor_id();
/*
* Need to run as early as possible, to initialize the
* lockdep hash:
*/
lockdep_init();
debug_objects_early_init();
/*
* Set up the the initial canary ASAP:
*/
boot_init_stack_canary();
cgroup_init_early();
local_irq_disable();
early_boot_irqs_off();
early_init_irq_lock_class();
/*
* Interrupts are still disabled. Do necessary setups, then
* enable them
*/
lock_kernel();
tick_init();
boot_cpu_init();
page_address_init();
printk(KERN_NOTICE "%s", linux_banner);
setup_arch(&command_line);
mm_init_owner(&init_mm, &init_task);
setup_command_line(command_line);
setup_nr_cpu_ids();
setup_per_cpu_areas();
smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */
build_all_zonelists(NULL);
page_alloc_init();
printk(KERN_NOTICE "Kernel command line: %s\n", boot_command_line);
//[email protected] 2011.11.14 begin
//support lcd compatible
//reviewed by [email protected]
#if defined(CONFIG_LCD_DRV_ALL)
char *p = strstr(boot_command_line, "lcd=");
if (p)
{
lcd_drv_index = p[4] - 'A';
printk("lcd index = %d", lcd_drv_index);
}
#endif
//[email protected] 2011.11.14 end
parse_early_param();
parse_args("Booting kernel", static_command_line, __start___param,
__stop___param - __start___param,
&unknown_bootoption);
/*
* These use large bootmem allocations and must precede
* kmem_cache_init()
*/
pidhash_init();
vfs_caches_init_early();
sort_main_extable();
trap_init();
mm_init();
/*
* Set up the scheduler prior starting any interrupts (such as the
* timer interrupt). Full topology setup happens at smp_init()
* time - but meanwhile we still have a functioning scheduler.
*/
sched_init();
/*
* Disable preemption - early bootup scheduling is extremely
* fragile until we cpu_idle() for the first time.
*/
preempt_disable();
if (!irqs_disabled()) {
printk(KERN_WARNING "start_kernel(): bug: interrupts were "
"enabled *very* early, fixing it\n");
local_irq_disable();
}
rcu_init();
radix_tree_init();
/* init some links before init_ISA_irqs() */
early_irq_init();
init_IRQ();
prio_tree_init();
init_timers();
hrtimers_init();
softirq_init();
timekeeping_init();
time_init();
profile_init();
if (!irqs_disabled())
printk(KERN_CRIT "start_kernel(): bug: interrupts were "
"enabled early\n");
early_boot_irqs_on();
local_irq_enable();
/* Interrupts are enabled now so all GFP allocations are safe. */
gfp_allowed_mask = __GFP_BITS_MASK;
kmem_cache_init_late();
/*
* HACK ALERT! This is early. We're enabling the console before
* we've done PCI setups etc, and console_init() must be aware of
* this. But we do want output early, in case something goes wrong.
*/
console_init();
if (panic_later)
panic(panic_later, panic_param);
lockdep_info();
/*
* Need to run this when irqs are enabled, because it wants
* to self-test [hard/soft]-irqs on/off lock inversion bugs
* too:
*/
locking_selftest();
#ifdef CONFIG_BLK_DEV_INITRD
if (initrd_start && !initrd_below_start_ok &&
page_to_pfn(virt_to_page((void *)initrd_start)) < min_low_pfn) {
printk(KERN_CRIT "initrd overwritten (0x%08lx < 0x%08lx) - "
"disabling it.\n",
page_to_pfn(virt_to_page((void *)initrd_start)),
min_low_pfn);
initrd_start = 0;
}
#endif
page_cgroup_init();
enable_debug_pagealloc();
kmemtrace_init();
kmemleak_init();
debug_objects_mem_init();
idr_init_cache();
setup_per_cpu_pageset();
numa_policy_init();
if (late_time_init)
late_time_init();
sched_clock_init();
calibrate_delay();
pidmap_init();
anon_vma_init();
#ifdef CONFIG_X86
if (efi_enabled)
efi_enter_virtual_mode();
#endif
thread_info_cache_init();
cred_init();
fork_init(totalram_pages);
proc_caches_init();
buffer_init();
key_init();
security_init();
dbg_late_init();
vfs_caches_init(totalram_pages);
signals_init();
/* rootfs populating might need page-writeback */
page_writeback_init();
#ifdef CONFIG_PROC_FS
proc_root_init();
#endif
cgroup_init();
cpuset_init();
taskstats_init_early();
delayacct_init();
check_bugs();
acpi_early_init(); /* before LAPIC and SMP init */
sfi_init_late();
ftrace_init();
/* Do the rest non-__init'ed, we're now alive */
rest_init();
}
void probe_hugetlb_page_alloc(void *_data, struct page *page)
{
if (page)
trace_mark_tp(mm, huge_page_alloc, hugetlb_page_alloc,
probe_hugetlb_page_alloc, "pfn %lu", page_to_pfn(page));
}
void probe_hugetlb_page_free(void *_data, struct page *page)
{
trace_mark_tp(mm, huge_page_free, hugetlb_page_free,
probe_hugetlb_page_free, "pfn %lu", page_to_pfn(page));
}
void probe_wait_on_page_end(void *_data, struct page *page, int bit_nr)
{
trace_mark_tp(mm, wait_on_page_end, wait_on_page_end,
probe_wait_on_page_end, "pfn %lu bit_nr %d",
page_to_pfn(page), bit_nr);
}
