int p2m_alloc_table(struct domain *d) { struct p2m_domain *p2m = &d->arch.p2m; struct page_info *page; void *p; /* First level P2M is 2 consecutive pages */ page = alloc_domheap_pages(NULL, 1, 0); if ( page == NULL ) return -ENOMEM; spin_lock(&p2m->lock); page_list_add(page, &p2m->pages); /* Clear both first level pages */ p = __map_domain_page(page); clear_page(p); unmap_domain_page(p); p = __map_domain_page(page + 1); clear_page(p); unmap_domain_page(p); p2m->first_level = page; d->arch.vttbr = page_to_maddr(p2m->first_level) | ((uint64_t)p2m->vmid&0xff)<<48; spin_unlock(&p2m->lock); return 0; }
int p2m_alloc_table(struct domain *d) { struct p2m_domain *p2m = &d->arch.p2m; struct page_info *page; void *p; page = alloc_domheap_pages(NULL, P2M_FIRST_ORDER, 0); if ( page == NULL ) return -ENOMEM; spin_lock(&p2m->lock); /* Clear both first level pages */ p = __map_domain_page(page); clear_page(p); unmap_domain_page(p); p = __map_domain_page(page + 1); clear_page(p); unmap_domain_page(p); p2m->first_level = page; d->arch.vttbr = page_to_maddr(p2m->first_level) | ((uint64_t)p2m->vmid&0xff)<<48; /* Make sure that all TLBs corresponding to the new VMID are flushed * before using it */ flush_tlb_domain(d); spin_unlock(&p2m->lock); return 0; }
/* * clear_user_page * @to: P1 address * @address: U0 address to be mapped * @page: page (virt_to_page(to)) */ void clear_user_page(void *to, unsigned long address, struct page *page) { __set_bit(PG_mapped, &page->flags); if (((address ^ (unsigned long)to) & CACHE_ALIAS) == 0) clear_page(to); else { pgprot_t pgprot = __pgprot(_PAGE_PRESENT | _PAGE_RW | _PAGE_CACHABLE | _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_HW_SHARED | _PAGE_FLAGS_HARD); unsigned long phys_addr = PHYSADDR(to); unsigned long p3_addr = P3SEG + (address & CACHE_ALIAS); pgd_t *dir = pgd_offset_k(p3_addr); pmd_t *pmd = pmd_offset(dir, p3_addr); pte_t *pte = pte_offset_kernel(pmd, p3_addr); pte_t entry; unsigned long flags; entry = pfn_pte(phys_addr >> PAGE_SHIFT, pgprot); down(&p3map_sem[(address & CACHE_ALIAS)>>12]); set_pte(pte, entry); local_irq_save(flags); __flush_tlb_page(get_asid(), p3_addr); local_irq_restore(flags); update_mmu_cache(NULL, p3_addr, entry); __clear_user_page((void *)p3_addr, to); pte_clear(&init_mm, p3_addr, pte); up(&p3map_sem[(address & CACHE_ALIAS)>>12]); } }
int dax_clear_blocks(struct inode *inode, sector_t block, long size) { struct block_device *bdev = inode->i_sb->s_bdev; sector_t sector = block << (inode->i_blkbits - 9); might_sleep(); do { void *addr; unsigned long pfn; long count; count = bdev_direct_access(bdev, sector, &addr, &pfn, size); if (count < 0) return count; BUG_ON(size < count); while (count > 0) { unsigned pgsz = PAGE_SIZE - offset_in_page(addr); if (pgsz > count) pgsz = count; if (pgsz < PAGE_SIZE) memset(addr, 0, pgsz); else clear_page(addr); addr += pgsz; size -= pgsz; count -= pgsz; BUG_ON(pgsz & 511); sector += pgsz / 512; cond_resched(); } } while (size); return 0; }
/* * clear_user_page * @to: P1 address * @address: U0 address to be mapped * @page: page (virt_to_page(to)) */ void clear_user_page(void *to, unsigned long address, struct page *page) { __set_bit(PG_mapped, &page->flags); if (((address ^ (unsigned long)to) & CACHE_ALIAS) == 0) clear_page(to); else { unsigned long phys_addr = PHYSADDR(to); unsigned long p3_addr = P3SEG + (address & CACHE_ALIAS); pgd_t *pgd = pgd_offset_k(p3_addr); pud_t *pud = pud_offset(pgd, p3_addr); pmd_t *pmd = pmd_offset(pud, p3_addr); pte_t *pte = pte_offset_kernel(pmd, p3_addr); pte_t entry; unsigned long flags; entry = pfn_pte(phys_addr >> PAGE_SHIFT, PAGE_KERNEL); mutex_lock(&p3map_mutex[(address & CACHE_ALIAS)>>12]); set_pte(pte, entry); local_irq_save(flags); __flush_tlb_page(get_asid(), p3_addr); local_irq_restore(flags); update_mmu_cache(NULL, p3_addr, entry); __clear_user_page((void *)p3_addr, to); pte_clear(&init_mm, p3_addr, pte); mutex_unlock(&p3map_mutex[(address & CACHE_ALIAS)>>12]); } }
static int zram_decompress_page(struct zram *zram, char *mem, u32 index) { int ret = LZO_E_OK; size_t clen = PAGE_SIZE; unsigned char *cmem; struct zram_meta *meta = zram->meta; unsigned long handle = meta->table[index].handle; if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) { clear_page(mem); return 0; } cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO); if (meta->table[index].size == PAGE_SIZE) copy_page(mem, cmem); else ret = lzo1x_decompress_safe(cmem, meta->table[index].size, mem, &clen); zs_unmap_object(meta->mem_pool, handle); /* Should NEVER happen. Return bio error if it does. */ if (unlikely(ret != LZO_E_OK)) { pr_err("Decompression failed! err=%d, page=%u\n", ret, index); zram_stat64_inc(zram, &zram->stats.failed_reads); return ret; } return 0; }
static int __init init_vdso(void) { struct mips_vdso *vdso; vdso_page = alloc_page(GFP_KERNEL); if (!vdso_page) panic("Cannot allocate vdso"); vdso = vmap(&vdso_page, 1, 0, PAGE_KERNEL); if (!vdso) panic("Cannot map vdso"); clear_page(vdso); install_trampoline(vdso->rt_signal_trampoline, __NR_rt_sigreturn); #ifdef CONFIG_32BIT install_trampoline(vdso->signal_trampoline, __NR_sigreturn); #else install_trampoline(vdso->n32_rt_signal_trampoline, __NR_N32_rt_sigreturn); install_trampoline(vdso->o32_signal_trampoline, __NR_O32_sigreturn); install_trampoline(vdso->o32_rt_signal_trampoline, __NR_O32_rt_sigreturn); #endif vunmap(vdso); return 0; }
static void shmedia_mapioaddr(unsigned long pa, unsigned long va) { pgd_t *pgdp; pmd_t *pmdp; pte_t *ptep; unsigned long flags = 1; /* 1 = CB0-1 device */ DEBUG_IOREMAP(("shmedia_mapiopage pa %08x va %08x\n", pa, va)); pgdp = pgd_offset_k(va); if (pgd_none(*pgdp)) { pmdp = alloc_bootmem_low_pages(PTRS_PER_PMD * sizeof(pmd_t)); if (pmdp == NULL) panic("No memory for pmd\n"); memset(pmdp, 0, PTRS_PER_PGD * sizeof(pmd_t)); set_pgd(pgdp, __pgd((unsigned long)pmdp | _KERNPG_TABLE)); } pmdp = pmd_offset(pgdp, va); if (pmd_none(*pmdp)) { ptep = alloc_bootmem_low_pages(PTRS_PER_PTE * sizeof(pte_t)); if (ptep == NULL) panic("No memory for pte\n"); clear_page((void *)ptep); set_pmd(pmdp, __pmd((unsigned long)ptep + _PAGE_TABLE)); } ptep = pte_offset(pmdp, va); set_pte(ptep, mk_pte_phys(pa, __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | _PAGE_DIRTY | _PAGE_ACCESSED |_PAGE_SHARED | flags))); }
static void shmedia_unmapioaddr(unsigned long vaddr) { pgd_t *pgdp; pud_t *pudp; pmd_t *pmdp; pte_t *ptep; pgdp = pgd_offset_k(vaddr); if (pgd_none(*pgdp) || pgd_bad(*pgdp)) return; pudp = pud_offset(pgdp, vaddr); if (pud_none(*pudp) || pud_bad(*pudp)) return; pmdp = pmd_offset(pudp, vaddr); if (pmd_none(*pmdp) || pmd_bad(*pmdp)) return; ptep = pte_offset_kernel(pmdp, vaddr); if (pte_none(*ptep) || !pte_present(*ptep)) return; clear_page((void *)ptep); pte_clear(&init_mm, vaddr, ptep); }
/** * kvm_mips_walk_pgd() - Walk page table with optional allocation. * @pgd: Page directory pointer. * @addr: Address to index page table using. * @cache: MMU page cache to allocate new page tables from, or NULL. * * Walk the page tables pointed to by @pgd to find the PTE corresponding to the * address @addr. If page tables don't exist for @addr, they will be created * from the MMU cache if @cache is not NULL. * * Returns: Pointer to pte_t corresponding to @addr. * NULL if a page table doesn't exist for @addr and !@cache. * NULL if a page table allocation failed. */ static pte_t *kvm_mips_walk_pgd(pgd_t *pgd, struct kvm_mmu_memory_cache *cache, unsigned long addr) { pud_t *pud; pmd_t *pmd; pgd += pgd_index(addr); if (pgd_none(*pgd)) { /* Not used on MIPS yet */ BUG(); return NULL; } pud = pud_offset(pgd, addr); if (pud_none(*pud)) { pmd_t *new_pmd; if (!cache) return NULL; new_pmd = mmu_memory_cache_alloc(cache); pmd_init((unsigned long)new_pmd, (unsigned long)invalid_pte_table); pud_populate(NULL, pud, new_pmd); } pmd = pmd_offset(pud, addr); if (pmd_none(*pmd)) { pte_t *new_pte; if (!cache) return NULL; new_pte = mmu_memory_cache_alloc(cache); clear_page(new_pte); pmd_populate_kernel(NULL, pmd, new_pte); } return pte_offset(pmd, addr); }
/* Allocate a new page table page and hook it in via the given entry */ static int p2m_create_table(struct domain *d, lpae_t *entry) { struct p2m_domain *p2m = &d->arch.p2m; struct page_info *page; void *p; lpae_t pte; BUG_ON(entry->p2m.valid); page = alloc_domheap_page(NULL, 0); if ( page == NULL ) return -ENOMEM; page_list_add(page, &p2m->pages); p = __map_domain_page(page); clear_page(p); unmap_domain_page(p); pte = mfn_to_p2m_entry(page_to_mfn(page), MATTR_MEM); write_pte(entry, pte); return 0; }
/* * Clear the user page. We need to deal with the aliasing issues, * so remap the kernel page into the same cache colour as the user * page. */ void v6_clear_user_page_aliasing(void *kaddr, unsigned long vaddr) { unsigned int offset = DCACHE_COLOUR(vaddr); unsigned long to = to_address + (offset << PAGE_SHIFT); /* * Discard data in the kernel mapping for the new page * FIXME: needs this MCRR to be supported. */ __asm__("mcrr p15, 0, %1, %0, c6 @ 0xec401f06" : : "r" (kaddr), "r" ((unsigned long)kaddr + PAGE_SIZE - L1_CACHE_BYTES) : "cc"); /* * Now clear the page using the same cache colour as * the pages ultimate destination. */ spin_lock(&v6_lock); set_pte(to_pte + offset, pfn_pte(__pa(kaddr) >> PAGE_SHIFT, to_pgprot)); flush_tlb_kernel_page(to); clear_page((void *)to); spin_unlock(&v6_lock); }
void __init init_apic_mappings(void) { unsigned long apic_phys; if ( x2apic_enabled ) goto __next; /* * If no local APIC can be found then set up a fake all * zeroes page to simulate the local APIC and another * one for the IO-APIC. */ if (!smp_found_config && detect_init_APIC()) { apic_phys = __pa(alloc_xenheap_page()); clear_page(__va(apic_phys)); } else apic_phys = mp_lapic_addr; set_fixmap_nocache(FIX_APIC_BASE, apic_phys); apic_printk(APIC_VERBOSE, "mapped APIC to %08lx (%08lx)\n", APIC_BASE, apic_phys); __next: /* * Fetch the APIC ID of the BSP in case we have a * default configuration (or the MP table is broken). */ if (boot_cpu_physical_apicid == -1U) boot_cpu_physical_apicid = get_apic_id(); x86_cpu_to_apicid[0] = get_apic_id(); init_ioapic_mappings(); }
/* * clear_user_page * @to: P1 address * @address: U0 address to be mapped */ void clear_user_page(void *to, unsigned long address, struct page *pg) { struct page *page = virt_to_page(to); __set_bit(PG_mapped, &page->flags); if (((address ^ (unsigned long)to) & CACHE_ALIAS) == 0) { clear_page(to); __flush_wback_region(to, PAGE_SIZE); } else { __flush_purge_virtual_region(to, (void *)(address & 0xfffff000), PAGE_SIZE); clear_page(to); __flush_wback_region(to, PAGE_SIZE); } }
pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT); if (pte) clear_page(pte); return pte; }
int arch_domain_create(struct domain *d, unsigned int domcr_flags) { int rc; d->arch.relmem = RELMEM_not_started; /* Idle domains do not need this setup */ if ( is_idle_domain(d) ) return 0; if ( (rc = p2m_init(d)) != 0 ) goto fail; rc = -ENOMEM; if ( (d->shared_info = alloc_xenheap_pages(0, 0)) == NULL ) goto fail; /* Default the virtual ID to match the physical */ d->arch.vpidr = boot_cpu_data.midr.bits; clear_page(d->shared_info); share_xen_page_with_guest( virt_to_page(d->shared_info), d, XENSHARE_writable); if ( (rc = p2m_alloc_table(d)) != 0 ) goto fail; if ( (rc = gicv_setup(d)) != 0 ) goto fail; if ( (rc = domain_vgic_init(d)) != 0 ) goto fail; if ( (rc = domain_vtimer_init(d)) != 0 ) goto fail; if ( d->domain_id ) d->arch.evtchn_irq = GUEST_EVTCHN_PPI; else d->arch.evtchn_irq = platform_dom0_evtchn_ppi(); /* * Virtual UART is only used by linux early printk and decompress code. * Only use it for the hardware domain because the linux kernel may not * support multi-platform. */ if ( is_hardware_domain(d) && (rc = domain_vuart_init(d)) ) goto fail; if ( (rc = iommu_domain_init(d)) != 0 ) goto fail; return 0; fail: d->is_dying = DOMDYING_dead; arch_domain_destroy(d); return rc; }
pgd_t *pgd_alloc(struct mm_struct *mm) { pgd_t *ret; if ((ret = (pgd_t *)__get_free_pages(GFP_KERNEL, PGDIR_ORDER)) != NULL) clear_page(ret); return ret; }
pte_t __bad_page(void) { extern char empty_bad_page[PAGE_SIZE]; unsigned long page = (unsigned long)empty_bad_page; clear_page(page); return pte_mkdirty(mk_pte(page, PAGE_SHARED)); }
/* * Allocate a new page table page and hook it in via the given entry. * apply_one_level relies on this returning 0 on success * and -ve on failure. * * If the existing entry is present then it must be a mapping and not * a table and it will be shattered into the next level down. * * level_shift is the number of bits at the level we want to create. */ static int p2m_create_table(struct domain *d, lpae_t *entry, int level_shift, bool_t flush_cache) { struct p2m_domain *p2m = &d->arch.p2m; struct page_info *page; lpae_t *p; lpae_t pte; int splitting = p2m_valid(*entry); BUG_ON(p2m_table(*entry)); page = alloc_domheap_page(NULL, 0); if ( page == NULL ) return -ENOMEM; page_list_add(page, &p2m->pages); p = __map_domain_page(page); if ( splitting ) { p2m_type_t t = entry->p2m.type; unsigned long base_pfn = entry->p2m.base; int i; /* * We are either splitting a first level 1G page into 512 second level * 2M pages, or a second level 2M page into 512 third level 4K pages. */ for ( i=0 ; i < LPAE_ENTRIES; i++ ) { pte = mfn_to_p2m_entry(base_pfn + (i<<(level_shift-LPAE_SHIFT)), MATTR_MEM, t, p2m->default_access); /* * First and second level super pages set p2m.table = 0, but * third level entries set table = 1. */ if ( level_shift - LPAE_SHIFT ) pte.p2m.table = 0; write_pte(&p[i], pte); } } else clear_page(p); if ( flush_cache ) clean_dcache_va_range(p, PAGE_SIZE); unmap_domain_page(p); pte = mfn_to_p2m_entry(page_to_mfn(page), MATTR_MEM, p2m_invalid, p2m->default_access); p2m_write_pte(entry, pte, flush_cache); return 0; }
struct vcpu *alloc_vcpu_struct(void) { struct vcpu *v; BUILD_BUG_ON(sizeof(*v) > PAGE_SIZE); v = alloc_xenheap_pages(0, 0); if ( v != NULL ) clear_page(v); return v; }
/*-----------------------------------------------------------------------------------*/ static void start_loading(void) { loading = 1; x = y = 0; pagey = 0; webpageptr = webpage; clear_page(); }
void clear_user_highpage(struct page *page, unsigned long vaddr) { void *kaddr = kmap_atomic(page); clear_page(kaddr); if (pages_do_alias((unsigned long)kaddr, vaddr & PAGE_MASK)) __flush_purge_region(kaddr, PAGE_SIZE); kunmap_atomic(kaddr); }
void clear_user_page(void *page, unsigned long vaddr, struct page *pg) { clear_page(page); /* * We shouldnt have to do this, but some versions of glibc * require it (ld.so assumes zero filled pages are icache clean) * - Anton */ flush_dcache_page(pg); }
static __init void *alloc_low_page(void) { unsigned long pfn = pgt_buf_end++; void *adr; if (pfn >= pgt_buf_top) panic("alloc_low_page: ran out of memory"); adr = __va(pfn * PAGE_SIZE); clear_page(adr); return adr; }
int ext2_clear_xip_target(struct inode *inode, sector_t block) { void *kaddr; unsigned long pfn; int rc; rc = __inode_direct_access(inode, block, &kaddr, &pfn); if (!rc) clear_page(kaddr); return rc; }
struct domain *alloc_domain_struct(void) { struct domain *d; BUILD_BUG_ON(sizeof(*d) > PAGE_SIZE); d = alloc_xenheap_pages(0, 0); if ( d == NULL ) return NULL; clear_page(d); d->arch.grant_table_gpfn = xmalloc_array(xen_pfn_t, max_nr_grant_frames); return d; }
__ref pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { pte_t *pte; if (slab_is_available()) { pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_ZERO); } else { pte = __va(memblock_alloc(PAGE_SIZE, PAGE_SIZE)); if (pte) clear_page(pte); } return pte; }
static pte_t * __init kernel_page_table(void) { pte_t *ptablep; ptablep = (pte_t *)alloc_bootmem_low_pages(PAGE_SIZE); clear_page(ptablep); __flush_page_to_ram(ptablep); flush_tlb_kernel_page(ptablep); nocache_page(ptablep); return ptablep; }
/* get a free page by physical address */ void *get_free_page(void){ while(1){ foreach(i,CONST_MEM >> 12,KMEM >> 12){ if(mmap[i] == 0){ mmap[i]++; /* clear the page */ clear_page((i << 12)); return (void*)(i << 12); } } } return NULL; }
static inline int get_maptrack_handle( struct grant_table *lgt) { int i; grant_handle_t handle; struct grant_mapping *new_mt; unsigned int new_mt_limit, nr_frames; if ( unlikely((handle = __get_maptrack_handle(lgt)) == -1) ) { spin_lock(&lgt->lock); if ( unlikely((handle = __get_maptrack_handle(lgt)) == -1) ) { nr_frames = nr_maptrack_frames(lgt); if ( nr_frames >= max_nr_maptrack_frames() ) { spin_unlock(&lgt->lock); return -1; } new_mt = alloc_xenheap_page(); if ( new_mt == NULL ) { spin_unlock(&lgt->lock); return -1; } clear_page(new_mt); new_mt_limit = lgt->maptrack_limit + MAPTRACK_PER_PAGE; for ( i = lgt->maptrack_limit; i < new_mt_limit; i++ ) { new_mt[i % MAPTRACK_PER_PAGE].ref = i+1; new_mt[i % MAPTRACK_PER_PAGE].flags = 0; } lgt->maptrack[nr_frames] = new_mt; lgt->maptrack_limit = new_mt_limit; gdprintk(XENLOG_INFO, "Increased maptrack size to %u frames.\n", nr_frames + 1); handle = __get_maptrack_handle(lgt); } spin_unlock(&lgt->lock); } return handle; }