/* Allocate a new page table page and hook it in via the given entry */
static int p2m_create_table(struct domain *d,
lpae_t *entry)
{
struct p2m_domain *p2m = &d->arch.p2m;
struct page_info *page;
void *p;
lpae_t pte;
BUG_ON(entry->p2m.valid);
page = alloc_domheap_page(NULL, 0);
if ( page == NULL )
return -ENOMEM;
page_list_add(page, &p2m->pages);
p = __map_domain_page(page);
clear_page(p);
unmap_domain_page(p);
pte = mfn_to_p2m_entry(page_to_mfn(page), MATTR_MEM);
write_pte(entry, pte);
return 0;
}
/*
* Allocate a new page table page and hook it in via the given entry.
* apply_one_level relies on this returning 0 on success
* and -ve on failure.
*
* If the existing entry is present then it must be a mapping and not
* a table and it will be shattered into the next level down.
*
* level_shift is the number of bits at the level we want to create.
*/
static int p2m_create_table(struct domain *d, lpae_t *entry,
int level_shift, bool_t flush_cache)
{
struct p2m_domain *p2m = &d->arch.p2m;
struct page_info *page;
lpae_t *p;
lpae_t pte;
int splitting = p2m_valid(*entry);
BUG_ON(p2m_table(*entry));
page = alloc_domheap_page(NULL, 0);
if ( page == NULL )
return -ENOMEM;
page_list_add(page, &p2m->pages);
p = __map_domain_page(page);
if ( splitting )
{
p2m_type_t t = entry->p2m.type;
unsigned long base_pfn = entry->p2m.base;
int i;
/*
* We are either splitting a first level 1G page into 512 second level
* 2M pages, or a second level 2M page into 512 third level 4K pages.
*/
for ( i=0 ; i < LPAE_ENTRIES; i++ )
{
pte = mfn_to_p2m_entry(base_pfn + (i<<(level_shift-LPAE_SHIFT)),
MATTR_MEM, t, p2m->default_access);
/*
* First and second level super pages set p2m.table = 0, but
* third level entries set table = 1.
*/
if ( level_shift - LPAE_SHIFT )
pte.p2m.table = 0;
write_pte(&p[i], pte);
}
}
else
clear_page(p);
if ( flush_cache )
clean_dcache_va_range(p, PAGE_SIZE);
unmap_domain_page(p);
pte = mfn_to_p2m_entry(page_to_mfn(page), MATTR_MEM, p2m_invalid,
p2m->default_access);
p2m_write_pte(entry, pte, flush_cache);
return 0;
}
/* Set the pool of pages to the required number of pages.
* Returns 0 for success, non-zero for failure. */
static unsigned int
hap_set_allocation(struct domain *d, unsigned int pages, int *preempted)
{
struct page_info *pg;
ASSERT(hap_locked_by_me(d));
while ( d->arch.paging.hap.total_pages != pages )
{
if ( d->arch.paging.hap.total_pages < pages )
{
/* Need to allocate more memory from domheap */
pg = alloc_domheap_page(NULL);
if ( pg == NULL )
{
HAP_PRINTK("failed to allocate hap pages.\n");
return -ENOMEM;
}
d->arch.paging.hap.free_pages++;
d->arch.paging.hap.total_pages++;
list_add_tail(&pg->list, &d->arch.paging.hap.freelist);
}
else if ( d->arch.paging.hap.total_pages > pages )
{
/* Need to return memory to domheap */
ASSERT(!list_empty(&d->arch.paging.hap.freelist));
pg = list_entry(d->arch.paging.hap.freelist.next,
struct page_info, list);
list_del(&pg->list);
d->arch.paging.hap.free_pages--;
d->arch.paging.hap.total_pages--;
pg->count_info = 0;
free_domheap_page(pg);
}
/* Check to see if we need to yield and try again */
if ( preempted && hypercall_preempt_check() )
{
*preempted = 1;
return 0;
}
}
void __init vm_init(void)
{
unsigned int i, nr;
unsigned long va;
vm_base = (void *)VMAP_VIRT_START;
vm_end = PFN_DOWN(arch_vmap_virt_end() - vm_base);
vm_low = PFN_UP((vm_end + 7) / 8);
nr = PFN_UP((vm_low + 7) / 8);
vm_top = nr * PAGE_SIZE * 8;
for ( i = 0, va = (unsigned long)vm_bitmap; i < nr; ++i, va += PAGE_SIZE )
{
struct page_info *pg = alloc_domheap_page(NULL, 0);
map_pages_to_xen(va, page_to_mfn(pg), 1, PAGE_HYPERVISOR);
clear_page((void *)va);
}
bitmap_fill(vm_bitmap, vm_low);
/* Populate page tables for the bitmap if necessary. */
map_pages_to_xen(va, 0, vm_low - nr, MAP_SMALL_PAGES);
}
static int setup_compat_l4(struct vcpu *v)
{
struct page_info *pg;
l4_pgentry_t *l4tab;
mfn_t mfn;
pg = alloc_domheap_page(v->domain, MEMF_no_owner);
if ( pg == NULL )
return -ENOMEM;
mfn = page_to_mfn(pg);
l4tab = map_domain_page(mfn);
clear_page(l4tab);
init_xen_l4_slots(l4tab, mfn, v->domain, INVALID_MFN, false);
unmap_domain_page(l4tab);
/* This page needs to look like a pagetable so that it can be shadowed */
pg->u.inuse.type_info = PGT_l4_page_table | PGT_validated | 1;
v->arch.guest_table = pagetable_from_page(pg);
v->arch.guest_table_user = v->arch.guest_table;
return 0;
}
static lpae_t mfn_to_p2m_entry(unsigned long mfn, unsigned int mattr,
p2m_type_t t)
{
paddr_t pa = ((paddr_t) mfn) << PAGE_SHIFT;
/* xn and write bit will be defined in the switch */
lpae_t e = (lpae_t) {
.p2m.af = 1,
.p2m.sh = LPAE_SH_OUTER,
.p2m.read = 1,
.p2m.mattr = mattr,
.p2m.table = 1,
.p2m.valid = 1,
.p2m.type = t,
};
BUILD_BUG_ON(p2m_max_real_type > (1 << 4));
switch (t)
{
case p2m_ram_rw:
e.p2m.xn = 0;
e.p2m.write = 1;
break;
case p2m_ram_ro:
e.p2m.xn = 0;
e.p2m.write = 0;
break;
case p2m_map_foreign:
case p2m_grant_map_rw:
case p2m_mmio_direct:
e.p2m.xn = 1;
e.p2m.write = 1;
break;
case p2m_grant_map_ro:
case p2m_invalid:
e.p2m.xn = 1;
e.p2m.write = 0;
break;
case p2m_max_real_type:
BUG();
break;
}
ASSERT(!(pa & ~PAGE_MASK));
ASSERT(!(pa & ~PADDR_MASK));
e.bits |= pa;
return e;
}
/* Allocate a new page table page and hook it in via the given entry */
static int p2m_create_table(struct domain *d,
lpae_t *entry)
{
struct p2m_domain *p2m = &d->arch.p2m;
struct page_info *page;
void *p;
lpae_t pte;
BUG_ON(entry->p2m.valid);
page = alloc_domheap_page(NULL, 0);
if ( page == NULL )
return -ENOMEM;
page_list_add(page, &p2m->pages);
p = __map_domain_page(page);
clear_page(p);
unmap_domain_page(p);
pte = mfn_to_p2m_entry(page_to_mfn(page), MATTR_MEM, p2m_invalid);
write_pte(entry, pte);
return 0;
}
enum p2m_operation {
INSERT,
ALLOCATE,
REMOVE,
RELINQUISH,
CACHEFLUSH,
};
static int apply_p2m_changes(struct domain *d,
enum p2m_operation op,
paddr_t start_gpaddr,
paddr_t end_gpaddr,
paddr_t maddr,
int mattr,
p2m_type_t t)
{
int rc;
struct p2m_domain *p2m = &d->arch.p2m;
lpae_t *first = NULL, *second = NULL, *third = NULL;
paddr_t addr;
unsigned long cur_first_page = ~0,
cur_first_offset = ~0,
cur_second_offset = ~0;
unsigned long count = 0;
unsigned int flush = 0;
bool_t populate = (op == INSERT || op == ALLOCATE);
lpae_t pte;
spin_lock(&p2m->lock);
if ( d != current->domain )
p2m_load_VTTBR(d);
addr = start_gpaddr;
while ( addr < end_gpaddr )
{
if ( cur_first_page != p2m_first_level_index(addr) )
{
if ( first ) unmap_domain_page(first);
first = p2m_map_first(p2m, addr);
if ( !first )
{
rc = -EINVAL;
goto out;
}
cur_first_page = p2m_first_level_index(addr);
}
if ( !first[first_table_offset(addr)].p2m.valid )
{
if ( !populate )
{
addr = (addr + FIRST_SIZE) & FIRST_MASK;
continue;
}
rc = p2m_create_table(d, &first[first_table_offset(addr)]);
if ( rc < 0 )
{
printk("p2m_populate_ram: L1 failed\n");
goto out;
}
}
BUG_ON(!first[first_table_offset(addr)].p2m.valid);
if ( cur_first_offset != first_table_offset(addr) )
{
if (second) unmap_domain_page(second);
second = map_domain_page(first[first_table_offset(addr)].p2m.base);
cur_first_offset = first_table_offset(addr);
}
/* else: second already valid */
if ( !second[second_table_offset(addr)].p2m.valid )
{
if ( !populate )
{
addr = (addr + SECOND_SIZE) & SECOND_MASK;
continue;
}
rc = p2m_create_table(d, &second[second_table_offset(addr)]);
if ( rc < 0 ) {
printk("p2m_populate_ram: L2 failed\n");
goto out;
}
}
BUG_ON(!second[second_table_offset(addr)].p2m.valid);
if ( cur_second_offset != second_table_offset(addr) )
{
/* map third level */
if (third) unmap_domain_page(third);
third = map_domain_page(second[second_table_offset(addr)].p2m.base);
cur_second_offset = second_table_offset(addr);
}
pte = third[third_table_offset(addr)];
flush |= pte.p2m.valid;
/* TODO: Handle other p2m type
*
* It's safe to do the put_page here because page_alloc will
* flush the TLBs if the page is reallocated before the end of
* this loop.
*/
if ( pte.p2m.valid && p2m_is_foreign(pte.p2m.type) )
{
unsigned long mfn = pte.p2m.base;
ASSERT(mfn_valid(mfn));
put_page(mfn_to_page(mfn));
}
/* Allocate a new RAM page and attach */
switch (op) {
case ALLOCATE:
{
struct page_info *page;
ASSERT(!pte.p2m.valid);
rc = -ENOMEM;
page = alloc_domheap_page(d, 0);
if ( page == NULL ) {
printk("p2m_populate_ram: failed to allocate page\n");
goto out;
}
pte = mfn_to_p2m_entry(page_to_mfn(page), mattr, t);
write_pte(&third[third_table_offset(addr)], pte);
}
break;
case INSERT:
{
pte = mfn_to_p2m_entry(maddr >> PAGE_SHIFT, mattr, t);
write_pte(&third[third_table_offset(addr)], pte);
maddr += PAGE_SIZE;
}
break;
case RELINQUISH:
case REMOVE:
{
if ( !pte.p2m.valid )
{
count++;
break;
}
count += 0x10;
memset(&pte, 0x00, sizeof(pte));
write_pte(&third[third_table_offset(addr)], pte);
count++;
}
break;
case CACHEFLUSH:
{
if ( !pte.p2m.valid || !p2m_is_ram(pte.p2m.type) )
break;
flush_page_to_ram(pte.p2m.base);
}
break;
}
/* Preempt every 2MiB (mapped) or 32 MiB (unmapped) - arbitrary */
if ( op == RELINQUISH && count >= 0x2000 )
{
if ( hypercall_preempt_check() )
{
p2m->lowest_mapped_gfn = addr >> PAGE_SHIFT;
rc = -EAGAIN;
goto out;
}
count = 0;
}
/* Got the next page */
addr += PAGE_SIZE;
}
if ( flush )
{
/* At the beginning of the function, Xen is updating VTTBR
* with the domain where the mappings are created. In this
* case it's only necessary to flush TLBs on every CPUs with
* the current VMID (our domain).
*/
flush_tlb();
}
if ( op == ALLOCATE || op == INSERT )
{
unsigned long sgfn = paddr_to_pfn(start_gpaddr);
unsigned long egfn = paddr_to_pfn(end_gpaddr);
p2m->max_mapped_gfn = MAX(p2m->max_mapped_gfn, egfn);
p2m->lowest_mapped_gfn = MIN(p2m->lowest_mapped_gfn, sgfn);
}
rc = 0;
out:
if (third) unmap_domain_page(third);
if (second) unmap_domain_page(second);
if (first) unmap_domain_page(first);
if ( d != current->domain )
p2m_load_VTTBR(current->domain);
spin_unlock(&p2m->lock);
return rc;
}
static int create_p2m_entries(struct domain *d,
enum p2m_operation op,
paddr_t start_gpaddr,
paddr_t end_gpaddr,
paddr_t maddr,
int mattr)
{
int rc, flush;
struct p2m_domain *p2m = &d->arch.p2m;
lpae_t *first = NULL, *second = NULL, *third = NULL;
paddr_t addr;
unsigned long cur_first_offset = ~0, cur_second_offset = ~0;
spin_lock(&p2m->lock);
/* XXX Don't actually handle 40 bit guest physical addresses */
BUG_ON(start_gpaddr & 0x8000000000ULL);
BUG_ON(end_gpaddr & 0x8000000000ULL);
first = __map_domain_page(p2m->first_level);
for(addr = start_gpaddr; addr < end_gpaddr; addr += PAGE_SIZE)
{
if ( !first[first_table_offset(addr)].p2m.valid )
{
rc = p2m_create_table(d, &first[first_table_offset(addr)]);
if ( rc < 0 ) {
printk("p2m_populate_ram: L1 failed\n");
goto out;
}
}
BUG_ON(!first[first_table_offset(addr)].p2m.valid);
if ( cur_first_offset != first_table_offset(addr) )
{
if (second) unmap_domain_page(second);
second = map_domain_page(first[first_table_offset(addr)].p2m.base);
cur_first_offset = first_table_offset(addr);
}
/* else: second already valid */
if ( !second[second_table_offset(addr)].p2m.valid )
{
rc = p2m_create_table(d, &second[second_table_offset(addr)]);
if ( rc < 0 ) {
printk("p2m_populate_ram: L2 failed\n");
goto out;
}
}
BUG_ON(!second[second_table_offset(addr)].p2m.valid);
if ( cur_second_offset != second_table_offset(addr) )
{
/* map third level */
if (third) unmap_domain_page(third);
third = map_domain_page(second[second_table_offset(addr)].p2m.base);
cur_second_offset = second_table_offset(addr);
}
flush = third[third_table_offset(addr)].p2m.valid;
/* Allocate a new RAM page and attach */
switch (op) {
case ALLOCATE:
{
struct page_info *page;
lpae_t pte;
rc = -ENOMEM;
page = alloc_domheap_page(d, 0);
if ( page == NULL ) {
printk("p2m_populate_ram: failed to allocate page\n");
goto out;
}
pte = mfn_to_p2m_entry(page_to_mfn(page), mattr);
write_pte(&third[third_table_offset(addr)], pte);
}
break;
case INSERT:
{
lpae_t pte = mfn_to_p2m_entry(maddr >> PAGE_SHIFT, mattr);
write_pte(&third[third_table_offset(addr)], pte);
maddr += PAGE_SIZE;
}
break;
case REMOVE:
{
lpae_t pte;
memset(&pte, 0x00, sizeof(pte));
write_pte(&third[third_table_offset(addr)], pte);
maddr += PAGE_SIZE;
}
break;
}
if ( flush )
flush_tlb_all_local();
}
rc = 0;
out:
if (third) unmap_domain_page(third);
if (second) unmap_domain_page(second);
if (first) unmap_domain_page(first);
spin_unlock(&p2m->lock);
return rc;
}
void *vm_alloc(unsigned int nr, unsigned int align)
{
unsigned int start, bit;
if ( !align )
align = 1;
else if ( align & (align - 1) )
align &= -align;
spin_lock(&vm_lock);
for ( ; ; )
{
struct page_info *pg;
ASSERT(vm_low == vm_top || !test_bit(vm_low, vm_bitmap));
for ( start = vm_low; start < vm_top; )
{
bit = find_next_bit(vm_bitmap, vm_top, start + 1);
if ( bit > vm_top )
bit = vm_top;
/*
* Note that this skips the first bit, making the
* corresponding page a guard one.
*/
start = (start + align) & ~(align - 1);
if ( bit < vm_top )
{
if ( start + nr < bit )
break;
start = find_next_zero_bit(vm_bitmap, vm_top, bit + 1);
}
else
{
if ( start + nr <= bit )
break;
start = bit;
}
}
if ( start < vm_top )
break;
spin_unlock(&vm_lock);
if ( vm_top >= vm_end )
return NULL;
pg = alloc_domheap_page(NULL, 0);
if ( !pg )
return NULL;
spin_lock(&vm_lock);
if ( start >= vm_top )
{
unsigned long va = (unsigned long)vm_bitmap + vm_top / 8;
if ( !map_pages_to_xen(va, page_to_mfn(pg), 1, PAGE_HYPERVISOR) )
{
clear_page((void *)va);
vm_top += PAGE_SIZE * 8;
if ( vm_top > vm_end )
vm_top = vm_end;
continue;
}
}
free_domheap_page(pg);
if ( start >= vm_top )
{
spin_unlock(&vm_lock);
return NULL;
}
}
for ( bit = start; bit < start + nr; ++bit )
__set_bit(bit, vm_bitmap);
if ( bit < vm_top )
ASSERT(!test_bit(bit, vm_bitmap));
else
ASSERT(bit == vm_top);
if ( start <= vm_low + 2 )
vm_low = bit;
spin_unlock(&vm_lock);
return vm_base + start * PAGE_SIZE;
}
