static struct page_info * __init alloc_chunk(
struct domain *d, unsigned long max_pages)
{
static unsigned int __initdata last_order = MAX_ORDER;
static unsigned int __initdata memflags = MEMF_no_dma;
struct page_info *page;
unsigned int order = get_order_from_pages(max_pages), free_order;
if ( order > last_order )
order = last_order;
else if ( max_pages & (max_pages - 1) )
--order;
while ( (page = alloc_domheap_pages(d, order, memflags)) == NULL )
if ( order-- == 0 )
break;
if ( page )
last_order = order;
else if ( memflags )
{
/*
* Allocate up to 2MB at a time: It prevents allocating very large
* chunks from DMA pools before the >4GB pool is fully depleted.
*/
last_order = 21 - PAGE_SHIFT;
memflags = 0;
return alloc_chunk(d, max_pages);
}
/*
* Make a reasonable attempt at finding a smaller chunk at a higher
* address, to avoid allocating from low memory as much as possible.
*/
for ( free_order = order; !memflags && page && order--; )
{
struct page_info *pg2;
if ( d->tot_pages + (1 << order) > d->max_pages )
continue;
pg2 = alloc_domheap_pages(d, order, 0);
if ( pg2 > page )
{
free_domheap_pages(page, free_order);
page = pg2;
free_order = order;
}
else if ( pg2 )
free_domheap_pages(pg2, order);
}
return page;
}
int p2m_alloc_table(struct domain *d)
{
struct p2m_domain *p2m = &d->arch.p2m;
struct page_info *page;
void *p;
/* First level P2M is 2 consecutive pages */
page = alloc_domheap_pages(NULL, 1, 0);
if ( page == NULL )
return -ENOMEM;
spin_lock(&p2m->lock);
page_list_add(page, &p2m->pages);
/* Clear both first level pages */
p = __map_domain_page(page);
clear_page(p);
unmap_domain_page(p);
p = __map_domain_page(page + 1);
clear_page(p);
unmap_domain_page(p);
p2m->first_level = page;
d->arch.vttbr = page_to_maddr(p2m->first_level)
| ((uint64_t)p2m->vmid&0xff)<<48;
spin_unlock(&p2m->lock);
return 0;
}
int p2m_alloc_table(struct domain *d)
{
struct p2m_domain *p2m = &d->arch.p2m;
struct page_info *page;
void *p;
page = alloc_domheap_pages(NULL, P2M_FIRST_ORDER, 0);
if ( page == NULL )
return -ENOMEM;
spin_lock(&p2m->lock);
/* Clear both first level pages */
p = __map_domain_page(page);
clear_page(p);
unmap_domain_page(p);
p = __map_domain_page(page + 1);
clear_page(p);
unmap_domain_page(p);
p2m->first_level = page;
d->arch.vttbr = page_to_maddr(p2m->first_level)
| ((uint64_t)p2m->vmid&0xff)<<48;
/* Make sure that all TLBs corresponding to the new VMID are flushed
* before using it
*/
flush_tlb_domain(d);
spin_unlock(&p2m->lock);
return 0;
}
static void populate_physmap(struct memop_args *a)
{
struct page_info *page;
unsigned long i, j;
xen_pfn_t gpfn, mfn;
struct domain *d = a->domain;
if ( !guest_handle_subrange_okay(a->extent_list, a->nr_done,
a->nr_extents-1) )
return;
if ( (a->extent_order != 0) &&
!multipage_allocation_permitted(current->domain) )
return;
for ( i = a->nr_done; i < a->nr_extents; i++ )
{
if ( hypercall_preempt_check() )
{
a->preempted = 1;
goto out;
}
if ( unlikely(__copy_from_guest_offset(&gpfn, a->extent_list, i, 1)) )
goto out;
page = alloc_domheap_pages(d, a->extent_order, a->memflags);
if ( unlikely(page == NULL) )
{
gdprintk(XENLOG_INFO, "Could not allocate order=%d extent: "
"id=%d memflags=%x (%ld of %d)\n",
a->extent_order, d->domain_id, a->memflags,
i, a->nr_extents);
goto out;
}
mfn = page_to_mfn(page);
guest_physmap_add_page(d, gpfn, mfn, a->extent_order);
if ( !paging_mode_translate(d) )
{
for ( j = 0; j < (1 << a->extent_order); j++ )
set_gpfn_from_mfn(mfn + j, gpfn + j);
/* Inform the domain of the new page's machine address. */
if ( unlikely(__copy_to_guest_offset(a->extent_list, i, &mfn, 1)) )
goto out;
}
}
out:
a->nr_done = i;
}
/* Populate a HVM memory range using the biggest possible order. */
static int __init pvh_populate_memory_range(struct domain *d,
unsigned long start,
unsigned long nr_pages)
{
unsigned int order, i = 0;
struct page_info *page;
int rc;
#define MAP_MAX_ITER 64
order = MAX_ORDER;
while ( nr_pages != 0 )
{
unsigned int range_order = get_order_from_pages(nr_pages + 1);
order = min(range_order ? range_order - 1 : 0, order);
page = alloc_domheap_pages(d, order, dom0_memflags);
if ( page == NULL )
{
if ( order == 0 && dom0_memflags )
{
/* Try again without any dom0_memflags. */
dom0_memflags = 0;
order = MAX_ORDER;
continue;
}
if ( order == 0 )
{
printk("Unable to allocate memory with order 0!\n");
return -ENOMEM;
}
order--;
continue;
}
rc = guest_physmap_add_page(d, _gfn(start), _mfn(page_to_mfn(page)),
order);
if ( rc != 0 )
{
printk("Failed to populate memory: [%#lx,%lx): %d\n",
start, start + (1UL << order), rc);
return -ENOMEM;
}
start += 1UL << order;
nr_pages -= 1UL << order;
if ( (++i % MAP_MAX_ITER) == 0 )
process_pending_softirqs();
}
return 0;
#undef MAP_MAX_ITER
}
static void increase_reservation(struct memop_args *a)
{
struct page_info *page;
unsigned long i;
xen_pfn_t mfn;
struct domain *d = a->domain;
if ( !guest_handle_is_null(a->extent_list) &&
!guest_handle_subrange_okay(a->extent_list, a->nr_done,
a->nr_extents-1) )
return;
if ( !multipage_allocation_permitted(current->domain, a->extent_order) )
return;
mcd_mem_inc_trap(a->domain, (a->nr_extents - a->nr_done));
for ( i = a->nr_done; i < a->nr_extents; i++ )
{
if ( hypercall_preempt_check() )
{
a->preempted = 1;
goto out;
}
page = alloc_domheap_pages(d, a->extent_order, a->memflags);
if ( unlikely(page == NULL) )
{
gdprintk(XENLOG_INFO, "Could not allocate order=%d extent: "
"id=%d memflags=%x (%ld of %d)\n",
a->extent_order, d->domain_id, a->memflags,
i, a->nr_extents);
goto out;
}
/* Inform the domain of the new page's machine address. */
if ( !guest_handle_is_null(a->extent_list) )
{
mfn = page_to_mfn(page);
if ( unlikely(__copy_to_guest_offset(a->extent_list, i, &mfn, 1)) )
goto out;
}
}
out:
a->nr_done = i;
mcd_mem_upt_trap(d);
}
static struct page_info * __init alloc_chunk(
struct domain *d, unsigned long max_pages)
{
struct page_info *page;
unsigned int order;
/*
* Allocate up to 2MB at a time: It prevents allocating very large chunks
* from DMA pools before the >4GB pool is fully depleted.
*/
if ( max_pages > (2UL << (20 - PAGE_SHIFT)) )
max_pages = 2UL << (20 - PAGE_SHIFT);
order = get_order_from_pages(max_pages);
if ( (max_pages & (max_pages-1)) != 0 )
order--;
while ( (page = alloc_domheap_pages(d, order, 0)) == NULL )
if ( order-- == 0 )
break;
return page;
}
static struct page_info *hap_alloc_p2m_page(struct domain *d)
{
struct page_info *pg;
hap_lock(d);
pg = hap_alloc(d);
#if CONFIG_PAGING_LEVELS == 3
/* Under PAE mode, top-level P2M table should be allocated below 4GB space
* because the size of h_cr3 is only 32-bit. We use alloc_domheap_pages to
* force this requirement, and exchange the guaranteed 32-bit-clean
* page for the one we just hap_alloc()ed. */
if ( d->arch.paging.hap.p2m_pages == 0
&& mfn_x(page_to_mfn(pg)) >= (1UL << (32 - PAGE_SHIFT)) )
{
free_domheap_page(pg);
pg = alloc_domheap_pages(NULL, 0, MEMF_bits(32));
if ( likely(pg != NULL) )
{
void *p = hap_map_domain_page(page_to_mfn(pg));
clear_page(p);
hap_unmap_domain_page(p);
}
}
#endif
if ( likely(pg != NULL) )
{
d->arch.paging.hap.total_pages--;
d->arch.paging.hap.p2m_pages++;
page_set_owner(pg, d);
pg->count_info = 1;
}
hap_unlock(d);
return pg;
}
static void populate_physmap(struct memop_args *a)
{
struct page_info *page;
unsigned int i, j;
xen_pfn_t gpfn, mfn;
struct domain *d = a->domain;
if ( !guest_handle_subrange_okay(a->extent_list, a->nr_done,
a->nr_extents-1) )
return;
if ( a->extent_order > (a->memflags & MEMF_populate_on_demand ? MAX_ORDER :
max_order(current->domain)) )
return;
for ( i = a->nr_done; i < a->nr_extents; i++ )
{
if ( i != a->nr_done && hypercall_preempt_check() )
{
a->preempted = 1;
goto out;
}
if ( unlikely(__copy_from_guest_offset(&gpfn, a->extent_list, i, 1)) )
goto out;
if ( a->memflags & MEMF_populate_on_demand )
{
if ( guest_physmap_mark_populate_on_demand(d, gpfn,
a->extent_order) < 0 )
goto out;
}
else
{
if ( is_domain_direct_mapped(d) )
{
mfn = gpfn;
for ( j = 0; j < (1U << a->extent_order); j++, mfn++ )
{
if ( !mfn_valid(mfn) )
{
gdprintk(XENLOG_INFO, "Invalid mfn %#"PRI_xen_pfn"\n",
mfn);
goto out;
}
page = mfn_to_page(mfn);
if ( !get_page(page, d) )
{
gdprintk(XENLOG_INFO,
"mfn %#"PRI_xen_pfn" doesn't belong to d%d\n",
mfn, d->domain_id);
goto out;
}
put_page(page);
}
mfn = gpfn;
page = mfn_to_page(mfn);
}
else
{
page = alloc_domheap_pages(d, a->extent_order, a->memflags);
if ( unlikely(!page) )
{
if ( !opt_tmem || a->extent_order )
gdprintk(XENLOG_INFO,
"Could not allocate order=%u extent: id=%d memflags=%#x (%u of %u)\n",
a->extent_order, d->domain_id, a->memflags,
i, a->nr_extents);
goto out;
}
mfn = page_to_mfn(page);
}
guest_physmap_add_page(d, gpfn, mfn, a->extent_order);
if ( !paging_mode_translate(d) )
{
for ( j = 0; j < (1U << a->extent_order); j++ )
set_gpfn_from_mfn(mfn + j, gpfn + j);
/* Inform the domain of the new page's machine address. */
if ( unlikely(__copy_to_guest_offset(a->extent_list, i, &mfn, 1)) )
goto out;
}
}
}
out:
a->nr_done = i;
}
/*
* 0 == (P2M_ONE_DESCEND) continue to descend the tree
* +ve == (P2M_ONE_PROGRESS_*) handled at this level, continue, flush,
* entry, addr and maddr updated. Return value is an
* indication of the amount of work done (for preemption).
* -ve == (-Exxx) error.
*/
static int apply_one_level(struct domain *d,
lpae_t *entry,
unsigned int level,
bool_t flush_cache,
enum p2m_operation op,
paddr_t start_gpaddr,
paddr_t end_gpaddr,
paddr_t *addr,
paddr_t *maddr,
bool_t *flush,
int mattr,
p2m_type_t t,
p2m_access_t a)
{
const paddr_t level_size = level_sizes[level];
const paddr_t level_mask = level_masks[level];
const paddr_t level_shift = level_shifts[level];
struct p2m_domain *p2m = &d->arch.p2m;
lpae_t pte;
const lpae_t orig_pte = *entry;
int rc;
BUG_ON(level > 3);
switch ( op )
{
case ALLOCATE:
ASSERT(level < 3 || !p2m_valid(orig_pte));
ASSERT(*maddr == 0);
if ( p2m_valid(orig_pte) )
return P2M_ONE_DESCEND;
if ( is_mapping_aligned(*addr, end_gpaddr, 0, level_size) &&
/* We only create superpages when mem_access is not in use. */
(level == 3 || (level < 3 && !p2m->mem_access_enabled)) )
{
struct page_info *page;
page = alloc_domheap_pages(d, level_shift - PAGE_SHIFT, 0);
if ( page )
{
rc = p2m_mem_access_radix_set(p2m, paddr_to_pfn(*addr), a);
if ( rc < 0 )
{
free_domheap_page(page);
return rc;
}
pte = mfn_to_p2m_entry(page_to_mfn(page), mattr, t, a);
if ( level < 3 )
pte.p2m.table = 0;
p2m_write_pte(entry, pte, flush_cache);
p2m->stats.mappings[level]++;
*addr += level_size;
return P2M_ONE_PROGRESS;
}
else if ( level == 3 )
return -ENOMEM;
}
/* L3 is always suitably aligned for mapping (handled, above) */
BUG_ON(level == 3);
/*
* If we get here then we failed to allocate a sufficiently
* large contiguous region for this level (which can't be
* L3) or mem_access is in use. Create a page table and
* continue to descend so we try smaller allocations.
*/
rc = p2m_create_table(d, entry, 0, flush_cache);
if ( rc < 0 )
return rc;
return P2M_ONE_DESCEND;
case INSERT:
if ( is_mapping_aligned(*addr, end_gpaddr, *maddr, level_size) &&
/*
* We do not handle replacing an existing table with a superpage
* or when mem_access is in use.
*/
(level == 3 || (!p2m_table(orig_pte) && !p2m->mem_access_enabled)) )
{
rc = p2m_mem_access_radix_set(p2m, paddr_to_pfn(*addr), a);
if ( rc < 0 )
return rc;
/* New mapping is superpage aligned, make it */
pte = mfn_to_p2m_entry(*maddr >> PAGE_SHIFT, mattr, t, a);
if ( level < 3 )
pte.p2m.table = 0; /* Superpage entry */
p2m_write_pte(entry, pte, flush_cache);
*flush |= p2m_valid(orig_pte);
*addr += level_size;
*maddr += level_size;
if ( p2m_valid(orig_pte) )
{
/*
* We can't currently get here for an existing table
* mapping, since we don't handle replacing an
* existing table with a superpage. If we did we would
* need to handle freeing (and accounting) for the bit
* of the p2m tree which we would be about to lop off.
*/
BUG_ON(level < 3 && p2m_table(orig_pte));
if ( level == 3 )
p2m_put_l3_page(orig_pte);
}
else /* New mapping */
p2m->stats.mappings[level]++;
return P2M_ONE_PROGRESS;
}
else
{
