static unsigned find_get_pages(unsigned long start,
unsigned int nr_pages, struct page **pages)
{
unsigned int i;
unsigned int ret;
unsigned int nr_found;
rcu_read_lock();
restart:
nr_found = radix_tree_gang_lookup_slot(&mt_tree,
(void ***)pages, NULL, start, nr_pages);
ret = 0;
for (i = 0; i < nr_found; i++) {
struct page *page;
repeat:
page = radix_tree_deref_slot((void **)pages[i]);
if (unlikely(!page))
continue;
if (radix_tree_exception(page)) {
if (radix_tree_deref_retry(page)) {
/*
* Transient condition which can only trigger
* when entry at index 0 moves out of or back
* to root: none yet gotten, safe to restart.
*/
assert((start | i) == 0);
goto restart;
}
/*
* No exceptional entries are inserted in this test.
*/
assert(0);
}
pthread_mutex_lock(&page->lock);
if (!page->count) {
pthread_mutex_unlock(&page->lock);
goto repeat;
}
/* don't actually update page refcount */
pthread_mutex_unlock(&page->lock);
/* Has the page moved? */
if (unlikely(page != *((void **)pages[i]))) {
goto repeat;
}
pages[ret] = page;
ret++;
}
rcu_read_unlock();
return ret;
}
void uvm_pmm_sysmem_mappings_merge_gpu_mappings(uvm_pmm_sysmem_mappings_t *sysmem_mappings,
NvU64 dma_addr,
NvU64 new_region_size)
{
uvm_reverse_map_t *first_reverse_map;
uvm_page_index_t running_page_index;
NvU64 key;
const NvU64 base_key = dma_addr / PAGE_SIZE;
const size_t num_pages = new_region_size / PAGE_SIZE;
size_t num_mapping_pages;
UVM_ASSERT(IS_ALIGNED(dma_addr, new_region_size));
UVM_ASSERT(new_region_size <= UVM_VA_BLOCK_SIZE);
UVM_ASSERT(is_power_of_2(new_region_size));
if (!sysmem_mappings->gpu->access_counters_supported)
return;
uvm_spin_lock(&sysmem_mappings->reverse_map_lock);
// Find the first mapping in the region
first_reverse_map = radix_tree_lookup(&sysmem_mappings->reverse_map_tree, base_key);
UVM_ASSERT(first_reverse_map);
num_mapping_pages = uvm_va_block_region_num_pages(first_reverse_map->region);
UVM_ASSERT(num_pages >= num_mapping_pages);
UVM_ASSERT(IS_ALIGNED(base_key, num_mapping_pages));
// The region in the tree matches the size of the merged region, just return
if (num_pages == num_mapping_pages)
goto unlock_no_update;
// Otherwise update the rest of slots to point at the same reverse map
// descriptor
key = base_key + uvm_va_block_region_num_pages(first_reverse_map->region);
running_page_index = first_reverse_map->region.outer;
while (key < base_key + num_pages) {
uvm_reverse_map_t *reverse_map = NULL;
void **slot = radix_tree_lookup_slot(&sysmem_mappings->reverse_map_tree, key);
size_t slot_index;
UVM_ASSERT(slot);
reverse_map = radix_tree_deref_slot(slot);
UVM_ASSERT(reverse_map);
UVM_ASSERT(reverse_map != first_reverse_map);
UVM_ASSERT(reverse_map->va_block == first_reverse_map->va_block);
UVM_ASSERT(reverse_map->owner == first_reverse_map->owner);
UVM_ASSERT(reverse_map->region.first == running_page_index);
NV_RADIX_TREE_REPLACE_SLOT(&sysmem_mappings->reverse_map_tree, slot, first_reverse_map);
num_mapping_pages = uvm_va_block_region_num_pages(reverse_map->region);
UVM_ASSERT(IS_ALIGNED(key, num_mapping_pages));
UVM_ASSERT(key + num_mapping_pages <= base_key + num_pages);
for (slot_index = 1; slot_index < num_mapping_pages; ++slot_index) {
slot = radix_tree_lookup_slot(&sysmem_mappings->reverse_map_tree, key + slot_index);
UVM_ASSERT(slot);
UVM_ASSERT(reverse_map == radix_tree_deref_slot(slot));
NV_RADIX_TREE_REPLACE_SLOT(&sysmem_mappings->reverse_map_tree, slot, first_reverse_map);
}
key += num_mapping_pages;
running_page_index = reverse_map->region.outer;
kmem_cache_free(g_reverse_page_map_cache, reverse_map);
}
// Grow the first mapping to cover the whole region
first_reverse_map->region.outer = first_reverse_map->region.first + num_pages;
unlock_no_update:
uvm_spin_unlock(&sysmem_mappings->reverse_map_lock);
}
NV_STATUS uvm_pmm_sysmem_mappings_split_gpu_mappings(uvm_pmm_sysmem_mappings_t *sysmem_mappings,
NvU64 dma_addr,
NvU64 new_region_size)
{
uvm_reverse_map_t *orig_reverse_map;
const NvU64 base_key = dma_addr / PAGE_SIZE;
const size_t num_pages = new_region_size / PAGE_SIZE;
size_t old_num_pages;
size_t subregion, num_subregions;
uvm_reverse_map_t **new_reverse_maps;
UVM_ASSERT(IS_ALIGNED(dma_addr, new_region_size));
UVM_ASSERT(new_region_size <= UVM_VA_BLOCK_SIZE);
UVM_ASSERT(is_power_of_2(new_region_size));
if (!sysmem_mappings->gpu->access_counters_supported)
return NV_OK;
uvm_spin_lock(&sysmem_mappings->reverse_map_lock);
orig_reverse_map = radix_tree_lookup(&sysmem_mappings->reverse_map_tree, base_key);
uvm_spin_unlock(&sysmem_mappings->reverse_map_lock);
// We can access orig_reverse_map outside the tree lock because we hold the
// VA block lock so we cannot have concurrent modifications in the tree for
// the mappings of the chunks that belong to that VA block.
UVM_ASSERT(orig_reverse_map);
UVM_ASSERT(orig_reverse_map->va_block);
uvm_assert_mutex_locked(&orig_reverse_map->va_block->lock);
old_num_pages = uvm_va_block_region_num_pages(orig_reverse_map->region);
UVM_ASSERT(num_pages < old_num_pages);
num_subregions = old_num_pages / num_pages;
new_reverse_maps = uvm_kvmalloc_zero(sizeof(*new_reverse_maps) * (num_subregions - 1));
if (!new_reverse_maps)
return NV_ERR_NO_MEMORY;
// Allocate the descriptors for the new subregions
for (subregion = 1; subregion < num_subregions; ++subregion) {
uvm_reverse_map_t *new_reverse_map = kmem_cache_zalloc(g_reverse_page_map_cache, NV_UVM_GFP_FLAGS);
uvm_page_index_t page_index = orig_reverse_map->region.first + num_pages * subregion;
if (new_reverse_map == NULL) {
// On error, free the previously-created descriptors
while (--subregion != 0)
kmem_cache_free(g_reverse_page_map_cache, new_reverse_maps[subregion - 1]);
uvm_kvfree(new_reverse_maps);
return NV_ERR_NO_MEMORY;
}
new_reverse_map->va_block = orig_reverse_map->va_block;
new_reverse_map->region = uvm_va_block_region(page_index, page_index + num_pages);
new_reverse_map->owner = orig_reverse_map->owner;
new_reverse_maps[subregion - 1] = new_reverse_map;
}
uvm_spin_lock(&sysmem_mappings->reverse_map_lock);
for (subregion = 1; subregion < num_subregions; ++subregion) {
NvU64 key;
for (key = base_key + num_pages * subregion; key < base_key + num_pages * (subregion + 1); ++key) {
void **slot = radix_tree_lookup_slot(&sysmem_mappings->reverse_map_tree, key);
UVM_ASSERT(slot);
UVM_ASSERT(radix_tree_deref_slot(slot) == orig_reverse_map);
NV_RADIX_TREE_REPLACE_SLOT(&sysmem_mappings->reverse_map_tree, slot, new_reverse_maps[subregion - 1]);
}
}
orig_reverse_map->region = uvm_va_block_region(orig_reverse_map->region.first,
orig_reverse_map->region.first + num_pages);
uvm_spin_unlock(&sysmem_mappings->reverse_map_lock);
uvm_kvfree(new_reverse_maps);
return NV_OK;
}
