static unsigned find_get_pages(unsigned long start, unsigned int nr_pages, struct page **pages) { unsigned int i; unsigned int ret; unsigned int nr_found; rcu_read_lock(); restart: nr_found = radix_tree_gang_lookup_slot(&mt_tree, (void ***)pages, NULL, start, nr_pages); ret = 0; for (i = 0; i < nr_found; i++) { struct page *page; repeat: page = radix_tree_deref_slot((void **)pages[i]); if (unlikely(!page)) continue; if (radix_tree_exception(page)) { if (radix_tree_deref_retry(page)) { /* * Transient condition which can only trigger * when entry at index 0 moves out of or back * to root: none yet gotten, safe to restart. */ assert((start | i) == 0); goto restart; } /* * No exceptional entries are inserted in this test. */ assert(0); } pthread_mutex_lock(&page->lock); if (!page->count) { pthread_mutex_unlock(&page->lock); goto repeat; } /* don't actually update page refcount */ pthread_mutex_unlock(&page->lock); /* Has the page moved? */ if (unlikely(page != *((void **)pages[i]))) { goto repeat; } pages[ret] = page; ret++; } rcu_read_unlock(); return ret; }
void uvm_pmm_sysmem_mappings_merge_gpu_mappings(uvm_pmm_sysmem_mappings_t *sysmem_mappings, NvU64 dma_addr, NvU64 new_region_size) { uvm_reverse_map_t *first_reverse_map; uvm_page_index_t running_page_index; NvU64 key; const NvU64 base_key = dma_addr / PAGE_SIZE; const size_t num_pages = new_region_size / PAGE_SIZE; size_t num_mapping_pages; UVM_ASSERT(IS_ALIGNED(dma_addr, new_region_size)); UVM_ASSERT(new_region_size <= UVM_VA_BLOCK_SIZE); UVM_ASSERT(is_power_of_2(new_region_size)); if (!sysmem_mappings->gpu->access_counters_supported) return; uvm_spin_lock(&sysmem_mappings->reverse_map_lock); // Find the first mapping in the region first_reverse_map = radix_tree_lookup(&sysmem_mappings->reverse_map_tree, base_key); UVM_ASSERT(first_reverse_map); num_mapping_pages = uvm_va_block_region_num_pages(first_reverse_map->region); UVM_ASSERT(num_pages >= num_mapping_pages); UVM_ASSERT(IS_ALIGNED(base_key, num_mapping_pages)); // The region in the tree matches the size of the merged region, just return if (num_pages == num_mapping_pages) goto unlock_no_update; // Otherwise update the rest of slots to point at the same reverse map // descriptor key = base_key + uvm_va_block_region_num_pages(first_reverse_map->region); running_page_index = first_reverse_map->region.outer; while (key < base_key + num_pages) { uvm_reverse_map_t *reverse_map = NULL; void **slot = radix_tree_lookup_slot(&sysmem_mappings->reverse_map_tree, key); size_t slot_index; UVM_ASSERT(slot); reverse_map = radix_tree_deref_slot(slot); UVM_ASSERT(reverse_map); UVM_ASSERT(reverse_map != first_reverse_map); UVM_ASSERT(reverse_map->va_block == first_reverse_map->va_block); UVM_ASSERT(reverse_map->owner == first_reverse_map->owner); UVM_ASSERT(reverse_map->region.first == running_page_index); NV_RADIX_TREE_REPLACE_SLOT(&sysmem_mappings->reverse_map_tree, slot, first_reverse_map); num_mapping_pages = uvm_va_block_region_num_pages(reverse_map->region); UVM_ASSERT(IS_ALIGNED(key, num_mapping_pages)); UVM_ASSERT(key + num_mapping_pages <= base_key + num_pages); for (slot_index = 1; slot_index < num_mapping_pages; ++slot_index) { slot = radix_tree_lookup_slot(&sysmem_mappings->reverse_map_tree, key + slot_index); UVM_ASSERT(slot); UVM_ASSERT(reverse_map == radix_tree_deref_slot(slot)); NV_RADIX_TREE_REPLACE_SLOT(&sysmem_mappings->reverse_map_tree, slot, first_reverse_map); } key += num_mapping_pages; running_page_index = reverse_map->region.outer; kmem_cache_free(g_reverse_page_map_cache, reverse_map); } // Grow the first mapping to cover the whole region first_reverse_map->region.outer = first_reverse_map->region.first + num_pages; unlock_no_update: uvm_spin_unlock(&sysmem_mappings->reverse_map_lock); }
NV_STATUS uvm_pmm_sysmem_mappings_split_gpu_mappings(uvm_pmm_sysmem_mappings_t *sysmem_mappings, NvU64 dma_addr, NvU64 new_region_size) { uvm_reverse_map_t *orig_reverse_map; const NvU64 base_key = dma_addr / PAGE_SIZE; const size_t num_pages = new_region_size / PAGE_SIZE; size_t old_num_pages; size_t subregion, num_subregions; uvm_reverse_map_t **new_reverse_maps; UVM_ASSERT(IS_ALIGNED(dma_addr, new_region_size)); UVM_ASSERT(new_region_size <= UVM_VA_BLOCK_SIZE); UVM_ASSERT(is_power_of_2(new_region_size)); if (!sysmem_mappings->gpu->access_counters_supported) return NV_OK; uvm_spin_lock(&sysmem_mappings->reverse_map_lock); orig_reverse_map = radix_tree_lookup(&sysmem_mappings->reverse_map_tree, base_key); uvm_spin_unlock(&sysmem_mappings->reverse_map_lock); // We can access orig_reverse_map outside the tree lock because we hold the // VA block lock so we cannot have concurrent modifications in the tree for // the mappings of the chunks that belong to that VA block. UVM_ASSERT(orig_reverse_map); UVM_ASSERT(orig_reverse_map->va_block); uvm_assert_mutex_locked(&orig_reverse_map->va_block->lock); old_num_pages = uvm_va_block_region_num_pages(orig_reverse_map->region); UVM_ASSERT(num_pages < old_num_pages); num_subregions = old_num_pages / num_pages; new_reverse_maps = uvm_kvmalloc_zero(sizeof(*new_reverse_maps) * (num_subregions - 1)); if (!new_reverse_maps) return NV_ERR_NO_MEMORY; // Allocate the descriptors for the new subregions for (subregion = 1; subregion < num_subregions; ++subregion) { uvm_reverse_map_t *new_reverse_map = kmem_cache_zalloc(g_reverse_page_map_cache, NV_UVM_GFP_FLAGS); uvm_page_index_t page_index = orig_reverse_map->region.first + num_pages * subregion; if (new_reverse_map == NULL) { // On error, free the previously-created descriptors while (--subregion != 0) kmem_cache_free(g_reverse_page_map_cache, new_reverse_maps[subregion - 1]); uvm_kvfree(new_reverse_maps); return NV_ERR_NO_MEMORY; } new_reverse_map->va_block = orig_reverse_map->va_block; new_reverse_map->region = uvm_va_block_region(page_index, page_index + num_pages); new_reverse_map->owner = orig_reverse_map->owner; new_reverse_maps[subregion - 1] = new_reverse_map; } uvm_spin_lock(&sysmem_mappings->reverse_map_lock); for (subregion = 1; subregion < num_subregions; ++subregion) { NvU64 key; for (key = base_key + num_pages * subregion; key < base_key + num_pages * (subregion + 1); ++key) { void **slot = radix_tree_lookup_slot(&sysmem_mappings->reverse_map_tree, key); UVM_ASSERT(slot); UVM_ASSERT(radix_tree_deref_slot(slot) == orig_reverse_map); NV_RADIX_TREE_REPLACE_SLOT(&sysmem_mappings->reverse_map_tree, slot, new_reverse_maps[subregion - 1]); } } orig_reverse_map->region = uvm_va_block_region(orig_reverse_map->region.first, orig_reverse_map->region.first + num_pages); uvm_spin_unlock(&sysmem_mappings->reverse_map_lock); uvm_kvfree(new_reverse_maps); return NV_OK; }