/* * Insert block at <page, offset> in freelist of given pool. * freelist used depends on block size. */ static void insert_block(struct xv_pool *pool, struct page *page, u32 offset, struct block_header *block) { u32 flindex, slindex; struct block_header *nextblock; slindex = get_index_for_insert(block->size); flindex = slindex / BITS_PER_LONG; block->link.prev_page = NULL; block->link.prev_offset = 0; block->link.next_page = pool->freelist[slindex].page; block->link.next_offset = pool->freelist[slindex].offset; pool->freelist[slindex].page = page; pool->freelist[slindex].offset = offset; if (block->link.next_page) { nextblock = get_ptr_atomic(block->link.next_page, block->link.next_offset); nextblock->link.prev_page = page; nextblock->link.prev_offset = offset; put_ptr_atomic(nextblock); /* If there was a next page then the free bits are set. */ return; } __set_bit(slindex % BITS_PER_LONG, &pool->slbitmap[flindex]); __set_bit(flindex, &pool->flbitmap); }
/* * Insert block at <pagenum, offset> in freelist of given pool. * freelist used depends on block size. */ static void insert_block(struct xv_pool *pool, u32 pagenum, u32 offset, struct block_header *block) { u32 flindex, slindex; struct block_header *nextblock; slindex = get_index_for_insert(block->size); flindex = slindex / BITS_PER_LONG; block->link.prev_pagenum = 0; block->link.prev_offset = 0; block->link.next_pagenum = pool->freelist[slindex].pagenum; block->link.next_offset = pool->freelist[slindex].offset; pool->freelist[slindex].pagenum = pagenum; pool->freelist[slindex].offset = offset; if (block->link.next_pagenum) { nextblock = get_ptr_atomic(block->link.next_pagenum, block->link.next_offset, KM_USER1); nextblock->link.prev_pagenum = pagenum; nextblock->link.prev_offset = offset; put_ptr_atomic(nextblock, KM_USER1); } __set_bit(slindex % BITS_PER_LONG, &pool->slbitmap[flindex]); __set_bit(flindex, &pool->flbitmap); }
/* * Create a memory pool. Allocates freelist, bitmaps and other * per-pool metadata. */ struct xv_pool *xv_create_pool(void) { u32 ovhd_size; struct xv_pool *pool; ovhd_size = roundup(sizeof(*pool), PAGE_SIZE); pool = kzalloc(ovhd_size, GFP_KERNEL); if (!pool) return NULL; /* cache the first/second-level indices for PAGE_SIZE allocations */ pagesize_slindex = get_index_for_insert(PAGE_SIZE); pagesize_flindex = pagesize_slindex / BITS_PER_LONG; spin_lock_init(&pool->lock); return pool; }
/* * Free block identified with <page, offset> */ void xv_free(struct xv_pool *pool, struct page *page, u32 offset) { void *page_start; struct block_header *block, *tmpblock; offset -= XV_ALIGN; spin_lock(&pool->lock); page_start = get_ptr_atomic(page, 0); block = (struct block_header *)((char *)page_start + offset); /* Catch double free bugs */ BUG_ON(test_flag(block, BLOCK_FREE)); block->size = ALIGN(block->size, XV_ALIGN); tmpblock = BLOCK_NEXT(block); if (offset + block->size + XV_ALIGN == PAGE_SIZE) tmpblock = NULL; /* Merge next block if its free */ if (tmpblock && test_flag(tmpblock, BLOCK_FREE)) { /* * Blocks smaller than XV_MIN_ALLOC_SIZE * are not inserted in any free list. */ if (tmpblock->size >= XV_MIN_ALLOC_SIZE) { remove_block(pool, page, offset + block->size + XV_ALIGN, tmpblock, get_index_for_insert(tmpblock->size)); } block->size += tmpblock->size + XV_ALIGN; } /* Merge previous block if its free */ if (test_flag(block, PREV_FREE)) { tmpblock = (struct block_header *)((char *)(page_start) + get_blockprev(block)); offset = offset - tmpblock->size - XV_ALIGN; if (tmpblock->size >= XV_MIN_ALLOC_SIZE) remove_block(pool, page, offset, tmpblock, get_index_for_insert(tmpblock->size)); tmpblock->size += block->size + XV_ALIGN; block = tmpblock; } /* No used objects in this page. Free it. */ if (block->size == PAGE_SIZE - XV_ALIGN) { put_ptr_atomic(page_start); spin_unlock(&pool->lock); __free_page(page); stat_dec(&pool->total_pages); return; } set_flag(block, BLOCK_FREE); if (block->size >= XV_MIN_ALLOC_SIZE) insert_block(pool, page, offset, block); if (offset + block->size + XV_ALIGN != PAGE_SIZE) { tmpblock = BLOCK_NEXT(block); set_flag(tmpblock, PREV_FREE); set_blockprev(tmpblock, offset); } put_ptr_atomic(page_start); spin_unlock(&pool->lock); }