static int wmem_block_verify_recycler(wmem_block_allocator_t *allocator) { wmem_block_chunk_t *cur; wmem_block_free_t *cur_free; int free_space = 0; cur = allocator->recycler_head; if (!cur) { return 0; } do { free_space += cur->len; cur_free = WMEM_GET_FREE(cur); g_assert(! cur->used); g_assert(cur_free->prev); g_assert(cur_free->next); g_assert(WMEM_GET_FREE(cur_free->prev)->next == cur); g_assert(WMEM_GET_FREE(cur_free->next)->prev == cur); cur = cur_free->next; } while (cur != allocator->recycler_head); return free_space; }
/* Adds a chunk from the recycler. */ static void wmem_block_add_to_recycler(wmem_block_allocator_t *allocator, wmem_block_chunk_t *chunk) { wmem_block_free_t *free_chunk; if (WMEM_CHUNK_DATA_LEN(chunk) < sizeof(wmem_block_free_t)) { return; } free_chunk = WMEM_GET_FREE(chunk); if (! allocator->recycler_head) { /* First one */ free_chunk->next = chunk; free_chunk->prev = chunk; allocator->recycler_head = chunk; } else { free_chunk->next = allocator->recycler_head; free_chunk->prev = WMEM_GET_FREE(allocator->recycler_head)->prev; WMEM_GET_FREE(free_chunk->next)->prev = chunk; WMEM_GET_FREE(free_chunk->prev)->next = chunk; if (chunk->len > allocator->recycler_head->len) { allocator->recycler_head = chunk; } } }
static int wmem_block_verify_master_list(wmem_block_allocator_t *allocator) { wmem_block_chunk_t *cur; wmem_block_free_t *cur_free; int free_space = 0; cur = allocator->master_head; if (!cur) { return 0; } g_assert(WMEM_GET_FREE(cur)->prev == NULL); while (cur) { free_space += cur->len; cur_free = WMEM_GET_FREE(cur); g_assert(! cur->used); if (cur_free->next) { g_assert(WMEM_GET_FREE(cur_free->next)->prev == cur); } if (cur != allocator->master_head) { g_assert(cur->len == WMEM_BLOCK_SIZE); } cur = cur_free->next; } return free_space; }
/* Removes a chunk from the recycler. */ static void wmem_block_remove_from_recycler(wmem_block_allocator_t *allocator, wmem_block_chunk_t *chunk) { wmem_block_free_t *free_chunk; g_assert (! chunk->used); free_chunk = WMEM_GET_FREE(chunk); g_assert(free_chunk->prev && free_chunk->next); if (free_chunk->prev == chunk && free_chunk->next == chunk) { /* Only one item in recycler, just empty it. */ g_assert(allocator->recycler_head == chunk); allocator->recycler_head = NULL; } else { /* Two or more items, usual doubly-linked-list removal. It's circular * so we don't need to worry about null-checking anything, which is * nice. */ WMEM_GET_FREE(free_chunk->prev)->next = free_chunk->next; WMEM_GET_FREE(free_chunk->next)->prev = free_chunk->prev; if (allocator->recycler_head == chunk) { allocator->recycler_head = free_chunk->next; } } }
/* Pushes a chunk onto the master stack. */ static void wmem_block_push_master(wmem_block_allocator_t *allocator, wmem_block_chunk_t *chunk) { wmem_block_free_t *free_chunk; free_chunk = WMEM_GET_FREE(chunk); free_chunk->prev = NULL; free_chunk->next = allocator->master_head; if (free_chunk->next) { WMEM_GET_FREE(free_chunk->next)->prev = chunk; } allocator->master_head = chunk; }
static void wmem_block_gc(void *private_data) { wmem_block_allocator_t *allocator = (wmem_block_allocator_t*) private_data; wmem_block_hdr_t *cur, *next; wmem_block_chunk_t *chunk; wmem_block_free_t *free_chunk; /* Walk through the blocks, adding used blocks to the new list and * completely destroying unused blocks. */ cur = allocator->block_list; allocator->block_list = NULL; while (cur) { chunk = WMEM_BLOCK_TO_CHUNK(cur); next = cur->next; if (!chunk->jumbo && !chunk->used && chunk->last) { /* If the first chunk is also the last, and is unused, then * the block as a whole is entirely unused, so return it to * the OS and remove it from whatever lists it is in. */ free_chunk = WMEM_GET_FREE(chunk); if (free_chunk->next) { WMEM_GET_FREE(free_chunk->next)->prev = free_chunk->prev; } if (free_chunk->prev) { WMEM_GET_FREE(free_chunk->prev)->next = free_chunk->next; } if (allocator->recycler_head == chunk) { if (free_chunk->next == chunk) { allocator->recycler_head = NULL; } else { allocator->recycler_head = free_chunk->next; } } else if (allocator->master_head == chunk) { allocator->master_head = free_chunk->next; } wmem_free(NULL, cur); } else { /* part of this block is used, so add it to the new block list */ wmem_block_add_to_block_list(allocator, cur); } cur = next; } }
/* Removes the top chunk from the master stack. */ static void wmem_block_pop_master(wmem_block_allocator_t *allocator) { wmem_block_chunk_t *chunk; wmem_block_free_t *free_chunk; chunk = allocator->master_head; free_chunk = WMEM_GET_FREE(chunk); allocator->master_head = free_chunk->next; if (free_chunk->next) { WMEM_GET_FREE(free_chunk->next)->prev = NULL; } }
/* DEBUG AND TEST */ static int wmem_block_verify_block(wmem_block_hdr_t *block) { int total_free_space = 0; guint32 total_len; wmem_block_chunk_t *chunk; chunk = WMEM_BLOCK_TO_CHUNK(block); total_len = WMEM_BLOCK_HEADER_SIZE; if (chunk->jumbo) { /* We can tell nothing else about jumbo chunks except that they are * always used. */ return 0; } g_assert(chunk->prev == 0); do { total_len += chunk->len; g_assert(chunk->len >= WMEM_CHUNK_HEADER_SIZE); g_assert(!chunk->jumbo); if (WMEM_CHUNK_NEXT(chunk)) { g_assert(chunk->len == WMEM_CHUNK_NEXT(chunk)->prev); } if (!chunk->used && WMEM_CHUNK_DATA_LEN(chunk) >= sizeof(wmem_block_free_t)) { total_free_space += chunk->len; if (!chunk->last) { g_assert(WMEM_GET_FREE(chunk)->next); g_assert(WMEM_GET_FREE(chunk)->prev); } } chunk = WMEM_CHUNK_NEXT(chunk); } while (chunk); g_assert(total_len == WMEM_BLOCK_SIZE); return total_free_space; }
/* Cycles the recycler. See the design notes at the top of this file for more * details. */ static void wmem_block_cycle_recycler(wmem_block_allocator_t *allocator) { wmem_block_chunk_t *chunk; wmem_block_free_t *free_chunk; chunk = allocator->recycler_head; if (chunk == NULL) { return; } free_chunk = WMEM_GET_FREE(chunk); if (free_chunk->next->len < chunk->len) { /* Hold the current head fixed during rotation. */ WMEM_GET_FREE(free_chunk->next)->prev = free_chunk->prev; WMEM_GET_FREE(free_chunk->prev)->next = free_chunk->next; free_chunk->prev = free_chunk->next; free_chunk->next = WMEM_GET_FREE(free_chunk->next)->next; WMEM_GET_FREE(free_chunk->next)->prev = chunk; WMEM_GET_FREE(free_chunk->prev)->next = chunk; } else { /* Just rotate everything. */ allocator->recycler_head = free_chunk->next; } }
/* Takes an unused chunk and a size, and splits it into two chunks if possible. * The first chunk (at the same address as the input chunk) is guaranteed to * hold at least `size` bytes of data, and to not be in either the master or * recycler lists. * * The second chunk gets whatever data is left over. It is marked unused and * replaces the input chunk in whichever list it originally inhabited. */ static void wmem_block_split_free_chunk(wmem_block_allocator_t *allocator, wmem_block_chunk_t *chunk, const size_t size) { wmem_block_chunk_t *extra; wmem_block_free_t *old_blk, *new_blk; size_t aligned_size, available; gboolean last; aligned_size = WMEM_ALIGN_SIZE(size) + WMEM_CHUNK_HEADER_SIZE; if (WMEM_CHUNK_DATA_LEN(chunk) < aligned_size + sizeof(wmem_block_free_t)) { /* If the available space is not enought to store all of * (hdr + requested size + alignment padding + hdr + free-header) then * just remove the current chunk from the free list and return, since we * can't usefully split it. */ if (chunk == allocator->master_head) { wmem_block_pop_master(allocator); } else if (WMEM_CHUNK_DATA_LEN(chunk) >= sizeof(wmem_block_free_t)) { wmem_block_remove_from_recycler(allocator, chunk); } return; } /* preserve a few values from chunk that we'll need to manipulate */ last = chunk->last; available = chunk->len - aligned_size; /* set new values for chunk */ chunk->len = (guint32) aligned_size; chunk->last = FALSE; /* with chunk's values set, we can use the standard macro to calculate * the location and size of the new free chunk */ extra = WMEM_CHUNK_NEXT(chunk); /* Now we move the free chunk's address without changing its location * in whichever list it is in. * * Note that the new chunk header 'extra' may overlap the old free header, * so we have to copy the free header before we write anything to extra. */ old_blk = WMEM_GET_FREE(chunk); new_blk = WMEM_GET_FREE(extra); if (allocator->master_head == chunk) { new_blk->prev = old_blk->prev; new_blk->next = old_blk->next; if (old_blk->next) { WMEM_GET_FREE(old_blk->next)->prev = extra; } allocator->master_head = extra; } else { if (old_blk->prev == chunk) { new_blk->prev = extra; new_blk->next = extra; } else { new_blk->prev = old_blk->prev; new_blk->next = old_blk->next; WMEM_GET_FREE(old_blk->prev)->next = extra; WMEM_GET_FREE(old_blk->next)->prev = extra; } if (allocator->recycler_head == chunk) { allocator->recycler_head = extra; } } /* Now that we've copied over the free-list stuff (which may have overlapped * with our new chunk header) we can safely write our new chunk header. */ extra->len = (guint32) available; extra->last = last; extra->prev = chunk->len; extra->used = FALSE; extra->jumbo = FALSE; /* Correctly update the following chunk's back-pointer */ if (!last) { WMEM_CHUNK_NEXT(extra)->prev = extra->len; } }
/* Takes a free chunk and checks the chunks to its immediate right and left in * the block. If they are also free, the contigous free chunks are merged into * a single free chunk. The resulting chunk ends up in either the master list or * the recycler, depending on where the merged chunks were originally. */ static void wmem_block_merge_free(wmem_block_allocator_t *allocator, wmem_block_chunk_t *chunk) { wmem_block_chunk_t *tmp; wmem_block_chunk_t *left_free = NULL; wmem_block_chunk_t *right_free = NULL; /* Check the chunk to our right. If it is free, merge it into our current * chunk. If it is big enough to hold a free-header, save it for later (we * need to know about the left chunk before we decide what goes where). */ tmp = WMEM_CHUNK_NEXT(chunk); if (tmp && !tmp->used) { if (WMEM_CHUNK_DATA_LEN(tmp) >= sizeof(wmem_block_free_t)) { right_free = tmp; } chunk->len += tmp->len; chunk->last = tmp->last; } /* Check the chunk to our left. If it is free, merge our current chunk into * it (thus chunk = tmp). As before, save it if it has enough space to * hold a free-header. */ tmp = WMEM_CHUNK_PREV(chunk); if (tmp && !tmp->used) { if (WMEM_CHUNK_DATA_LEN(tmp) >= sizeof(wmem_block_free_t)) { left_free = tmp; } tmp->len += chunk->len; tmp->last = chunk->last; chunk = tmp; } /* The length of our chunk may have changed. If we have a chunk following, * update its 'prev' count. */ if (!chunk->last) { WMEM_CHUNK_NEXT(chunk)->prev = chunk->len; } /* Now that the chunk headers are merged and consistent, we need to figure * out what goes where in which free list. */ if (right_free && right_free == allocator->master_head) { /* If we merged right, and that chunk was the head of the master list, * then we leave the resulting chunk at the head of the master list. */ wmem_block_free_t *moved; if (left_free) { wmem_block_remove_from_recycler(allocator, left_free); } moved = WMEM_GET_FREE(chunk); moved->prev = NULL; moved->next = WMEM_GET_FREE(right_free)->next; allocator->master_head = chunk; if (moved->next) { WMEM_GET_FREE(moved->next)->prev = chunk; } } else { /* Otherwise, we remove the right-merged chunk (if there was one) from * the recycler. Then, if we merged left we have nothing to do, since * that recycler entry is still valid. If not, we add the chunk. */ if (right_free) { wmem_block_remove_from_recycler(allocator, right_free); } if (!left_free) { wmem_block_add_to_recycler(allocator, chunk); } } }