static void
aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
{
AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr;
AOFF_RBTree_t *blk = (AOFF_RBTree_t *) block;
AOFF_RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC)
? &alc->sbmbc_root : &alc->mbc_root);
Uint blk_sz = BLK_SZ(blk);
#ifdef HARD_DEBUG
check_tree(*root, 0);
#endif
blk->flags = 0;
blk->left = NULL;
blk->right = NULL;
blk->max_sz = blk_sz;
if (!*root) {
blk->parent = NULL;
SET_BLACK(blk);
*root = blk;
}
else {
AOFF_RBTree_t *x = *root;
while (1) {
if (x->max_sz < blk_sz) {
x->max_sz = blk_sz;
}
if (blk < x) {
if (!x->left) {
blk->parent = x;
x->left = blk;
break;
}
x = x->left;
}
else {
if (!x->right) {
blk->parent = x;
x->right = blk;
break;
}
x = x->right;
}
}
/* Insert block into size tree */
RBT_ASSERT(blk->parent);
SET_RED(blk);
if (IS_RED(blk->parent))
tree_insert_fixup(root, blk);
}
#ifdef HARD_DEBUG
check_tree(*root, 0);
#endif
}
static void
link_free_block(Allctr_t *allctr, Block_t *block)
{
AFFreeBlock_t *blk = (AFFreeBlock_t *) block;
AFAllctr_t *afallctr = (AFAllctr_t *) allctr;
if (afallctr->free_list && BLK_SZ(afallctr->free_list) > BLK_SZ(blk)) {
blk->next = afallctr->free_list->next;
blk->prev = afallctr->free_list;
afallctr->free_list->next = blk;
}
else {
blk->next = afallctr->free_list;
blk->prev = NULL;
afallctr->free_list = blk;
}
if (blk->next)
blk->next->prev = blk;
}
/* Calculate 'max_size' of tree node x by only looking at the direct children
* of x and x itself.
*/
static ERTS_INLINE Uint node_max_size(AOFF_RBTree_t *x)
{
Uint sz = BLK_SZ(x);
if (x->left && x->left->max_sz > sz) {
sz = x->left->max_sz;
}
if (x->right && x->right->max_sz > sz) {
sz = x->right->max_sz;
}
return sz;
}
static Block_t *
get_free_block(Allctr_t *allctr, Uint size)
{
AFFreeBlock_t *res;
AFAllctr_t *afallctr = (AFAllctr_t *) allctr;
if (afallctr->free_list && BLK_SZ(afallctr->free_list) >= size) {
res = afallctr->free_list;
afallctr->free_list = res->next;
if (res->next)
res->next->prev = NULL;
}
else
res = NULL;
return (Block_t *) res;
}
static void
print_tree_aux(AOFF_RBTree_t *x, int indent)
{
int i;
if (x) {
print_tree_aux(x->right, indent + INDENT_STEP);
for (i = 0; i < indent; i++) {
putc(' ', stderr);
}
fprintf(stderr, "%s: sz=%lu addr=0x%lx max_size=%lu\r\n",
IS_BLACK(x) ? "BLACK" : "RED",
BLK_SZ(x), (Uint)x, x->max_sz);
print_tree_aux(x->left, indent + INDENT_STEP);
}
}
void
check_ablk(TestCaseState_t *tcs, Allctr_t *a, void *ptr, Ulong umem_sz)
{
Ulong unit_sz = UNIT_SZ;
Block_t *blk = UMEM2BLK(ptr);
Block_t *nxt_blk = NXT_BLK(blk);
Ulong real_sz = ((Ulong) nxt_blk) - ((Ulong) (blk));
ASSERT(tcs, real_sz == BLK_SZ(blk));
ASSERT(tcs, !IS_FREE_BLK(blk));
ASSERT(tcs, real_sz >= CEILING(ABLK_HDR_SZ + umem_sz, unit_sz));
if (real_sz > MIN_BLK_SZ(a)
&& real_sz > CEILING(ABLK_HDR_SZ+umem_sz, unit_sz)) {
ASSERT(tcs,
real_sz <= CEILING(MIN_BLK_SZ(a)+ABLK_HDR_SZ+umem_sz,
unit_sz));
ASSERT(tcs, IS_LAST_BLK(blk) || !IS_FREE_BLK(nxt_blk));
}
}
static Block_t *
aoff_get_free_block(Allctr_t *allctr, Uint size,
Block_t *cand_blk, Uint cand_size, Uint32 flags)
{
AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr;
AOFF_RBTree_t *x = ((flags & ERTS_ALCU_FLG_SBMBC)
? alc->sbmbc_root : alc->mbc_root);
AOFF_RBTree_t *blk = NULL;
#ifdef HARD_DEBUG
AOFF_RBTree_t* dbg_blk = check_tree(x, size);
#endif
ASSERT(!cand_blk || cand_size >= size);
while (x) {
if (x->left && x->left->max_sz >= size) {
x = x->left;
}
else if (BLK_SZ(x) >= size) {
blk = x;
break;
}
else {
x = x->right;
}
}
#ifdef HARD_DEBUG
ASSERT(blk == dbg_blk);
#endif
if (!blk)
return NULL;
if (cand_blk && cand_blk < &blk->hdr) {
return NULL; /* cand_blk was better */
}
aoff_unlink_free_block(allctr, (Block_t *) blk, flags);
return (Block_t *) blk;
}
void
testcase_run(TestCaseState_t *tcs)
{
void *tmp;
void **fence;
void **blk;
Ulong sz;
Ulong smbcs;
int i;
int bi;
int bi_tests;
Ulong sbct = (SBCT/1024)*1024;
Ulong min_blk_sz;
Ulong ablk_hdr_sz = ABLK_HDR_SZ;
char smbcs_buf[30];
char sbct_buf[30];
int no_bkts = (int) NO_OF_BKTS;
char *argv1[] = {"-tasgf", "-tmmbcs0", sbct_buf, NULL};
char *argv2[] = {"-tasgf", "-tmmbcs0", sbct_buf, NULL, NULL};
Allctr_t *a;
sprintf(sbct_buf, "-tsbct%lu", sbct/1024);
a = START_ALC("bkt_mask_1_", 0, argv1);
tcs->extra = (void *) a;
ASSERT(tcs, a);
min_blk_sz = MIN_BLK_SZ(a);
smbcs = 2*(no_bkts*sizeof(void *) + min_blk_sz) + min_blk_sz;
for (i = 0; i < no_bkts; i++) {
sz = BKT_MIN_SZ(a, i);
if (sz >= sbct)
break;
smbcs += sz + min_blk_sz;
}
bi_tests = i;
testcase_printf(tcs, "Will test %d buckets\n", bi_tests);
STOP_ALC(a);
tcs->extra = NULL;
smbcs /= 1024;
smbcs++;
testcase_printf(tcs, "smbcs = %lu\n", smbcs);
sprintf(smbcs_buf, "-tsmbcs%lu", smbcs);
argv2[3] = smbcs_buf;
a = START_ALC("bkt_mask_2_", 0, argv2);
tcs->extra = (void *) a;
ASSERT(tcs, a);
blk = (void **) ALLOC(a, no_bkts*sizeof(void *));
fence = (void **) ALLOC(a, no_bkts*sizeof(void *));
ASSERT(tcs, blk && fence);
testcase_printf(tcs, "Allocating blocks and fences\n");
for (i = 0; i < bi_tests; i++) {
sz = BKT_MIN_SZ(a, i);
blk[i] = ALLOC(a, sz - ablk_hdr_sz);
fence[i] = ALLOC(a, 1);
ASSERT(tcs, blk[i] && fence[i]);
}
tmp = (void *) UMEM2BLK(fence[bi_tests - 1]);
tmp = (void *) NXT_BLK((Block_t *) tmp);
ASSERT(tcs, IS_LAST_BLK(tmp));
sz = BLK_SZ((Block_t *) tmp);
testcase_printf(tcs, "Allocating leftover size = %lu\n", sz);
tmp = ALLOC(a, sz - ablk_hdr_sz);
ASSERT(tcs, tmp);
bi = FIND_BKT(a, 0);
ASSERT(tcs, bi < 0);
for (i = 0; i < bi_tests; i++) {
sz = BKT_MIN_SZ(a, i);
testcase_printf(tcs, "Testing bucket %d\n", i);
FREE(a, blk[i]);
bi = FIND_BKT(a, i);
ASSERT(tcs, bi == i);
blk[i] = ALLOC(a, sz - ablk_hdr_sz);
bi = FIND_BKT(a, i);
ASSERT(tcs, bi != i);
}
for (i = 0; i < bi_tests; i++) {
FREE(a, blk[i]);
FREE(a, fence[i]);
}
FREE(a, (void *) blk);
FREE(a, (void *) fence);
bi = FIND_BKT(a, 0);
ASSERT(tcs, bi == no_bkts - 1);
FREE(a, tmp);
bi = FIND_BKT(a, 0);
ASSERT(tcs, bi < 0);
STOP_ALC(a);
tcs->extra = NULL;
}
static void
test_realloc(TestCaseState_t *tcs, Allctr_t *a, Ulong bsz)
{
Block_t *blk;
void *ptr;
void *p[3];
Ulong nbsz;
testcase_printf(tcs," --- Testing realloc() with block size %lu ---\n",
bsz);
setup_sequence(tcs, a, bsz, 3, p);
check_ablk(tcs, a, p[0], bsz);
check_ablk(tcs, a, p[1], bsz);
check_ablk(tcs, a, p[2], bsz);
/* Grow to the end of the carrier */
blk = NXT_BLK(UMEM2BLK(p[2]));
ASSERT(tcs, IS_FREE_BLK(blk));
ASSERT(tcs, IS_LAST_BLK(blk));
nbsz = bsz + BLK_SZ(blk);
ptr = REALLOC(a, p[2], nbsz);
ASSERT(tcs, p[2] == ptr);
check_ablk(tcs, a, p[2], nbsz);
blk = UMEM2BLK(p[2]);
ASSERT(tcs, IS_LAST_BLK(blk));
/* Shrink from the end of the carrier */
ptr = REALLOC(a, p[2], bsz);
ASSERT(tcs, p[2] == ptr);
blk = UMEM2BLK(p[2]);
ASSERT(tcs, !IS_LAST_BLK(blk));
blk = NXT_BLK(blk);
ASSERT(tcs, IS_LAST_BLK(blk));
check_ablk(tcs, a, p[2], bsz);
/* Shrink and coalecse with next free */
FREE(a, p[1]);
blk = NXT_BLK(UMEM2BLK(p[0]));
ASSERT(tcs, IS_FREE_BLK(blk));
nbsz = bsz/2;
ptr = REALLOC(a, p[0], nbsz);
ASSERT(tcs, p[0] == ptr);
check_ablk(tcs, a, p[0], nbsz);
blk = NXT_BLK(UMEM2BLK(p[0]));
ASSERT(tcs, IS_FREE_BLK(blk));
ASSERT(tcs, NXT_BLK(blk) == UMEM2BLK(p[2]));
/* Grow into next free; but leave free block at end */
nbsz *= 3;
ptr = REALLOC(a, p[0], nbsz);
ASSERT(tcs, p[0] == ptr);
check_ablk(tcs, a, p[0], nbsz);
blk = NXT_BLK(UMEM2BLK(p[0]));
ASSERT(tcs, IS_FREE_BLK(blk));
ASSERT(tcs, NXT_BLK(blk) == UMEM2BLK(p[2]));
/* Grow upto next alloced block by allocating just enough so that no
free block fits between them */
nbsz = BLK_SZ(blk) + UMEM_SZ(UMEM2BLK(p[0]));
nbsz -= MIN_BLK_SZ(a) - 1;
ptr = REALLOC(a, p[0], nbsz);
ASSERT(tcs, p[0] == ptr);
check_ablk(tcs, a, p[0], nbsz);
blk = NXT_BLK(UMEM2BLK(p[0]));
ASSERT(tcs, !IS_FREE_BLK(blk));
ASSERT(tcs, blk == UMEM2BLK(p[2]));
/* Grow into unused part at end */
nbsz += MIN_BLK_SZ(a) - 1;
ptr = REALLOC(a, p[0], nbsz);
ASSERT(tcs, p[0] == ptr);
check_ablk(tcs, a, p[0], nbsz);
ASSERT(tcs, !IS_FREE_BLK(blk));
ASSERT(tcs, blk == UMEM2BLK(p[2]));
/* Shrink *almost* as much so that a free block would fit between the
allocated blocks, and make sure that we don't get a free block
in between */
nbsz -= MIN_BLK_SZ(a) - 1;
ptr = REALLOC(a, p[0], nbsz);
ASSERT(tcs, p[0] == ptr);
check_ablk(tcs, a, p[0], nbsz);
blk = NXT_BLK(UMEM2BLK(p[0]));
ASSERT(tcs, !IS_FREE_BLK(blk));
ASSERT(tcs, blk == UMEM2BLK(p[2]));
/* Shrink just as much so that a free block can fit between
the alloced blocks */
nbsz -= 1;
ptr = REALLOC(a, p[0], nbsz);
ASSERT(tcs, p[0] == ptr);
check_ablk(tcs, a, p[0], nbsz);
blk = NXT_BLK(UMEM2BLK(p[0]));
ASSERT(tcs, IS_FREE_BLK(blk));
ASSERT(tcs, blk < UMEM2BLK(p[2]));
ASSERT(tcs, NXT_BLK(blk) == UMEM2BLK(p[2]));
/* Shrink so little that no free block would fit between allocated
blocks, and make sure that we shrink the allocated block and
coalesce the extra free part with the next free block. */
nbsz -= MIN_BLK_SZ(a) - 1;
ptr = REALLOC(a, p[0], nbsz);
ASSERT(tcs, p[0] == ptr);
check_ablk(tcs, a, p[0], nbsz);
blk = NXT_BLK(UMEM2BLK(p[0]));
ASSERT(tcs, IS_FREE_BLK(blk));
ASSERT(tcs, blk < UMEM2BLK(p[2]));
ASSERT(tcs, NXT_BLK(blk) == UMEM2BLK(p[2]));
/* Cleanup */
FREE(a, p[0]);
FREE(a, p[2]);
testcase_printf(tcs, " --- realloc() with block size %lu succeded ---\n",
bsz);
}
static AOFF_RBTree_t *
check_tree(AOFF_RBTree_t* root, Uint size)
{
AOFF_RBTree_t *res = NULL;
Sint blacks;
Sint curr_blacks;
AOFF_RBTree_t *x;
#ifdef PRINT_TREE
print_tree(root);
#endif
if (!root)
return res;
x = root;
ASSERT(IS_BLACK(x));
ASSERT(!x->parent);
curr_blacks = 1;
blacks = -1;
while (x) {
if (!IS_LEFT_VISITED(x)) {
SET_LEFT_VISITED(x);
if (x->left) {
x = x->left;
if (IS_BLACK(x))
curr_blacks++;
continue;
}
else {
if (blacks < 0)
blacks = curr_blacks;
ASSERT(blacks == curr_blacks);
}
}
if (!IS_RIGHT_VISITED(x)) {
SET_RIGHT_VISITED(x);
if (x->right) {
x = x->right;
if (IS_BLACK(x))
curr_blacks++;
continue;
}
else {
if (blacks < 0)
blacks = curr_blacks;
ASSERT(blacks == curr_blacks);
}
}
if (IS_RED(x)) {
ASSERT(IS_BLACK(x->right));
ASSERT(IS_BLACK(x->left));
}
ASSERT(x->parent || x == root);
if (x->left) {
ASSERT(x->left->parent == x);
ASSERT(x->left < x);
ASSERT(x->left->max_sz <= x->max_sz);
}
if (x->right) {
ASSERT(x->right->parent == x);
ASSERT(x->right > x);
ASSERT(x->right->max_sz <= x->max_sz);
}
ASSERT(x->max_sz >= BLK_SZ(x));
ASSERT(x->max_sz == BLK_SZ(x)
|| x->max_sz == (x->left ? x->left->max_sz : 0)
|| x->max_sz == (x->right ? x->right->max_sz : 0));
if (size && BLK_SZ(x) >= size) {
if (!res || x < res) {
res = x;
}
}
UNSET_LEFT_VISITED(x);
UNSET_RIGHT_VISITED(x);
if (IS_BLACK(x))
curr_blacks--;
x = x->parent;
}
ASSERT(curr_blacks == 0);
UNSET_LEFT_VISITED(root);
UNSET_RIGHT_VISITED(root);
return res;
}