/*
* Replace node x with node y
* NOTE: block header of y is not changed
*/
static ERTS_INLINE void
replace(RBTree_t **root, RBTree_t *x, RBTree_t *y)
{
if (!x->parent) {
RBT_ASSERT(*root == x);
*root = y;
}
else if (x == x->parent->left)
x->parent->left = y;
else {
RBT_ASSERT(x == x->parent->right);
x->parent->right = y;
}
if (x->left) {
RBT_ASSERT(x->left->parent == x);
x->left->parent = y;
}
if (x->right) {
RBT_ASSERT(x->right->parent == x);
x->right->parent = y;
}
y->flags = x->flags;
y->parent = x->parent;
y->right = x->right;
y->left = x->left;
DESTROY_TREE_NODE(x);
}
/*
* Replace node x with node y
* NOTE: block header of y is not changed
*/
static ERTS_INLINE void
replace(AOFF_RBTree_t **root, AOFF_RBTree_t *x, AOFF_RBTree_t *y)
{
if (!x->parent) {
RBT_ASSERT(*root == x);
*root = y;
}
else if (x == x->parent->left)
x->parent->left = y;
else {
RBT_ASSERT(x == x->parent->right);
x->parent->right = y;
}
if (x->left) {
RBT_ASSERT(x->left->parent == x);
x->left->parent = y;
}
if (x->right) {
RBT_ASSERT(x->right->parent == x);
x->right->parent = y;
}
y->flags = x->flags;
y->parent = x->parent;
y->right = x->right;
y->left = x->left;
y->max_sz = x->max_sz;
}
static ERTS_INLINE void
left_rotate(AOFF_RBTree_t **root, AOFF_RBTree_t *x)
{
AOFF_RBTree_t *y = x->right;
x->right = y->left;
if (y->left)
y->left->parent = x;
y->parent = x->parent;
if (!y->parent) {
RBT_ASSERT(*root == x);
*root = y;
}
else if (x == x->parent->left)
x->parent->left = y;
else {
RBT_ASSERT(x == x->parent->right);
x->parent->right = y;
}
y->left = x;
x->parent = y;
y->max_sz = x->max_sz;
x->max_sz = node_max_size(x);
ASSERT(y->max_sz >= x->max_sz);
}
/*
* Replace node x with node y
* NOTE: block header of y is not changed
*/
static ERTS_INLINE void
replace(AOFF_RBTree_t **root, AOFF_RBTree_t *x, AOFF_RBTree_t *y)
{
if (!x->parent) {
RBT_ASSERT(*root == x);
*root = y;
}
else if (x == x->parent->left)
x->parent->left = y;
else {
RBT_ASSERT(x == x->parent->right);
x->parent->right = y;
}
if (x->left) {
RBT_ASSERT(x->left->parent == x);
x->left->parent = y;
}
if (x->right) {
RBT_ASSERT(x->right->parent == x);
x->right->parent = y;
}
y->flags = x->flags;
y->parent = x->parent;
y->right = x->right;
y->left = x->left;
y->max_sz = x->max_sz;
lower_max_size(y, NULL);
DESTROY_TREE_NODE(x);
}
static void
aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags)
{
AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr;
AOFF_RBTree_t *blk = (AOFF_RBTree_t *) block;
AOFF_RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC)
? &alc->sbmbc_root : &alc->mbc_root);
Uint blk_sz = BLK_SZ(blk);
#ifdef HARD_DEBUG
check_tree(*root, 0);
#endif
blk->flags = 0;
blk->left = NULL;
blk->right = NULL;
blk->max_sz = blk_sz;
if (!*root) {
blk->parent = NULL;
SET_BLACK(blk);
*root = blk;
}
else {
AOFF_RBTree_t *x = *root;
while (1) {
if (x->max_sz < blk_sz) {
x->max_sz = blk_sz;
}
if (blk < x) {
if (!x->left) {
blk->parent = x;
x->left = blk;
break;
}
x = x->left;
}
else {
if (!x->right) {
blk->parent = x;
x->right = blk;
break;
}
x = x->right;
}
}
/* Insert block into size tree */
RBT_ASSERT(blk->parent);
SET_RED(blk);
if (IS_RED(blk->parent))
tree_insert_fixup(root, blk);
}
#ifdef HARD_DEBUG
check_tree(*root, 0);
#endif
}
static void
aobf_link_free_block(Allctr_t *allctr, Block_t *block)
{
BFAllctr_t *bfallctr = (BFAllctr_t *) allctr;
RBTree_t **root = &bfallctr->mbc_root;
RBTree_t *blk = (RBTree_t *) block;
Uint blk_sz = BF_BLK_SZ(blk);
blk->flags = 0;
blk->left = NULL;
blk->right = NULL;
if (!*root) {
blk->parent = NULL;
SET_BLACK(blk);
*root = blk;
}
else {
RBTree_t *x = *root;
while (1) {
Uint size;
size = BF_BLK_SZ(x);
if (blk_sz < size || (blk_sz == size && blk < x)) {
if (!x->left) {
blk->parent = x;
x->left = blk;
break;
}
x = x->left;
}
else {
if (!x->right) {
blk->parent = x;
x->right = blk;
break;
}
x = x->right;
}
}
/* Insert block into size tree */
RBT_ASSERT(blk->parent);
SET_RED(blk);
if (IS_RED(blk->parent))
tree_insert_fixup(root, blk);
}
#ifdef HARD_DEBUG
check_tree(root, 1, 0);
#endif
}
static ERTS_INLINE void
right_rotate(RBTree_t **root, RBTree_t *x)
{
RBTree_t *y = x->left;
x->left = y->right;
if (y->right)
y->right->parent = x;
y->parent = x->parent;
if (!y->parent) {
RBT_ASSERT(*root == x);
*root = y;
}
else if (x == x->parent->right)
x->parent->right = y;
else {
RBT_ASSERT(x == x->parent->left);
x->parent->left = y;
}
y->right = x;
x->parent = y;
}
static void
bf_link_free_block(Allctr_t *allctr, Block_t *block)
{
BFAllctr_t *bfallctr = (BFAllctr_t *) allctr;
RBTree_t **root = &bfallctr->mbc_root;
RBTree_t *blk = (RBTree_t *) block;
Uint blk_sz = BF_BLK_SZ(blk);
SET_TREE_NODE(blk);
blk->flags = 0;
blk->left = NULL;
blk->right = NULL;
if (!*root) {
blk->parent = NULL;
SET_BLACK(blk);
*root = blk;
}
else {
RBTree_t *x = *root;
while (1) {
Uint size;
size = BF_BLK_SZ(x);
if (blk_sz == size) {
SET_LIST_ELEM(blk);
LIST_NEXT(blk) = LIST_NEXT(x);
LIST_PREV(blk) = x;
if (LIST_NEXT(x))
LIST_PREV(LIST_NEXT(x)) = blk;
LIST_NEXT(x) = blk;
return; /* Finnished */
}
else if (blk_sz < size) {
if (!x->left) {
blk->parent = x;
x->left = blk;
break;
}
x = x->left;
}
else {
if (!x->right) {
blk->parent = x;
x->right = blk;
break;
}
x = x->right;
}
}
RBT_ASSERT(blk->parent);
SET_RED(blk);
if (IS_RED(blk->parent))
tree_insert_fixup(root, blk);
}
SET_TREE_NODE(blk);
LIST_NEXT(blk) = NULL;
#ifdef HARD_DEBUG
check_tree(root, 0, 0);
#endif
}
/*
* The argument types of "Allctr_t *" and "Block_t *" have been
* chosen since we then can use tree_delete() as unlink_free_block
* callback function in the address order case.
*/
static void
tree_delete(Allctr_t *allctr, Block_t *del)
{
BFAllctr_t *bfallctr = (BFAllctr_t *) allctr;
Uint spliced_is_black;
RBTree_t **root = &bfallctr->mbc_root;
RBTree_t *x, *y, *z = (RBTree_t *) del;
RBTree_t null_x; /* null_x is used to get the fixup started when we
splice out a node without children. */
null_x.parent = NULL;
#ifdef HARD_DEBUG
check_tree(*root, bfallctr->address_order, 0);
#endif
/* Remove node from tree... */
/* Find node to splice out */
if (!z->left || !z->right)
y = z;
else
/* Set y to z:s successor */
for(y = z->right; y->left; y = y->left);
/* splice out y */
x = y->left ? y->left : y->right;
spliced_is_black = IS_BLACK(y);
if (x) {
x->parent = y->parent;
}
else if (!x && spliced_is_black) {
x = &null_x;
x->flags = 0;
SET_BLACK(x);
x->right = x->left = NULL;
x->parent = y->parent;
y->left = x;
}
if (!y->parent) {
RBT_ASSERT(*root == y);
*root = x;
}
else if (y == y->parent->left)
y->parent->left = x;
else {
RBT_ASSERT(y == y->parent->right);
y->parent->right = x;
}
if (y != z) {
/* We spliced out the successor of z; replace z by the successor */
replace(root, z, y);
}
if (spliced_is_black) {
/* We removed a black node which makes the resulting tree
violate the Red-Black Tree properties. Fixup tree... */
while (IS_BLACK(x) && x->parent) {
/*
* x has an "extra black" which we move up the tree
* until we reach the root or until we can get rid of it.
*
* y is the sibbling of x
*/
if (x == x->parent->left) {
y = x->parent->right;
RBT_ASSERT(y);
if (IS_RED(y)) {
RBT_ASSERT(y->right);
RBT_ASSERT(y->left);
SET_BLACK(y);
RBT_ASSERT(IS_BLACK(x->parent));
SET_RED(x->parent);
left_rotate(root, x->parent);
y = x->parent->right;
}
RBT_ASSERT(y);
RBT_ASSERT(IS_BLACK(y));
if (IS_BLACK(y->left) && IS_BLACK(y->right)) {
SET_RED(y);
x = x->parent;
}
else {
if (IS_BLACK(y->right)) {
SET_BLACK(y->left);
SET_RED(y);
right_rotate(root, y);
y = x->parent->right;
}
RBT_ASSERT(y);
if (IS_RED(x->parent)) {
SET_BLACK(x->parent);
SET_RED(y);
}
RBT_ASSERT(y->right);
SET_BLACK(y->right);
left_rotate(root, x->parent);
x = *root;
break;
}
}
else {
RBT_ASSERT(x == x->parent->right);
y = x->parent->left;
RBT_ASSERT(y);
if (IS_RED(y)) {
RBT_ASSERT(y->right);
RBT_ASSERT(y->left);
SET_BLACK(y);
RBT_ASSERT(IS_BLACK(x->parent));
SET_RED(x->parent);
right_rotate(root, x->parent);
y = x->parent->left;
}
RBT_ASSERT(y);
RBT_ASSERT(IS_BLACK(y));
if (IS_BLACK(y->right) && IS_BLACK(y->left)) {
SET_RED(y);
x = x->parent;
}
else {
if (IS_BLACK(y->left)) {
SET_BLACK(y->right);
SET_RED(y);
left_rotate(root, y);
y = x->parent->left;
}
RBT_ASSERT(y);
if (IS_RED(x->parent)) {
SET_BLACK(x->parent);
SET_RED(y);
}
RBT_ASSERT(y->left);
SET_BLACK(y->left);
right_rotate(root, x->parent);
x = *root;
break;
}
}
}
SET_BLACK(x);
if (null_x.parent) {
if (null_x.parent->left == &null_x)
null_x.parent->left = NULL;
else {
RBT_ASSERT(null_x.parent->right == &null_x);
null_x.parent->right = NULL;
}
RBT_ASSERT(!null_x.left);
RBT_ASSERT(!null_x.right);
}
else if (*root == &null_x) {
*root = NULL;
RBT_ASSERT(!null_x.left);
RBT_ASSERT(!null_x.right);
}
}
DESTROY_TREE_NODE(del);
#ifdef HARD_DEBUG
check_tree(root, bfallctr->address_order, 0);
#endif
}
static void
tree_insert_fixup(RBTree_t **root, RBTree_t *blk)
{
RBTree_t *x = blk, *y;
/*
* Rearrange the tree so that it satisfies the Red-Black Tree properties
*/
RBT_ASSERT(x != *root && IS_RED(x->parent));
do {
/*
* x and its parent are both red. Move the red pair up the tree
* until we get to the root or until we can separate them.
*/
RBT_ASSERT(IS_RED(x));
RBT_ASSERT(IS_BLACK(x->parent->parent));
RBT_ASSERT(x->parent->parent);
if (x->parent == x->parent->parent->left) {
y = x->parent->parent->right;
if (IS_RED(y)) {
SET_BLACK(y);
x = x->parent;
SET_BLACK(x);
x = x->parent;
SET_RED(x);
}
else {
if (x == x->parent->right) {
x = x->parent;
left_rotate(root, x);
}
RBT_ASSERT(x == x->parent->parent->left->left);
RBT_ASSERT(IS_RED(x));
RBT_ASSERT(IS_RED(x->parent));
RBT_ASSERT(IS_BLACK(x->parent->parent));
RBT_ASSERT(IS_BLACK(y));
SET_BLACK(x->parent);
SET_RED(x->parent->parent);
right_rotate(root, x->parent->parent);
RBT_ASSERT(x == x->parent->left);
RBT_ASSERT(IS_RED(x));
RBT_ASSERT(IS_RED(x->parent->right));
RBT_ASSERT(IS_BLACK(x->parent));
break;
}
}
else {
RBT_ASSERT(x->parent == x->parent->parent->right);
y = x->parent->parent->left;
if (IS_RED(y)) {
SET_BLACK(y);
x = x->parent;
SET_BLACK(x);
x = x->parent;
SET_RED(x);
}
else {
if (x == x->parent->left) {
x = x->parent;
right_rotate(root, x);
}
RBT_ASSERT(x == x->parent->parent->right->right);
RBT_ASSERT(IS_RED(x));
RBT_ASSERT(IS_RED(x->parent));
RBT_ASSERT(IS_BLACK(x->parent->parent));
RBT_ASSERT(IS_BLACK(y));
SET_BLACK(x->parent);
SET_RED(x->parent->parent);
left_rotate(root, x->parent->parent);
RBT_ASSERT(x == x->parent->right);
RBT_ASSERT(IS_RED(x));
RBT_ASSERT(IS_RED(x->parent->left));
RBT_ASSERT(IS_BLACK(x->parent));
break;
}
}
} while (x != *root && IS_RED(x->parent));
SET_BLACK(*root);
}
static void
rbt_insert(enum AOFFSortOrder order, AOFF_RBTree_t** root, AOFF_RBTree_t* blk)
{
Uint blk_sz = AOFF_BLK_SZ(blk);
#ifdef DEBUG
blk->flags = (order == FF_BF) ? IS_BF_FLG : 0;
#else
blk->flags = 0;
#endif
blk->left = NULL;
blk->right = NULL;
blk->max_sz = blk_sz;
if (!*root) {
blk->parent = NULL;
SET_BLACK(blk);
*root = blk;
}
else {
AOFF_RBTree_t *x = *root;
while (1) {
SWord diff;
if (x->max_sz < blk_sz) {
x->max_sz = blk_sz;
}
diff = cmp_blocks(order, blk, x);
if (diff < 0) {
if (!x->left) {
blk->parent = x;
x->left = blk;
break;
}
x = x->left;
}
else if (diff > 0) {
if (!x->right) {
blk->parent = x;
x->right = blk;
break;
}
x = x->right;
}
else {
ASSERT(order == FF_BF);
ASSERT(blk->flags & IS_BF_FLG);
ASSERT(x->flags & IS_BF_FLG);
SET_LIST_ELEM(blk);
LIST_NEXT(blk) = LIST_NEXT(x);
LIST_PREV(blk) = x;
if (LIST_NEXT(x))
LIST_PREV(LIST_NEXT(x)) = blk;
LIST_NEXT(x) = blk;
return;
}
}
/* Insert block into size tree */
RBT_ASSERT(blk->parent);
SET_RED(blk);
if (IS_RED(blk->parent))
tree_insert_fixup(root, blk);
}
if (order == FF_BF) {
SET_TREE_NODE(blk);
LIST_NEXT(blk) = NULL;
}
}
static void
rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del)
{
Uint spliced_is_black;
AOFF_RBTree_t *x, *y, *z = del;
AOFF_RBTree_t null_x; /* null_x is used to get the fixup started when we
splice out a node without children. */
HARD_CHECK_IS_MEMBER(*root, del);
null_x.parent = NULL;
/* Remove node from tree... */
/* Find node to splice out */
if (!z->left || !z->right)
y = z;
else
/* Set y to z:s successor */
for(y = z->right; y->left; y = y->left);
/* splice out y */
x = y->left ? y->left : y->right;
spliced_is_black = IS_BLACK(y);
if (x) {
x->parent = y->parent;
}
else if (spliced_is_black) {
x = &null_x;
x->flags = 0;
SET_BLACK(x);
x->right = x->left = NULL;
x->max_sz = 0;
x->parent = y->parent;
y->left = x;
}
if (!y->parent) {
RBT_ASSERT(*root == y);
*root = x;
}
else {
if (y == y->parent->left) {
y->parent->left = x;
}
else {
RBT_ASSERT(y == y->parent->right);
y->parent->right = x;
}
if (y->parent != z) {
lower_max_size(y->parent, (y==z ? NULL : z));
}
}
if (y != z) {
/* We spliced out the successor of z; replace z by the successor */
ASSERT(z != &null_x);
replace(root, z, y);
lower_max_size(y, NULL);
}
if (spliced_is_black) {
/* We removed a black node which makes the resulting tree
violate the Red-Black Tree properties. Fixup tree... */
while (IS_BLACK(x) && x->parent) {
/*
* x has an "extra black" which we move up the tree
* until we reach the root or until we can get rid of it.
*
* y is the sibbling of x
*/
if (x == x->parent->left) {
y = x->parent->right;
RBT_ASSERT(y);
if (IS_RED(y)) {
RBT_ASSERT(y->right);
RBT_ASSERT(y->left);
SET_BLACK(y);
RBT_ASSERT(IS_BLACK(x->parent));
SET_RED(x->parent);
left_rotate(root, x->parent);
y = x->parent->right;
}
RBT_ASSERT(y);
RBT_ASSERT(IS_BLACK(y));
if (IS_BLACK(y->left) && IS_BLACK(y->right)) {
SET_RED(y);
x = x->parent;
}
else {
if (IS_BLACK(y->right)) {
SET_BLACK(y->left);
SET_RED(y);
right_rotate(root, y);
y = x->parent->right;
}
RBT_ASSERT(y);
if (IS_RED(x->parent)) {
SET_BLACK(x->parent);
SET_RED(y);
}
RBT_ASSERT(y->right);
SET_BLACK(y->right);
left_rotate(root, x->parent);
x = *root;
break;
}
}
else {
RBT_ASSERT(x == x->parent->right);
y = x->parent->left;
RBT_ASSERT(y);
if (IS_RED(y)) {
RBT_ASSERT(y->right);
RBT_ASSERT(y->left);
SET_BLACK(y);
RBT_ASSERT(IS_BLACK(x->parent));
SET_RED(x->parent);
right_rotate(root, x->parent);
y = x->parent->left;
}
RBT_ASSERT(y);
RBT_ASSERT(IS_BLACK(y));
if (IS_BLACK(y->right) && IS_BLACK(y->left)) {
SET_RED(y);
x = x->parent;
}
else {
if (IS_BLACK(y->left)) {
SET_BLACK(y->right);
SET_RED(y);
left_rotate(root, y);
y = x->parent->left;
}
RBT_ASSERT(y);
if (IS_RED(x->parent)) {
SET_BLACK(x->parent);
SET_RED(y);
}
RBT_ASSERT(y->left);
SET_BLACK(y->left);
right_rotate(root, x->parent);
x = *root;
break;
}
}
}
SET_BLACK(x);
if (null_x.parent) {
if (null_x.parent->left == &null_x)
null_x.parent->left = NULL;
else {
RBT_ASSERT(null_x.parent->right == &null_x);
null_x.parent->right = NULL;
}
RBT_ASSERT(!null_x.left);
RBT_ASSERT(!null_x.right);
}
else if (*root == &null_x) {
*root = NULL;
RBT_ASSERT(!null_x.left);
RBT_ASSERT(!null_x.right);
}
}
}