void
trim_map_write_done(zio_t *zio)
{
vdev_t *vd = zio->io_vd;
trim_map_t *tm = vd->vdev_trimmap;
/*
* Don't check for vdev_notrim, since the write could have
* started before vdev_notrim was set.
*/
if (!zfs_trim_enabled || tm == NULL)
return;
mutex_enter(&tm->tm_lock);
/*
* Don't fail if the write isn't in the tree, since the write
* could have started after vdev_notrim was set.
*/
if (zio->io_trim_node.avl_child[0] ||
zio->io_trim_node.avl_child[1] ||
AVL_XPARENT(&zio->io_trim_node) ||
tm->tm_inflight_writes.avl_root == &zio->io_trim_node)
avl_remove(&tm->tm_inflight_writes, zio);
mutex_exit(&tm->tm_lock);
}
/*
* Walk from one node to the previous valued node (ie. an infix walk
* towards the left). At any given node we do one of 2 things:
*
* - If there is a left child, go to it, then to it's rightmost descendant.
*
* - otherwise we return thru parent nodes until we've come from a right child.
*
* Return Value:
* NULL - if at the end of the nodes
* otherwise next node
*/
void *
avl_walk(avl_tree_t *tree, void *oldnode, int left)
{
size_t off = tree->avl_offset;
avl_node_t *node = AVL_DATA2NODE(oldnode, off);
int right = 1 - left;
int was_child;
/*
* nowhere to walk to if tree is empty
*/
if (node == NULL)
return (NULL);
/*
* Visit the previous valued node. There are two possibilities:
*
* If this node has a left child, go down one left, then all
* the way right.
*/
if (node->avl_child[left] != NULL) {
for (node = node->avl_child[left];
node->avl_child[right] != NULL;
node = node->avl_child[right])
;
/*
* Otherwise, return thru left children as far as we can.
*/
} else {
for (;;) {
was_child = AVL_XCHILD(node);
node = AVL_XPARENT(node);
if (node == NULL)
return (NULL);
if (was_child == right)
break;
}
}
return (AVL_NODE2DATA(node, off));
}
/*
* Insert a new node into an AVL tree at the specified (from avl_find()) place.
*
* Newly inserted nodes are always leaf nodes in the tree, since avl_find()
* searches out to the leaf positions. The avl_index_t indicates the node
* which will be the parent of the new node.
*
* After the node is inserted, a single rotation further up the tree may
* be necessary to maintain an acceptable AVL balance.
*/
void
avl_insert(avl_tree_t *tree, void *new_data, avl_index_t where)
{
avl_node_t *node;
avl_node_t *parent = AVL_INDEX2NODE(where);
int old_balance;
int new_balance;
int which_child = AVL_INDEX2CHILD(where);
size_t off = tree->avl_offset;
if (tree == NULL) {
filebench_log(LOG_ERROR, "No Tree Supplied");
return;
}
#if defined(_LP64) || (__WORDSIZE == 64)
if (((uintptr_t)new_data & 0x7) != 0) {
filebench_log(LOG_ERROR, "Missaligned pointer to new data");
return;
}
#endif
node = AVL_DATA2NODE(new_data, off);
/*
* First, add the node to the tree at the indicated position.
*/
++tree->avl_numnodes;
node->avl_child[0] = NULL;
node->avl_child[1] = NULL;
AVL_SETCHILD(node, which_child);
AVL_SETBALANCE(node, 0);
AVL_SETPARENT(node, parent);
if (parent != NULL) {
if (parent->avl_child[which_child] != NULL)
filebench_log(LOG_DEBUG_IMPL,
"Overwriting existing pointer");
parent->avl_child[which_child] = node;
} else {
if (tree->avl_root != NULL)
filebench_log(LOG_DEBUG_IMPL,
"Overwriting existing pointer");
tree->avl_root = node;
}
/*
* Now, back up the tree modifying the balance of all nodes above the
* insertion point. If we get to a highly unbalanced ancestor, we
* need to do a rotation. If we back out of the tree we are done.
* If we brought any subtree into perfect balance (0), we are also done.
*/
for (;;) {
node = parent;
if (node == NULL)
return;
/*
* Compute the new balance
*/
old_balance = AVL_XBALANCE(node);
new_balance = old_balance + avl_child2balance[which_child];
/*
* If we introduced equal balance, then we are done immediately
*/
if (new_balance == 0) {
AVL_SETBALANCE(node, 0);
return;
}
/*
* If both old and new are not zero we went
* from -1 to -2 balance, do a rotation.
*/
if (old_balance != 0)
break;
AVL_SETBALANCE(node, new_balance);
parent = AVL_XPARENT(node);
which_child = AVL_XCHILD(node);
}
/*
* perform a rotation to fix the tree and return
*/
(void) avl_rotation(tree, node, new_balance);
}
/*
* Perform a rotation to restore balance at the subtree given by depth.
*
* This routine is used by both insertion and deletion. The return value
* indicates:
* 0 : subtree did not change height
* !0 : subtree was reduced in height
*
* The code is written as if handling left rotations, right rotations are
* symmetric and handled by swapping values of variables right/left[_heavy]
*
* On input balance is the "new" balance at "node". This value is either
* -2 or +2.
*/
static int
avl_rotation(avl_tree_t *tree, avl_node_t *node, int balance)
{
int left = !(balance < 0); /* when balance = -2, left will be 0 */
int right = 1 - left;
int left_heavy = balance >> 1;
int right_heavy = -left_heavy;
avl_node_t *parent = AVL_XPARENT(node);
avl_node_t *child = node->avl_child[left];
avl_node_t *cright;
avl_node_t *gchild;
avl_node_t *gright;
avl_node_t *gleft;
int which_child = AVL_XCHILD(node);
int child_bal = AVL_XBALANCE(child);
/* BEGIN CSTYLED */
/*
* case 1 : node is overly left heavy, the left child is balanced or
* also left heavy. This requires the following rotation.
*
* (node bal:-2)
* / \
* / \
* (child bal:0 or -1)
* / \
* / \
* cright
*
* becomes:
*
* (child bal:1 or 0)
* / \
* / \
* (node bal:-1 or 0)
* / \
* / \
* cright
*
* we detect this situation by noting that child's balance is not
* right_heavy.
*/
/* END CSTYLED */
if (child_bal != right_heavy) {
/*
* compute new balance of nodes
*
* If child used to be left heavy (now balanced) we reduced
* the height of this sub-tree -- used in "return...;" below
*/
child_bal += right_heavy; /* adjust towards right */
/*
* move "cright" to be node's left child
*/
cright = child->avl_child[right];
node->avl_child[left] = cright;
if (cright != NULL) {
AVL_SETPARENT(cright, node);
AVL_SETCHILD(cright, left);
}
/*
* move node to be child's right child
*/
child->avl_child[right] = node;
AVL_SETBALANCE(node, -child_bal);
AVL_SETCHILD(node, right);
AVL_SETPARENT(node, child);
/*
* update the pointer into this subtree
*/
AVL_SETBALANCE(child, child_bal);
AVL_SETCHILD(child, which_child);
AVL_SETPARENT(child, parent);
if (parent != NULL)
parent->avl_child[which_child] = child;
else
tree->avl_root = child;
return (child_bal == 0);
}
/* BEGIN CSTYLED */
/*
* case 2 : When node is left heavy, but child is right heavy we use
* a different rotation.
*
* (node b:-2)
* / \
* / \
* / \
* (child b:+1)
* / \
* / \
* (gchild b: != 0)
* / \
* / \
* gleft gright
*
* becomes:
*
* (gchild b:0)
* / \
* / \
* / \
* (child b:?) (node b:?)
* / \ / \
* / \ / \
* gleft gright
*
* computing the new balances is more complicated. As an example:
* if gchild was right_heavy, then child is now left heavy
* else it is balanced
*/
/* END CSTYLED */
gchild = child->avl_child[right];
gleft = gchild->avl_child[left];
gright = gchild->avl_child[right];
/*
* move gright to left child of node and
*
* move gleft to right child of node
*/
node->avl_child[left] = gright;
if (gright != NULL) {
AVL_SETPARENT(gright, node);
AVL_SETCHILD(gright, left);
}
child->avl_child[right] = gleft;
if (gleft != NULL) {
AVL_SETPARENT(gleft, child);
AVL_SETCHILD(gleft, right);
}
/*
* move child to left child of gchild and
*
* move node to right child of gchild and
*
* fixup parent of all this to point to gchild
*/
balance = AVL_XBALANCE(gchild);
gchild->avl_child[left] = child;
AVL_SETBALANCE(child, (balance == right_heavy ? left_heavy : 0));
AVL_SETPARENT(child, gchild);
AVL_SETCHILD(child, left);
gchild->avl_child[right] = node;
AVL_SETBALANCE(node, (balance == left_heavy ? right_heavy : 0));
AVL_SETPARENT(node, gchild);
AVL_SETCHILD(node, right);
AVL_SETBALANCE(gchild, 0);
AVL_SETPARENT(gchild, parent);
AVL_SETCHILD(gchild, which_child);
if (parent != NULL)
parent->avl_child[which_child] = gchild;
else
tree->avl_root = gchild;
return (1); /* the new tree is always shorter */
}
/*
* Insert a new node into an AVL tree at the specified (from avl_find()) place.
*
* Newly inserted nodes are always leaf nodes in the tree, since avl_find()
* searches out to the leaf positions. The avl_index_t indicates the node
* which will be the parent of the new node.
*
* After the node is inserted, a single rotation further up the tree may
* be necessary to maintain an acceptable AVL balance.
*/
void
avl_insert(avl_tree_t *tree, void *new_data, avl_index_t where)
{
avl_node_t *node;
avl_node_t *parent = AVL_INDEX2NODE(where);
int old_balance;
int new_balance;
int which_child = AVL_INDEX2CHILD(where);
size_t off = tree->avl_offset;
node = AVL_DATA2NODE(new_data, off);
/*
* First, add the node to the tree at the indicated position.
*/
++tree->avl_numnodes;
node->avl_child[0] = NULL;
node->avl_child[1] = NULL;
AVL_SETCHILD(node, which_child);
AVL_SETBALANCE(node, 0);
AVL_SETPARENT(node, parent);
if (parent != NULL) {
parent->avl_child[which_child] = node;
} else {
tree->avl_root = node;
}
/*
* Now, back up the tree modifying the balance of all nodes above the
* insertion point. If we get to a highly unbalanced ancestor, we
* need to do a rotation. If we back out of the tree we are done.
* If we brought any subtree into perfect balance (0), we are also done.
*/
for (;;) {
node = parent;
if (node == NULL)
return;
/*
* Compute the new balance
*/
old_balance = AVL_XBALANCE(node);
new_balance = old_balance + avl_child2balance[which_child];
/*
* If we introduced equal balance, then we are done immediately
*/
if (new_balance == 0) {
AVL_SETBALANCE(node, 0);
return;
}
/*
* If both old and new are not zero we went
* from -1 to -2 balance, do a rotation.
*/
if (old_balance != 0)
break;
AVL_SETBALANCE(node, new_balance);
parent = AVL_XPARENT(node);
which_child = AVL_XCHILD(node);
}
/*
* perform a rotation to fix the tree and return
*/
(void) avl_rotation(tree, node, new_balance);
}
/*
* At each step, visit (callback) the current node, then move to the next
* in the AVL tree. Uses the same algorithm as avl_walk().
*/
int
avl_walk_step(mdb_walk_state_t *wsp)
{
struct aw_info *aw;
size_t offset;
size_t size;
uintptr_t addr;
avl_node_t *node;
int status;
int was_child;
/*
* don't walk past the end of the tree!
*/
addr = wsp->walk_addr;
if (addr == NULL)
return (WALK_DONE);
aw = (struct aw_info *)wsp->walk_data;
if (aw->aw_end != NULL && wsp->walk_addr == aw->aw_end)
return (WALK_DONE);
size = aw->aw_tree.avl_size;
offset = aw->aw_tree.avl_offset;
node = (avl_node_t *)((uintptr_t)aw->aw_buff + offset);
/*
* must read the current node for the call back to use
*/
if (mdb_vread(aw->aw_buff, size, addr) == -1) {
mdb_warn("failed to read %s at %#lx", aw->aw_elem_name, addr);
return (WALK_ERR);
}
if (aw->aw_elem_check != NULL) {
int rc = aw->aw_elem_check(aw->aw_buff, addr,
aw->aw_elem_check_arg);
if (rc == -1)
return (WALK_ERR);
else if (rc == 1)
return (WALK_DONE);
}
/*
* do the call back
*/
status = wsp->walk_callback(addr, aw->aw_buff, wsp->walk_cbdata);
if (status != WALK_NEXT)
return (status);
/*
* move to the next node....
* note we read in new nodes, so the pointer to the buffer is fixed
*/
/*
* if the node has a right child then go to it and then all the way
* thru as many left children as possible
*/
addr = (uintptr_t)node->avl_child[1];
if (addr != NULL) {
addr = avl_leftmostchild(addr, aw->aw_buff, offset, size,
aw->aw_elem_name);
if (addr == (uintptr_t)-1L)
return (WALK_ERR);
/*
* othewise return to parent nodes, stopping if we ever return from
* a left child
*/
} else {
for (;;) {
was_child = AVL_XCHILD(node);
addr = (uintptr_t)AVL_XPARENT(node);
if (addr == NULL)
break;
addr -= offset;
if (was_child == 0) /* stop on return from left child */
break;
if (mdb_vread(aw->aw_buff, size, addr) == -1) {
mdb_warn("failed to read %s at %#lx",
aw->aw_elem_name, addr);
return (WALK_ERR);
}
}
}
wsp->walk_addr = addr;
return (WALK_NEXT);
}