static struct fib6_node *fib6_add_1(struct fib6_node *root,
struct in6_addr *addr, int plen,
int offset, int allow_create,
int replace_required)
{
struct fib6_node *fn, *in, *ln;
struct fib6_node *pn = NULL;
struct rt6key *key;
int bit;
__be32 dir = 0;
__u32 sernum = fib6_new_sernum();
RT6_TRACE("fib6_add_1\n");
/* insert node in tree */
fn = root;
do {
key = (struct rt6key *)((u8 *)fn->leaf + offset);
/*
* Prefix match
*/
if (plen < fn->fn_bit ||
!ipv6_prefix_equal(&key->addr, addr, fn->fn_bit)) {
if (!allow_create) {
if (replace_required) {
pr_warn("Can't replace route, no match found\n");
return ERR_PTR(-ENOENT);
}
pr_warn("NLM_F_CREATE should be set when creating new route\n");
}
goto insert_above;
}
/*
* Exact match ?
*/
if (plen == fn->fn_bit) {
/* clean up an intermediate node */
if (!(fn->fn_flags & RTN_RTINFO)) {
rt6_release(fn->leaf);
fn->leaf = NULL;
}
fn->fn_sernum = sernum;
return fn;
}
/*
* We have more bits to go
*/
/* Try to walk down on tree. */
fn->fn_sernum = sernum;
dir = addr_bit_set(addr, fn->fn_bit);
pn = fn;
fn = dir ? fn->right : fn->left;
} while (fn);
if (!allow_create) {
/* We should not create new node because
* NLM_F_REPLACE was specified without NLM_F_CREATE
* I assume it is safe to require NLM_F_CREATE when
* REPLACE flag is used! Later we may want to remove the
* check for replace_required, because according
* to netlink specification, NLM_F_CREATE
* MUST be specified if new route is created.
* That would keep IPv6 consistent with IPv4
*/
if (replace_required) {
pr_warn("Can't replace route, no match found\n");
return ERR_PTR(-ENOENT);
}
pr_warn("NLM_F_CREATE should be set when creating new route\n");
}
/*
* We walked to the bottom of tree.
* Create new leaf node without children.
*/
ln = node_alloc();
if (!ln)
return ERR_PTR(-ENOMEM);
ln->fn_bit = plen;
ln->parent = pn;
ln->fn_sernum = sernum;
if (dir)
pn->right = ln;
else
pn->left = ln;
return ln;
insert_above:
/*
* split since we don't have a common prefix anymore or
* we have a less significant route.
* we've to insert an intermediate node on the list
* this new node will point to the one we need to create
* and the current
*/
pn = fn->parent;
/* find 1st bit in difference between the 2 addrs.
See comment in __ipv6_addr_diff: bit may be an invalid value,
but if it is >= plen, the value is ignored in any case.
*/
bit = __ipv6_addr_diff(addr, &key->addr, sizeof(*addr));
/*
* (intermediate)[in]
* / \
* (new leaf node)[ln] (old node)[fn]
*/
if (plen > bit) {
in = node_alloc();
ln = node_alloc();
if (!in || !ln) {
if (in)
node_free(in);
if (ln)
node_free(ln);
return ERR_PTR(-ENOMEM);
}
/*
* new intermediate node.
* RTN_RTINFO will
* be off since that an address that chooses one of
* the branches would not match less specific routes
* in the other branch
*/
in->fn_bit = bit;
in->parent = pn;
in->leaf = fn->leaf;
atomic_inc(&in->leaf->rt6i_ref);
in->fn_sernum = sernum;
/* update parent pointer */
if (dir)
pn->right = in;
else
pn->left = in;
ln->fn_bit = plen;
ln->parent = in;
fn->parent = in;
ln->fn_sernum = sernum;
if (addr_bit_set(addr, bit)) {
in->right = ln;
in->left = fn;
} else {
in->left = ln;
in->right = fn;
}
} else { /* plen <= bit */
/*
* (new leaf node)[ln]
* / \
* (old node)[fn] NULL
*/
ln = node_alloc();
if (!ln)
return ERR_PTR(-ENOMEM);
ln->fn_bit = plen;
ln->parent = pn;
ln->fn_sernum = sernum;
if (dir)
pn->right = ln;
else
pn->left = ln;
if (addr_bit_set(&key->addr, plen))
ln->right = fn;
else
ln->left = fn;
fn->parent = ln;
}
return ln;
}
static struct fib6_node * fib6_add_1(struct fib6_node *root, void *addr,
int addrlen, int plen,
int offset, int allow_create,
int replace_required)
{
struct fib6_node *fn, *in, *ln;
struct fib6_node *pn = NULL;
struct rt6key *key;
int bit;
__be32 dir = 0;
__u32 sernum = fib6_new_sernum();
RT6_TRACE("fib6_add_1\n");
fn = root;
do {
key = (struct rt6key *)((u8 *)fn->leaf + offset);
if (plen < fn->fn_bit ||
!ipv6_prefix_equal(&key->addr, addr, fn->fn_bit)) {
if (!allow_create) {
if (replace_required) {
pr_warn("IPv6: Can't replace route, "
"no match found\n");
return ERR_PTR(-ENOENT);
}
pr_warn("IPv6: NLM_F_CREATE should be set "
"when creating new route\n");
}
goto insert_above;
}
if (plen == fn->fn_bit) {
if (!(fn->fn_flags & RTN_RTINFO)) {
rt6_release(fn->leaf);
fn->leaf = NULL;
}
fn->fn_sernum = sernum;
return fn;
}
fn->fn_sernum = sernum;
dir = addr_bit_set(addr, fn->fn_bit);
pn = fn;
fn = dir ? fn->right: fn->left;
} while (fn);
if (!allow_create) {
if (replace_required) {
pr_warn("IPv6: Can't replace route, no match found\n");
return ERR_PTR(-ENOENT);
}
pr_warn("IPv6: NLM_F_CREATE should be set "
"when creating new route\n");
}
ln = node_alloc();
if (!ln)
return NULL;
ln->fn_bit = plen;
ln->parent = pn;
ln->fn_sernum = sernum;
if (dir)
pn->right = ln;
else
pn->left = ln;
return ln;
insert_above:
pn = fn->parent;
bit = __ipv6_addr_diff(addr, &key->addr, addrlen);
if (plen > bit) {
in = node_alloc();
ln = node_alloc();
if (!in || !ln) {
if (in)
node_free(in);
if (ln)
node_free(ln);
return NULL;
}
in->fn_bit = bit;
in->parent = pn;
in->leaf = fn->leaf;
atomic_inc(&in->leaf->rt6i_ref);
in->fn_sernum = sernum;
if (dir)
pn->right = in;
else
pn->left = in;
ln->fn_bit = plen;
ln->parent = in;
fn->parent = in;
ln->fn_sernum = sernum;
if (addr_bit_set(addr, bit)) {
in->right = ln;
in->left = fn;
} else {
in->left = ln;
in->right = fn;
}
} else {
ln = node_alloc();
if (!ln)
return NULL;
ln->fn_bit = plen;
ln->parent = pn;
ln->fn_sernum = sernum;
if (dir)
pn->right = ln;
else
pn->left = ln;
if (addr_bit_set(&key->addr, plen))
ln->right = fn;
else
ln->left = fn;
fn->parent = ln;
}
return ln;
}
int fib6_add(struct fib6_node *root, struct rt6_info *rt, struct nl_info *info)
{
struct fib6_node *fn, *pn = NULL;
int err = -ENOMEM;
int allow_create = 1;
int replace_required = 0;
if (info->nlh) {
if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
allow_create = 0;
if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
replace_required = 1;
}
if (!allow_create && !replace_required)
pr_warn("IPv6: RTM_NEWROUTE with no NLM_F_CREATE or NLM_F_REPLACE\n");
fn = fib6_add_1(root, &rt->rt6i_dst.addr, sizeof(struct in6_addr),
rt->rt6i_dst.plen, offsetof(struct rt6_info, rt6i_dst),
allow_create, replace_required);
if (IS_ERR(fn)) {
err = PTR_ERR(fn);
fn = NULL;
}
if (!fn)
goto out;
pn = fn;
#ifdef CONFIG_IPV6_SUBTREES
if (rt->rt6i_src.plen) {
struct fib6_node *sn;
if (!fn->subtree) {
struct fib6_node *sfn;
/*
* Create subtree.
*
* fn[main tree]
* |
* sfn[subtree root]
* \
* sn[new leaf node]
*/
/* Create subtree root node */
sfn = node_alloc();
if (!sfn)
goto st_failure;
sfn->leaf = info->nl_net->ipv6.ip6_null_entry;
atomic_inc(&info->nl_net->ipv6.ip6_null_entry->rt6i_ref);
sfn->fn_flags = RTN_ROOT;
sfn->fn_sernum = fib6_new_sernum();
/* Now add the first leaf node to new subtree */
sn = fib6_add_1(sfn, &rt->rt6i_src.addr,
sizeof(struct in6_addr), rt->rt6i_src.plen,
offsetof(struct rt6_info, rt6i_src),
allow_create, replace_required);
if (IS_ERR(sn)) {
err = PTR_ERR(sn);
sn = NULL;
}
if (!sn) {
/* If it is failed, discard just allocated
root, and then (in st_failure) stale node
in main tree.
*/
node_free(sfn);
goto st_failure;
}
/* Now link new subtree to main tree */
sfn->parent = fn;
fn->subtree = sfn;
} else {
sn = fib6_add_1(fn->subtree, &rt->rt6i_src.addr,
sizeof(struct in6_addr), rt->rt6i_src.plen,
offsetof(struct rt6_info, rt6i_src),
allow_create, replace_required);
if (IS_ERR(sn)) {
err = PTR_ERR(sn);
sn = NULL;
}
if (!sn)
goto st_failure;
}
if (!fn->leaf) {
fn->leaf = rt;
atomic_inc(&rt->rt6i_ref);
}
fn = sn;
}
map_val_t sl_cas (skiplist_t *sl, map_key_t key, map_val_t expectation, map_val_t new_val) {
TRACE("s1", "sl_cas: key %p skiplist %p", key, sl);
TRACE("s1", "sl_cas: expectation %p new value %p", expectation, new_val);
ASSERT((int64_t)new_val > 0);
node_t *preds[MAX_LEVELS];
node_t *nexts[MAX_LEVELS];
node_t *new_item = NULL;
int n = random_levels(sl);
node_t *old_item = find_preds(preds, nexts, n, sl, key, ASSIST_UNLINK);
// If there is already an item in the skiplist that matches the key just update its value.
if (old_item != NULL) {
map_val_t ret_val = update_item(old_item, expectation, new_val);
if (ret_val != DOES_NOT_EXIST)
return ret_val;
// If we lose a race with a thread removing the item we tried to update then we have to retry.
return sl_cas(sl, key, expectation, new_val); // tail call
}
if (EXPECT_FALSE(expectation != CAS_EXPECT_DOES_NOT_EXIST && expectation != CAS_EXPECT_WHATEVER)) {
TRACE("s1", "sl_cas: the expectation was not met, the skiplist was not changed", 0, 0);
return DOES_NOT_EXIST; // failure, the caller expected an item for the <key> to already exist
}
// Create a new node and insert it into the skiplist.
TRACE("s3", "sl_cas: attempting to insert a new item between %p and %p", preds[0], nexts[0]);
map_key_t new_key = sl->key_type == NULL ? key : (map_key_t)sl->key_type->clone((void *)key);
new_item = node_alloc(n, new_key, new_val);
// Set <new_item>'s next pointers to their proper values
markable_t next = new_item->next[0] = (markable_t)nexts[0];
for (int level = 1; level < new_item->num_levels; ++level) {
new_item->next[level] = (markable_t)nexts[level];
}
// Link <new_item> into <sl> from the bottom level up. After <new_item> is inserted into the bottom level
// it is officially part of the skiplist.
node_t *pred = preds[0];
markable_t other = SYNC_CAS(&pred->next[0], next, (markable_t)new_item);
if (other != next) {
TRACE("s3", "sl_cas: failed to change pred's link: expected %p found %p", next, other);
// Lost a race to another thread modifying the skiplist. Free the new item we allocated and retry.
if (sl->key_type != NULL) {
nbd_free((void *)new_key);
}
nbd_free(new_item);
return sl_cas(sl, key, expectation, new_val); // tail call
}
TRACE("s3", "sl_cas: successfully inserted a new item %p at the bottom level", new_item, 0);
ASSERT(new_item->num_levels <= MAX_LEVELS);
for (int level = 1; level < new_item->num_levels; ++level) {
TRACE("s3", "sl_cas: inserting the new item %p at level %p", new_item, level);
do {
node_t * pred = preds[level];
ASSERT(new_item->next[level]==(markable_t)nexts[level] || new_item->next[level]==MARK_NODE(nexts[level]));
TRACE("s3", "sl_cas: attempting to to insert the new item between %p and %p", pred, nexts[level]);
markable_t other = SYNC_CAS(&pred->next[level], (markable_t)nexts[level], (markable_t)new_item);
if (other == (markable_t)nexts[level])
break; // successfully linked <new_item> into the skiplist at the current <level>
TRACE("s3", "sl_cas: lost a race. failed to change pred's link. expected %p found %p", nexts[level], other);
// Find <new_item>'s new preds and nexts.
find_preds(preds, nexts, new_item->num_levels, sl, key, ASSIST_UNLINK);
for (int i = level; i < new_item->num_levels; ++i) {
markable_t old_next = new_item->next[i];
if ((markable_t)nexts[i] == old_next)
continue;
// Update <new_item>'s inconsistent next pointer before trying again. Use a CAS so if another thread
// is trying to remove the new item concurrently we do not stomp on the mark it places on the item.
TRACE("s3", "sl_cas: attempting to update the new item's link from %p to %p", old_next, nexts[i]);
other = SYNC_CAS(&new_item->next[i], old_next, (markable_t)nexts[i]);
ASSERT(other == old_next || other == MARK_NODE(old_next));
// If another thread is removing this item we can stop linking it into to skiplist
if (HAS_MARK(other)) {
find_preds(NULL, NULL, 0, sl, key, FORCE_UNLINK); // see comment below
return DOES_NOT_EXIST;
}
}
} while (1);
}
// In case another thread was in the process of removing the <new_item> while we were added it, we have to
// make sure it is completely unlinked before we return. We might have lost a race and inserted the new item
// at some level after the other thread thought it was fully removed. That is a problem because once a thread
// thinks it completely unlinks a node it queues it to be freed
if (HAS_MARK(new_item->next[new_item->num_levels - 1])) {
find_preds(NULL, NULL, 0, sl, key, FORCE_UNLINK);
}
return DOES_NOT_EXIST; // success, inserted a new item
}
int fib6_add(struct fib6_node *root, struct rt6_info *rt,
struct nlmsghdr *nlh, void *_rtattr, struct netlink_skb_parms *req)
{
struct fib6_node *fn;
int err = -ENOMEM;
fn = fib6_add_1(root, &rt->rt6i_dst.addr, sizeof(struct in6_addr),
rt->rt6i_dst.plen, offsetof(struct rt6_info, rt6i_dst));
if (fn == NULL)
goto out;
#ifdef CONFIG_IPV6_SUBTREES
if (rt->rt6i_src.plen) {
struct fib6_node *sn;
if (fn->subtree == NULL) {
struct fib6_node *sfn;
/*
* Create subtree.
*
* fn[main tree]
* |
* sfn[subtree root]
* \
* sn[new leaf node]
*/
/* Create subtree root node */
sfn = node_alloc();
if (sfn == NULL)
goto st_failure;
sfn->leaf = &ip6_null_entry;
atomic_inc(&ip6_null_entry.rt6i_ref);
sfn->fn_flags = RTN_ROOT;
sfn->fn_sernum = fib6_new_sernum();
/* Now add the first leaf node to new subtree */
sn = fib6_add_1(sfn, &rt->rt6i_src.addr,
sizeof(struct in6_addr), rt->rt6i_src.plen,
offsetof(struct rt6_info, rt6i_src));
if (sn == NULL) {
/* If it is failed, discard just allocated
root, and then (in st_failure) stale node
in main tree.
*/
node_free(sfn);
goto st_failure;
}
/* Now link new subtree to main tree */
sfn->parent = fn;
fn->subtree = sfn;
if (fn->leaf == NULL) {
fn->leaf = rt;
atomic_inc(&rt->rt6i_ref);
}
} else {
sn = fib6_add_1(fn->subtree, &rt->rt6i_src.addr,
sizeof(struct in6_addr), rt->rt6i_src.plen,
offsetof(struct rt6_info, rt6i_src));
if (sn == NULL)
goto st_failure;
}
fn = sn;
}
static struct fib6_node * fib6_add_1(struct fib6_node *root, void *addr,
int addrlen, int plen,
int offset)
{
struct fib6_node *fn, *in, *ln;
struct fib6_node *pn = NULL;
struct rt6key *key;
int bit;
int dir = 0;
__u32 sernum = fib6_new_sernum();
RT6_TRACE("fib6_add_1\n");
/* insert node in tree */
fn = root;
do {
key = (struct rt6key *)((u8 *)fn->leaf + offset);
/*
* Prefix match
*/
if (plen < fn->fn_bit ||
!ipv6_prefix_equal(&key->addr, addr, fn->fn_bit))
goto insert_above;
/*
* Exact match ?
*/
if (plen == fn->fn_bit) {
/* clean up an intermediate node */
if ((fn->fn_flags & RTN_RTINFO) == 0) {
rt6_release(fn->leaf);
fn->leaf = NULL;
}
fn->fn_sernum = sernum;
return fn;
}
/*
* We have more bits to go
*/
/* Try to walk down on tree. */
fn->fn_sernum = sernum;
dir = addr_bit_set(addr, fn->fn_bit);
pn = fn;
fn = dir ? fn->right: fn->left;
} while (fn);
/*
* We walked to the bottom of tree.
* Create new leaf node without children.
*/
ln = node_alloc();
if (ln == NULL)
return NULL;
ln->fn_bit = plen;
ln->parent = pn;
ln->fn_sernum = sernum;
if (dir)
pn->right = ln;
else
pn->left = ln;
return ln;
insert_above:
/*
* split since we don't have a common prefix anymore or
* we have a less significant route.
* we've to insert an intermediate node on the list
* this new node will point to the one we need to create
* and the current
*/
pn = fn->parent;
/* find 1st bit in difference between the 2 addrs.
See comment in __ipv6_addr_diff: bit may be an invalid value,
but if it is >= plen, the value is ignored in any case.
*/
bit = __ipv6_addr_diff(addr, &key->addr, addrlen);
/*
* (intermediate)[in]
* / \
* (new leaf node)[ln] (old node)[fn]
*/
if (plen > bit) {
in = node_alloc();
ln = node_alloc();
if (in == NULL || ln == NULL) {
if (in)
node_free(in);
if (ln)
node_free(ln);
return NULL;
}
/*
* new intermediate node.
* RTN_RTINFO will
* be off since that an address that chooses one of
* the branches would not match less specific routes
* in the other branch
*/
in->fn_bit = bit;
in->parent = pn;
in->leaf = fn->leaf;
atomic_inc(&in->leaf->rt6i_ref);
in->fn_sernum = sernum;
/* update parent pointer */
if (dir)
pn->right = in;
else
pn->left = in;
ln->fn_bit = plen;
ln->parent = in;
fn->parent = in;
ln->fn_sernum = sernum;
if (addr_bit_set(addr, bit)) {
in->right = ln;
in->left = fn;
} else {
in->left = ln;
in->right = fn;
}
} else { /* plen <= bit */
/*
* (new leaf node)[ln]
* / \
* (old node)[fn] NULL
*/
ln = node_alloc();
if (ln == NULL)
return NULL;
ln->fn_bit = plen;
ln->parent = pn;
ln->fn_sernum = sernum;
if (dir)
pn->right = ln;
else
pn->left = ln;
if (addr_bit_set(&key->addr, plen))
ln->right = fn;
else
ln->left = fn;
fn->parent = ln;
}
return ln;
}
inline void hash_buckets<A, G>::swap(hash_buckets<A, G>& other)
{
BOOST_ASSERT(node_alloc() == other.node_alloc());
std::swap(buckets_, other.buckets_);
std::swap(bucket_count_, other.bucket_count_);
}
/*
* Inserts item *x and subtree *xr into node p at position k, splitting it into
* nodes p and *xr with median item *x.
*
* Assumes p->count == MAX.
* Ignores original *xr if p is a leaf, but always sets it.
*/
static void node_split(const void **x, struct btree_node **xr,
struct btree_node *p, unsigned int k)
{
unsigned int i, split;
struct btree_node *l = p, *r;
/*
* If k <= MIN, item will be inserted into left subtree, so give l
* fewer items initially.
* Otherwise, item will be inserted into right subtree, so give r
* fewer items initially.
*/
if (k <= MIN)
split = MIN;
else
split = MIN + 1;
/*
* If l->depth is 0, allocate a leaf node.
* Otherwise, allocate an internal node.
*/
r = node_alloc(l->depth);
/* l and r will be siblings, so they will have the same parent and depth. */
r->parent = l->parent;
r->depth = l->depth;
/*
* Initialize items/branches of right side.
* Do not initialize r's leftmost branch yet because we don't know
* whether it will be l's current rightmost branch or if *xr will
* take its place.
*/
for (i = split; i < MAX; i++)
r->item[i-split] = l->item[i];
if (r->depth) {
for (i = split+1; i <= MAX; i++) {
r->branch[i-split] = l->branch[i];
r->branch[i-split]->parent = r;
r->branch[i-split]->k = i-split;
}
}
/* Update counts. */
l->count = split;
r->count = MAX - split;
/*
* The nodes are now split, but the key isn't inserted yet.
*
* Insert key into left or right half,
* depending on which side it fell on.
*/
if (k <= MIN)
node_insert(*x, *xr, l, k);
else
node_insert(*x, *xr, r, k - split);
/*
* Give l's rightmost branch to r because l's rightmost item
* is going up to become the median.
*/
if (r->depth) {
r->branch[0] = l->branch[l->count];
r->branch[0]->parent = r;
r->branch[0]->k = 0;
}
/*
* Take up l's rightmost item to make it the median.
* That item's right branch is now r.
*/
*x = l->item[--l->count];
*xr = r;
}
void btree_insert_at(btree_iterator iter, const void *item)
{
const void *x = item;
struct btree_node *xr = NULL;
struct btree_node *p;
struct btree *btree = iter->btree;
/* btree_insert_at always sets iter->item to item. */
iter->item = (void*)item;
/*
* If node is not a leaf, fall to the end of the left branch of item[k]
* so that it will be a leaf. This does not modify the iterator's logical
* position.
*/
if (iter->node->depth)
branch_end(iter);
/*
* First try inserting item into this node.
* If it's too big, split it, and repeat by
* trying to insert the median and right subtree into parent.
*/
if (iter->node->count < MAX) {
node_insert(x, xr, iter->node, iter->k);
goto finished;
} else {
for (;;) {
node_split(&x, &xr, iter->node, iter->k);
if (!ascend(iter))
break;
if (iter->node->count < MAX) {
node_insert(x, xr, iter->node, iter->k);
goto finished;
}
}
/*
* If splitting came all the way up to the root, create a new root whose
* left branch is the current root, median is x, and right branch is the
* half split off from the root.
*/
assert(iter->node == btree->root);
p = node_alloc(1);
p->parent = NULL;
p->count = 1;
p->depth = btree->root->depth + 1;
p->item[0] = x;
p->branch[0] = btree->root;
btree->root->parent = p;
btree->root->k = 0;
p->branch[1] = xr;
xr->parent = p;
xr->k = 1;
btree->root = p;
}
finished:
btree->count++;
iter->node = NULL;
}
