/*
Returns a valid lvalue pointer to the element number 'idx'.
Allocates memory if necessary.
*/
void *lf_dynarray_lvalue(LF_DYNARRAY *array, uint idx)
{
void * ptr, * volatile * ptr_ptr= 0;
int i;
for (i= LF_DYNARRAY_LEVELS-1; idx < dynarray_idxes_in_prev_levels[i]; i--)
/* no-op */;
ptr_ptr= &array->level[i];
idx-= dynarray_idxes_in_prev_levels[i];
for (; i > 0; i--)
{
if (!(ptr= *ptr_ptr))
{
void *alloc= my_malloc(key_memory_lf_dynarray,
LF_DYNARRAY_LEVEL_LENGTH * sizeof(void *),
MYF(MY_WME|MY_ZEROFILL));
if (unlikely(!alloc))
return(NULL);
if (my_atomic_casptr(ptr_ptr, &ptr, alloc))
ptr= alloc;
else
my_free(alloc);
}
ptr_ptr= ((void **)ptr) + idx / dynarray_idxes_in_prev_level[i];
idx%= dynarray_idxes_in_prev_level[i];
}
if (!(ptr= *ptr_ptr))
{
uchar *alloc, *data;
alloc= my_malloc(key_memory_lf_dynarray,
LF_DYNARRAY_LEVEL_LENGTH * array->size_of_element +
MY_MAX(array->size_of_element, sizeof(void *)),
MYF(MY_WME|MY_ZEROFILL));
if (unlikely(!alloc))
return(NULL);
/* reserve the space for free() address */
data= alloc + sizeof(void *);
{ /* alignment */
intptr mod= ((intptr)data) % array->size_of_element;
if (mod)
data+= array->size_of_element - mod;
}
((void **)data)[-1]= alloc; /* free() will need the original pointer */
if (my_atomic_casptr(ptr_ptr, &ptr, data))
ptr= data;
else
my_free(alloc);
}
return ((uchar*)ptr) + array->size_of_element * idx;
}
/*
Allocate and return an new object.
DESCRIPTION
Pop an unused object from the stack or malloc it is the stack is empty.
pin[0] is used, it's removed on return.
*/
void *_lf_alloc_new(LF_PINS *pins)
{
LF_ALLOCATOR *allocator= (LF_ALLOCATOR *)(pins->pinbox->free_func_arg);
uchar *node;
for (;;)
{
do
{
node= allocator->top;
_lf_pin(pins, 0, node);
} while (node != allocator->top && LF_BACKOFF);
if (!node)
{
node= (void *)my_malloc(allocator->element_size, MYF(MY_WME));
if (allocator->constructor)
allocator->constructor(node);
#ifdef MY_LF_EXTRA_DEBUG
if (likely(node != 0))
my_atomic_add32(&allocator->mallocs, 1);
#endif
break;
}
if (my_atomic_casptr((void **)(char *)&allocator->top,
(void *)&node, anext_node(node)))
break;
}
_lf_unpin(pins, 0);
return node;
}
/*
RETURN
0 - ok
-1 - out of memory
*/
static int initialize_bucket(LF_HASH *hash, LF_SLIST * volatile *node,
uint bucket, LF_PINS *pins)
{
uint parent= my_clear_highest_bit(bucket);
LF_SLIST *dummy= (LF_SLIST *)my_malloc(sizeof(LF_SLIST), MYF(MY_WME));
LF_SLIST **tmp= 0, *cur;
LF_SLIST * volatile *el= _lf_dynarray_lvalue(&hash->array, parent);
if (unlikely(!el || !dummy))
return -1;
if (*el == NULL && bucket &&
unlikely(initialize_bucket(hash, el, parent, pins)))
return -1;
dummy->hashnr= my_reverse_bits(bucket) | 0; /* dummy node */
dummy->key= dummy_key;
dummy->keylen= 0;
if ((cur= linsert(el, hash->charset, dummy, pins, LF_HASH_UNIQUE)))
{
my_free(dummy);
dummy= cur;
}
my_atomic_casptr((void **)node, (void **)(char*) &tmp, dummy);
/*
note that if the CAS above failed (after linsert() succeeded),
it would mean that some other thread has executed linsert() for
the same dummy node, its linsert() failed, it picked up our
dummy node (in "dummy= cur") and executed the same CAS as above.
Which means that even if CAS above failed we don't need to retry,
and we should not free(dummy) - there's no memory leak here
*/
return 0;
}
/*
DESCRIPTION
deletes a node as identified by hashnr/keey/keylen from the list
that starts from 'head'
RETURN
0 - ok
1 - not found
NOTE
it uses pins[0..2], on return all pins are removed.
*/
static int ldelete(LF_SLIST * volatile *head, CHARSET_INFO *cs, uint32 hashnr,
const uchar *key, uint keylen, LF_PINS *pins)
{
CURSOR cursor;
int res;
for (;;)
{
if (!lfind(head, cs, hashnr, key, keylen, &cursor, pins))
{
res= 1; /* not found */
break;
}
else
{
/* mark the node deleted */
if (my_atomic_casptr((void **) (char*) &(cursor.curr->link),
(void **) (char*) &cursor.next,
(void *)(((intptr)cursor.next) | 1)))
{
/* and remove it from the list */
if (my_atomic_casptr((void **)cursor.prev,
(void **)(char*)&cursor.curr, cursor.next))
_lf_alloc_free(pins, cursor.curr);
else
{
/*
somebody already "helped" us and removed the node ?
Let's check if we need to help that someone too!
(to ensure the number of "set DELETED flag" actions
is equal to the number of "remove from the list" actions)
*/
lfind(head, cs, hashnr, key, keylen, &cursor, pins);
}
res= 0;
break;
}
}
}
_lf_unpin(pins, 0);
_lf_unpin(pins, 1);
_lf_unpin(pins, 2);
return res;
}
/*
callback for _lf_pinbox_real_free to free a list of unpinned objects -
add it back to the allocator stack
DESCRIPTION
'first' and 'last' are the ends of the linked list of nodes:
first->el->el->....->el->last. Use first==last to free only one element.
*/
static void alloc_free(uchar *first,
uchar volatile *last,
LF_ALLOCATOR *allocator)
{
/*
we need a union here to access type-punned pointer reliably.
otherwise gcc -fstrict-aliasing will not see 'tmp' changed in the loop
*/
union { uchar * node; void *ptr; } tmp;
tmp.node= allocator->top;
do
{
anext_node(last)= tmp.node;
} while (!my_atomic_casptr((void **)(char *)&allocator->top,
(void **)&tmp.ptr, first) && LF_BACKOFF);
}
/*
DESCRIPTION
insert a 'node' in the list that starts from 'head' in the correct
position (as found by lfind)
RETURN
0 - inserted
not 0 - a pointer to a duplicate (not pinned and thus unusable)
NOTE
it uses pins[0..2], on return all pins are removed.
if there're nodes with the same key value, a new node is added before them.
*/
static LF_SLIST *linsert(LF_SLIST * volatile *head, CHARSET_INFO *cs,
LF_SLIST *node, LF_PINS *pins, uint flags)
{
CURSOR cursor;
int res;
for (;;)
{
if (lfind(head, cs, node->hashnr, node->key, node->keylen,
&cursor, pins) &&
(flags & LF_HASH_UNIQUE))
{
res= 0; /* duplicate found */
break;
}
else
{
node->link= (intptr)cursor.curr;
DBUG_ASSERT(node->link != (intptr)node); /* no circular references */
DBUG_ASSERT(cursor.prev != &node->link); /* no circular references */
if (my_atomic_casptr((void **) cursor.prev,
(void **)(char*) &cursor.curr, node))
{
res= 1; /* inserted ok */
break;
}
}
}
_lf_unpin(pins, 0);
_lf_unpin(pins, 1);
_lf_unpin(pins, 2);
/*
Note that cursor.curr is not pinned here and the pointer is unreliable,
the object may dissapear anytime. But if it points to a dummy node, the
pointer is safe, because dummy nodes are never freed - initialize_bucket()
uses this fact.
*/
return res ? 0 : cursor.curr;
}
/*
DESCRIPTION
Search for hashnr/key/keylen in the list starting from 'head' and
position the cursor. The list is ORDER BY hashnr, key
RETURN
0 - not found
1 - found
NOTE
cursor is positioned in either case
pins[0..2] are used, they are NOT removed on return
*/
static int lfind(LF_SLIST * volatile *head, CHARSET_INFO *cs, uint32 hashnr,
const uchar *key, uint keylen, CURSOR *cursor, LF_PINS *pins)
{
uint32 cur_hashnr;
const uchar *cur_key;
uint cur_keylen;
intptr link;
retry:
cursor->prev= (intptr *)head;
do { /* PTR() isn't necessary below, head is a dummy node */
cursor->curr= (LF_SLIST *)(*cursor->prev);
_lf_pin(pins, 1, cursor->curr);
} while (*cursor->prev != (intptr)cursor->curr && LF_BACKOFF);
for (;;)
{
if (unlikely(!cursor->curr))
return 0; /* end of the list */
do {
/* QQ: XXX or goto retry ? */
link= cursor->curr->link;
cursor->next= PTR(link);
_lf_pin(pins, 0, cursor->next);
} while (link != cursor->curr->link && LF_BACKOFF);
cur_hashnr= cursor->curr->hashnr;
cur_key= cursor->curr->key;
cur_keylen= cursor->curr->keylen;
if (*cursor->prev != (intptr)cursor->curr)
{
(void)LF_BACKOFF;
goto retry;
}
if (!DELETED(link))
{
if (cur_hashnr >= hashnr)
{
int r= 1;
if (cur_hashnr > hashnr ||
(r= my_strnncoll(cs, (uchar*) cur_key, cur_keylen, (uchar*) key,
keylen)) >= 0)
return !r;
}
cursor->prev= &(cursor->curr->link);
_lf_pin(pins, 2, cursor->curr);
}
else
{
/*
we found a deleted node - be nice, help the other thread
and remove this deleted node
*/
if (my_atomic_casptr((void **) cursor->prev,
(void **)(char*) &cursor->curr, cursor->next))
_lf_alloc_free(pins, cursor->curr);
else
{
(void)LF_BACKOFF;
goto retry;
}
}
cursor->curr= cursor->next;
_lf_pin(pins, 1, cursor->curr);
}
}
static int lfind_match(LF_SLIST * volatile *head,
uint32 first_hashnr, uint32 last_hashnr,
lf_hash_match_func *match,
CURSOR *cursor, LF_PINS *pins)
{
uint32 cur_hashnr;
intptr link;
retry:
cursor->prev= (intptr *)head;
do { /* PTR() isn't necessary below, head is a dummy node */
cursor->curr= (LF_SLIST *)(*cursor->prev);
lf_pin(pins, 1, cursor->curr);
} while (*cursor->prev != (intptr)cursor->curr && LF_BACKOFF);
for (;;)
{
if (unlikely(!cursor->curr))
return 0; /* end of the list */
do {
/* QQ: XXX or goto retry ? */
link= cursor->curr->link;
cursor->next= PTR(link);
lf_pin(pins, 0, cursor->next);
} while (link != cursor->curr->link && LF_BACKOFF);
cur_hashnr= cursor->curr->hashnr;
if (*cursor->prev != (intptr)cursor->curr)
{
(void)LF_BACKOFF;
goto retry;
}
if (!DELETED(link))
{
if (cur_hashnr >= first_hashnr)
{
if (cur_hashnr > last_hashnr)
return 0;
if (cur_hashnr & 1)
{
/* Normal node. Check if element matches condition. */
if ((*match)((uchar *)(cursor->curr + 1)))
return 1;
}
else
{
/*
Dummy node. Nothing to check here.
Still thanks to the fact that dummy nodes are never deleted we
can save it as a safe place to restart iteration if ever needed.
*/
head= (LF_SLIST * volatile *)&(cursor->curr->link);
}
}
cursor->prev= &(cursor->curr->link);
lf_pin(pins, 2, cursor->curr);
}
else
{
/*
we found a deleted node - be nice, help the other thread
and remove this deleted node
*/
if (my_atomic_casptr((void **)cursor->prev,
(void **)&cursor->curr, cursor->next))
lf_pinbox_free(pins, cursor->curr);
else
{
(void)LF_BACKOFF;
goto retry;
}
}
cursor->curr= cursor->next;
lf_pin(pins, 1, cursor->curr);
}
}
/** walk the list, searching for an element or invoking a callback
Search for hashnr/key/keylen in the list starting from 'head' and
position the cursor. The list is ORDER BY hashnr, key
@param head start walking the list from this node
@param cs charset for comparing keys, NULL if callback is used
@param hashnr hash number to search for
@param key key to search for OR data for the callback
@param keylen length of the key to compare, 0 if callback is used
@param cursor for returning the found element
@param pins see lf_alloc-pin.c
@param callback callback action, invoked for every element
@note
cursor is positioned in either case
pins[0..2] are used, they are NOT removed on return
callback might see some elements twice (because of retries)
@return
if find: 0 - not found
1 - found
if callback:
0 - ok
1 - error (callbck returned 1)
*/
static int l_find(LF_SLIST * volatile *head, CHARSET_INFO *cs, uint32 hashnr,
const uchar *key, uint keylen, CURSOR *cursor, LF_PINS *pins,
my_hash_walk_action callback)
{
uint32 cur_hashnr;
const uchar *cur_key;
uint cur_keylen;
intptr link;
DBUG_ASSERT(!cs || !callback); /* should not be set both */
DBUG_ASSERT(!keylen || !callback); /* should not be set both */
retry:
cursor->prev= (intptr *)head;
do { /* PTR() isn't necessary below, head is a dummy node */
cursor->curr= (LF_SLIST *)(*cursor->prev);
lf_pin(pins, 1, cursor->curr);
} while (*cursor->prev != (intptr)cursor->curr && LF_BACKOFF);
for (;;)
{
if (unlikely(!cursor->curr))
return 0; /* end of the list */
cur_hashnr= cursor->curr->hashnr;
cur_keylen= cursor->curr->keylen;
cur_key= cursor->curr->key;
do {
link= cursor->curr->link;
cursor->next= PTR(link);
lf_pin(pins, 0, cursor->next);
} while (link != cursor->curr->link && LF_BACKOFF);
if (!DELETED(link))
{
if (unlikely(callback))
{
if (cur_hashnr & 1 && callback(cursor->curr + 1, (void*)key))
return 1;
}
else if (cur_hashnr >= hashnr)
{
int r= 1;
if (cur_hashnr > hashnr ||
(r= my_strnncoll(cs, cur_key, cur_keylen, key, keylen)) >= 0)
return !r;
}
cursor->prev= &(cursor->curr->link);
if (!(cur_hashnr & 1)) /* dummy node */
head= (LF_SLIST **)cursor->prev;
lf_pin(pins, 2, cursor->curr);
}
else
{
/*
we found a deleted node - be nice, help the other thread
and remove this deleted node
*/
if (my_atomic_casptr((void **) cursor->prev,
(void **) &cursor->curr, cursor->next) && LF_BACKOFF)
lf_alloc_free(pins, cursor->curr);
else
goto retry;
}
cursor->curr= cursor->next;
lf_pin(pins, 1, cursor->curr);
}
}
