static inline void sfm_pool_insert (struct sfm_pool* pool, int hash,
struct sfm_frag* frag)
{
unsigned long hash_bit;
SFM_POOL_LOCK(pool, hash);
frag_push(&pool->pool_hash[hash].first, frag);
pool->pool_hash[hash].no++;
/* set it only if not already set (avoids an expensive
* cache trashing atomic write op) */
hash_bit=HASH_TO_BITMAP(hash);
if (!(atomic_get_long((long*)&pool->bitmap) & hash_bit))
atomic_or_long((long*)&pool->bitmap, hash_bit);
SFM_POOL_UNLOCK(pool, hash);
}
void* sfm_realloc(struct sfm_block* qm, void* p, unsigned long size)
#endif
{
struct sfm_frag *f;
unsigned long orig_size;
void *ptr;
#ifndef SFM_REALLOC_REMALLOC
struct sfm_frag *n;
struct sfm_frag **pf;
unsigned long diff;
unsigned long p_id;
int hash;
unsigned long n_size;
struct sfm_pool * pool;
#endif
#ifdef DBG_F_MALLOC
MDBG("sfm_realloc(%p, %p, %lu) called from %s: %s(%d)\n", qm, p, size,
file, func, line);
if ((p)&&(p>(void*)qm->last_frag || p<(void*)qm->first_frag)){
LOG(L_CRIT, "BUG: sfm_free: bad pointer %p (out of memory block!) - "
"aborting\n", p);
abort();
}
#endif
if (size==0) {
if (p)
#ifdef DBG_F_MALLOC
sfm_free(qm, p, file, func, line);
#else
sfm_free(qm, p);
#endif
return 0;
}
if (p==0)
#ifdef DBG_F_MALLOC
return sfm_malloc(qm, size, file, func, line);
#else
return sfm_malloc(qm, size);
#endif
f=(struct sfm_frag*) ((char*)p-sizeof(struct sfm_frag));
#ifdef DBG_F_MALLOC
MDBG("sfm_realloc: realloc'ing frag %p alloc'ed from %s: %s(%ld)\n",
f, f->file, f->func, f->line);
#endif
size=ROUNDUP(size);
orig_size=f->size;
if (f->size > size){
/* shrink */
#ifdef DBG_F_MALLOC
MDBG("sfm_realloc: shrinking from %lu to %lu\n", f->size, size);
sfm_split_frag(qm, f, size, file, "frag. from sfm_realloc", line);
#else
sfm_split_frag(qm, f, size);
#endif
}else if (f->size<size){
/* grow */
#ifdef DBG_F_MALLOC
MDBG("sfm_realloc: growing from %lu to %lu\n", f->size, size);
#endif
#ifndef SFM_REALLOC_REMALLOC
/* should set a magic value in list head and in push/pop if magic value =>
* lock and wait */
#error LL_MALLOC realloc not finished yet
diff=size-f->size;
n=FRAG_NEXT(f);
if (((char*)n < (char*)qm->last_frag) &&
(n->u.nxt_free)&&((n->size+FRAG_OVERHEAD)>=diff)){
/* join */
/* detach n from the free list */
try_again:
p_id=n->id;
n_size=n->size;
if ((unlikely(p_id >=SFM_POOLS_NO))){
hash=GET_HASH(n_size);
SFM_MAIN_HASH_LOCK(qm, hash);
if (unlikely((n->u.nxt_free==0) ||
((n->size+FRAG_OVERHEAD)<diff))){
SFM_MAIN_HASH_UNLOCK(qm, hash);
goto not_found;
}
if (unlikely((n->id!=p_id) || (n->size!=n_size))){
/* fragment still free, but changed, either
* moved to another pool or has a diff. size */
SFM_MAIN_HASH_UNLOCK(qm, hash);
goto try_again;
}
pf=&(qm->free_hash[hash].first);
/* find it */
for(;(*pf)&&(*pf!=n); pf=&((*pf)->u.nxt_free));/*FIXME slow */
if (*pf==0){
SFM_MAIN_HASH_UNLOCK(qm, hash);
/* not found, bad! */
LOG(L_WARN, "WARNING: sfm_realloc: could not find %p in "
"free " "list (hash=%d)\n", n, hash);
/* somebody is in the process of changing it ? */
goto not_found;
}
/* detach */
*pf=n->u.nxt_free;
n->u.nxt_free=0; /* mark it immediately as detached */
qm->free_hash[hash].no--;
SFM_MAIN_HASH_UNLOCK(qm, hash);
/* join */
f->size+=n->size+FRAG_OVERHEAD;
/* split it if necessary */
if (f->size > size){
#ifdef DBG_F_MALLOC
sfm_split_frag(qm, f, size, file, "fragm. from "
"sfm_realloc", line);
#else
sfm_split_frag(qm, f, size);
#endif
}
}else{ /* p_id < SFM_POOLS_NO (=> in a pool )*/
hash=GET_SMALL_HASH(n_size);
pool=&qm->pool[p_id];
SFM_POOL_LOCK(pool, hash);
if (unlikely((n->u.nxt_free==0) ||
((n->size+FRAG_OVERHEAD)<diff))){
SFM_POOL_UNLOCK(pool, hash);
goto not_found;
}
if (unlikely((n->id!=p_id) || (n->size!=n_size))){
/* fragment still free, but changed, either
* moved to another pool or has a diff. size */
SFM_POOL_UNLOCK(pool, hash);
goto try_again;
}
pf=&(pool->pool_hash[hash].first);
/* find it */
for(;(*pf)&&(*pf!=n); pf=&((*pf)->u.nxt_free));/*FIXME slow */
if (*pf==0){
SFM_POOL_UNLOCK(pool, hash);
/* not found, bad! */
LOG(L_WARN, "WARNING: sfm_realloc: could not find %p in "
"free " "list (hash=%d)\n", n, hash);
/* somebody is in the process of changing it ? */
goto not_found;
}
/* detach */
*pf=n->u.nxt_free;
n->u.nxt_free=0; /* mark it immediately as detached */
pool->pool_hash[hash].no--;
SFM_POOL_UNLOCK(pool, hash);
/* join */
f->size+=n->size+FRAG_OVERHEAD;
/* split it if necessary */
if (f->size > size){
#ifdef DBG_F_MALLOC
sfm_split_frag(qm, f, size, file, "fragm. from "
"sfm_realloc", line);
#else
sfm_split_frag(qm, f, size);
#endif
}
}
}else{
not_found:
/* could not join => realloc */
#else/* SFM_REALLOC_REMALLOC */
{
#endif /* SFM_REALLOC_REMALLOC */
#ifdef DBG_F_MALLOC
ptr=sfm_malloc(qm, size, file, func, line);
#else
ptr=sfm_malloc(qm, size);
#endif
if (ptr){
/* copy, need by libssl */
memcpy(ptr, p, orig_size);
#ifdef DBG_F_MALLOC
sfm_free(qm, p, file, func, line);
#else
sfm_free(qm, p);
#endif
}
p=ptr;
}
}else{
static inline struct sfm_frag* pool_get_frag(struct sfm_block* qm,
struct sfm_pool* pool,
int hash, unsigned long size)
#endif
{
int r;
int next_block;
struct sfm_frag* volatile* f;
struct sfm_frag* frag;
unsigned long b;
unsigned long eob;
/* special case for r=hash */
r=hash;
f=&pool->pool_hash[r].first;
if (*f==0)
goto not_found;
SFM_POOL_LOCK(pool, r);
if (unlikely(*f==0)){
SFM_POOL_UNLOCK(pool, r);
goto not_found;
}
found:
/* detach it from the free list*/
frag=frag_pop((struct sfm_frag**)f);
frag->u.nxt_free=0; /* mark it as 'taken' */
frag->id=pool_id;
pool->pool_hash[r].no--;
SFM_POOL_UNLOCK(pool, r);
#ifdef DBG_F_MALLOC
sfm_split_frag(qm, frag, size, file, func, line);
#else
sfm_split_frag(qm, frag, size);
#endif
if (&qm->pool[pool_id]==pool)
atomic_inc_long((long*)&pool->hits);
return frag;
not_found:
atomic_inc_long((long*)&pool->pool_hash[r].misses);
r++;
b=HASH_BIT_POS(r);
while(r<SF_HASH_POOL_SIZE){
b=_next_set_bit(b, &pool->bitmap);
next_block=_hash_range(b, &eob);
r=(r<next_block)?next_block:r;
for (; r<eob; r++){
f=&pool->pool_hash[r].first;
if (*f){
SFM_POOL_LOCK(pool, r);
if (unlikely(*f==0)){
/* not found */
SFM_POOL_UNLOCK(pool, r);
}else
goto found;
}
atomic_inc_long((long*)&pool->pool_hash[r].misses);
}
b++;
}
#if 0 /* EXPENSIVE BUG CHECK */
for (r=hash; r<SF_HASH_POOL_SIZE; r++){
f=&pool->pool_hash[r].first;
if (*f){
SFM_POOL_LOCK(pool, r);
if (unlikely(*f==0)){
/* not found */
SFM_POOL_UNLOCK(pool, r);
}else{
b=_next_set_bit(HASH_BIT_POS(r), &pool->bitmap);
next_block=_hash_range(b, &eob);
BUG("pool_get_frag: found fragm. %d at %d (bit %ld range %ld-%ld), next set bit=%ld"
" bitmap %ld (%p)\n", hash, r, HASH_BIT_POS(r),
next_block, eob, b, pool->bitmap, &pool->bitmap);
goto found;
}
}
}
#endif
atomic_inc_long((long*)&pool->missed);
return 0;
}
