static inline void sfm_insert_free(struct sfm_block* qm, struct sfm_frag* frag,
int split)
{
struct sfm_frag** f;
unsigned long p_id;
int hash;
unsigned long hash_bit;
if (likely(frag->size<=SF_POOL_MAX_SIZE)){
hash=GET_SMALL_HASH(frag->size);
if (unlikely((p_id=sfm_choose_pool(qm, frag, hash, split))==
(unsigned long)-1)){
/* add it back to the "main" hash */
SFM_MAIN_HASH_LOCK(qm, hash);
frag->id=(unsigned long)(-1); /* main hash marker */
/*insert it here*/
frag_push(&(qm->free_hash[hash].first), frag);
qm->free_hash[hash].no++;
/* set it only if not already set (avoids an expensive
* cache trashing atomic write op) */
hash_bit=HASH_TO_BITMAP(hash);
if (!(atomic_get_long((long*)&qm->bitmap) & hash_bit))
atomic_or_long((long*)&qm->bitmap, hash_bit);
SFM_MAIN_HASH_UNLOCK(qm, hash);
}else{
/* add it to one of the pools pool */
sfm_pool_insert(&qm->pool[p_id], hash, frag);
}
}else{
hash=GET_BIG_HASH(frag->size);
SFM_MAIN_HASH_LOCK(qm, hash);
f=&(qm->free_hash[hash].first);
for(; *f; f=&((*f)->u.nxt_free))
if (frag->size <= (*f)->size) break;
frag->id=(unsigned long)(-1); /* main hash marker */
/*insert it here*/
frag->u.nxt_free=*f;
*f=frag;
qm->free_hash[hash].no++;
/* inc. big hash free size ? */
SFM_MAIN_HASH_UNLOCK(qm, hash);
}
}
void* sfm_realloc(struct sfm_block* qm, void* p, unsigned long size)
#endif
{
struct sfm_frag *f;
unsigned long orig_size;
void *ptr;
#ifndef SFM_REALLOC_REMALLOC
struct sfm_frag *n;
struct sfm_frag **pf;
unsigned long diff;
unsigned long p_id;
int hash;
unsigned long n_size;
struct sfm_pool * pool;
#endif
#ifdef DBG_F_MALLOC
MDBG("sfm_realloc(%p, %p, %lu) called from %s: %s(%d)\n", qm, p, size,
file, func, line);
if ((p)&&(p>(void*)qm->last_frag || p<(void*)qm->first_frag)){
LOG(L_CRIT, "BUG: sfm_free: bad pointer %p (out of memory block!) - "
"aborting\n", p);
abort();
}
#endif
if (size==0) {
if (p)
#ifdef DBG_F_MALLOC
sfm_free(qm, p, file, func, line);
#else
sfm_free(qm, p);
#endif
return 0;
}
if (p==0)
#ifdef DBG_F_MALLOC
return sfm_malloc(qm, size, file, func, line);
#else
return sfm_malloc(qm, size);
#endif
f=(struct sfm_frag*) ((char*)p-sizeof(struct sfm_frag));
#ifdef DBG_F_MALLOC
MDBG("sfm_realloc: realloc'ing frag %p alloc'ed from %s: %s(%ld)\n",
f, f->file, f->func, f->line);
#endif
size=ROUNDUP(size);
orig_size=f->size;
if (f->size > size){
/* shrink */
#ifdef DBG_F_MALLOC
MDBG("sfm_realloc: shrinking from %lu to %lu\n", f->size, size);
sfm_split_frag(qm, f, size, file, "frag. from sfm_realloc", line);
#else
sfm_split_frag(qm, f, size);
#endif
}else if (f->size<size){
/* grow */
#ifdef DBG_F_MALLOC
MDBG("sfm_realloc: growing from %lu to %lu\n", f->size, size);
#endif
#ifndef SFM_REALLOC_REMALLOC
/* should set a magic value in list head and in push/pop if magic value =>
* lock and wait */
#error LL_MALLOC realloc not finished yet
diff=size-f->size;
n=FRAG_NEXT(f);
if (((char*)n < (char*)qm->last_frag) &&
(n->u.nxt_free)&&((n->size+FRAG_OVERHEAD)>=diff)){
/* join */
/* detach n from the free list */
try_again:
p_id=n->id;
n_size=n->size;
if ((unlikely(p_id >=SFM_POOLS_NO))){
hash=GET_HASH(n_size);
SFM_MAIN_HASH_LOCK(qm, hash);
if (unlikely((n->u.nxt_free==0) ||
((n->size+FRAG_OVERHEAD)<diff))){
SFM_MAIN_HASH_UNLOCK(qm, hash);
goto not_found;
}
if (unlikely((n->id!=p_id) || (n->size!=n_size))){
/* fragment still free, but changed, either
* moved to another pool or has a diff. size */
SFM_MAIN_HASH_UNLOCK(qm, hash);
goto try_again;
}
pf=&(qm->free_hash[hash].first);
/* find it */
for(;(*pf)&&(*pf!=n); pf=&((*pf)->u.nxt_free));/*FIXME slow */
if (*pf==0){
SFM_MAIN_HASH_UNLOCK(qm, hash);
/* not found, bad! */
LOG(L_WARN, "WARNING: sfm_realloc: could not find %p in "
"free " "list (hash=%d)\n", n, hash);
/* somebody is in the process of changing it ? */
goto not_found;
}
/* detach */
*pf=n->u.nxt_free;
n->u.nxt_free=0; /* mark it immediately as detached */
qm->free_hash[hash].no--;
SFM_MAIN_HASH_UNLOCK(qm, hash);
/* join */
f->size+=n->size+FRAG_OVERHEAD;
/* split it if necessary */
if (f->size > size){
#ifdef DBG_F_MALLOC
sfm_split_frag(qm, f, size, file, "fragm. from "
"sfm_realloc", line);
#else
sfm_split_frag(qm, f, size);
#endif
}
}else{ /* p_id < SFM_POOLS_NO (=> in a pool )*/
hash=GET_SMALL_HASH(n_size);
pool=&qm->pool[p_id];
SFM_POOL_LOCK(pool, hash);
if (unlikely((n->u.nxt_free==0) ||
((n->size+FRAG_OVERHEAD)<diff))){
SFM_POOL_UNLOCK(pool, hash);
goto not_found;
}
if (unlikely((n->id!=p_id) || (n->size!=n_size))){
/* fragment still free, but changed, either
* moved to another pool or has a diff. size */
SFM_POOL_UNLOCK(pool, hash);
goto try_again;
}
pf=&(pool->pool_hash[hash].first);
/* find it */
for(;(*pf)&&(*pf!=n); pf=&((*pf)->u.nxt_free));/*FIXME slow */
if (*pf==0){
SFM_POOL_UNLOCK(pool, hash);
/* not found, bad! */
LOG(L_WARN, "WARNING: sfm_realloc: could not find %p in "
"free " "list (hash=%d)\n", n, hash);
/* somebody is in the process of changing it ? */
goto not_found;
}
/* detach */
*pf=n->u.nxt_free;
n->u.nxt_free=0; /* mark it immediately as detached */
pool->pool_hash[hash].no--;
SFM_POOL_UNLOCK(pool, hash);
/* join */
f->size+=n->size+FRAG_OVERHEAD;
/* split it if necessary */
if (f->size > size){
#ifdef DBG_F_MALLOC
sfm_split_frag(qm, f, size, file, "fragm. from "
"sfm_realloc", line);
#else
sfm_split_frag(qm, f, size);
#endif
}
}
}else{
not_found:
/* could not join => realloc */
#else/* SFM_REALLOC_REMALLOC */
{
#endif /* SFM_REALLOC_REMALLOC */
#ifdef DBG_F_MALLOC
ptr=sfm_malloc(qm, size, file, func, line);
#else
ptr=sfm_malloc(qm, size);
#endif
if (ptr){
/* copy, need by libssl */
memcpy(ptr, p, orig_size);
#ifdef DBG_F_MALLOC
sfm_free(qm, p, file, func, line);
#else
sfm_free(qm, p);
#endif
}
p=ptr;
}
}else{
static inline struct sfm_frag* main_get_frag(struct sfm_block* qm, int hash,
unsigned long size)
#endif
{
int r;
int next_block;
struct sfm_frag* volatile* f;
struct sfm_frag* frag;
unsigned long b;
unsigned long eob;
r=hash;
b=HASH_BIT_POS(r);
while(r<=SF_MALLOC_OPTIMIZE/SF_ROUNDTO){
b=_next_set_bit(b, &qm->bitmap);
next_block=_hash_range(b, &eob);
r=(r<next_block)?next_block:r;
for (; r<eob; r++){
f=&qm->free_hash[r].first;
if ((frag=frag_pop((struct sfm_frag**)f))!=0){
atomic_dec_long((long*)&qm->free_hash[r].no);
frag->u.nxt_free=0; /* mark it as 'taken' */
frag->id=pool_id;
#ifdef DBG_F_MALLOC
sfm_split_frag(qm, frag, size, file, func, line);
#else
sfm_split_frag(qm, frag, size);
#endif
return frag;
}
}
b++;
}
/* big fragments */
SFM_BIG_GET_AND_SPLIT_LOCK(qm);
for (; r<= sfm_max_hash ; r++){
f=&qm->free_hash[r].first;
if (*f){
SFM_MAIN_HASH_LOCK(qm, r);
if (unlikely((*f)==0)){
/* not found */
SFM_MAIN_HASH_UNLOCK(qm, r);
continue;
}
for(;(*f); f=&((*f)->u.nxt_free))
if ((*f)->size>=size){
/* found, detach it from the free list*/
frag=*f;
*f=frag->u.nxt_free;
frag->u.nxt_free=0; /* mark it as 'taken' */
qm->free_hash[r].no--;
SFM_MAIN_HASH_UNLOCK(qm, r);
frag->id=pool_id;
#ifdef DBG_F_MALLOC
sfm_split_frag(qm, frag, size, file, func, line);
#else
sfm_split_frag(qm, frag, size);
#endif
SFM_BIG_GET_AND_SPLIT_UNLOCK(qm);
return frag;
};
SFM_MAIN_HASH_UNLOCK(qm, r);
/* try in a bigger bucket */
}
}
SFM_BIG_GET_AND_SPLIT_UNLOCK(qm);
return 0;
}