static inline void sfm_insert_free(struct sfm_block* qm, struct sfm_frag* frag,
int split)
{
struct sfm_frag** f;
unsigned long p_id;
int hash;
unsigned long hash_bit;
if (likely(frag->size<=SF_POOL_MAX_SIZE)){
hash=GET_SMALL_HASH(frag->size);
if (unlikely((p_id=sfm_choose_pool(qm, frag, hash, split))==
(unsigned long)-1)){
/* add it back to the "main" hash */
SFM_MAIN_HASH_LOCK(qm, hash);
frag->id=(unsigned long)(-1); /* main hash marker */
/*insert it here*/
frag_push(&(qm->free_hash[hash].first), frag);
qm->free_hash[hash].no++;
/* set it only if not already set (avoids an expensive
* cache trashing atomic write op) */
hash_bit=HASH_TO_BITMAP(hash);
if (!(atomic_get_long((long*)&qm->bitmap) & hash_bit))
atomic_or_long((long*)&qm->bitmap, hash_bit);
SFM_MAIN_HASH_UNLOCK(qm, hash);
}else{
/* add it to one of the pools pool */
sfm_pool_insert(&qm->pool[p_id], hash, frag);
}
}else{
hash=GET_BIG_HASH(frag->size);
SFM_MAIN_HASH_LOCK(qm, hash);
f=&(qm->free_hash[hash].first);
for(; *f; f=&((*f)->u.nxt_free))
if (frag->size <= (*f)->size) break;
frag->id=(unsigned long)(-1); /* main hash marker */
/*insert it here*/
frag->u.nxt_free=*f;
*f=frag;
qm->free_hash[hash].no++;
/* inc. big hash free size ? */
SFM_MAIN_HASH_UNLOCK(qm, hash);
}
}
void* sfm_realloc(struct sfm_block* qm, void* p, unsigned long size)
#endif
{
struct sfm_frag *f;
unsigned long orig_size;
void *ptr;
#ifndef SFM_REALLOC_REMALLOC
struct sfm_frag *n;
struct sfm_frag **pf;
unsigned long diff;
unsigned long p_id;
int hash;
unsigned long n_size;
struct sfm_pool * pool;
#endif
#ifdef DBG_F_MALLOC
MDBG("sfm_realloc(%p, %p, %lu) called from %s: %s(%d)\n", qm, p, size,
file, func, line);
if ((p)&&(p>(void*)qm->last_frag || p<(void*)qm->first_frag)){
LOG(L_CRIT, "BUG: sfm_free: bad pointer %p (out of memory block!) - "
"aborting\n", p);
abort();
}
#endif
if (size==0) {
if (p)
#ifdef DBG_F_MALLOC
sfm_free(qm, p, file, func, line);
#else
sfm_free(qm, p);
#endif
return 0;
}
if (p==0)
#ifdef DBG_F_MALLOC
return sfm_malloc(qm, size, file, func, line);
#else
return sfm_malloc(qm, size);
#endif
f=(struct sfm_frag*) ((char*)p-sizeof(struct sfm_frag));
#ifdef DBG_F_MALLOC
MDBG("sfm_realloc: realloc'ing frag %p alloc'ed from %s: %s(%ld)\n",
f, f->file, f->func, f->line);
#endif
size=ROUNDUP(size);
orig_size=f->size;
if (f->size > size){
/* shrink */
#ifdef DBG_F_MALLOC
MDBG("sfm_realloc: shrinking from %lu to %lu\n", f->size, size);
sfm_split_frag(qm, f, size, file, "frag. from sfm_realloc", line);
#else
sfm_split_frag(qm, f, size);
#endif
}else if (f->size<size){
/* grow */
#ifdef DBG_F_MALLOC
MDBG("sfm_realloc: growing from %lu to %lu\n", f->size, size);
#endif
#ifndef SFM_REALLOC_REMALLOC
/* should set a magic value in list head and in push/pop if magic value =>
* lock and wait */
#error LL_MALLOC realloc not finished yet
diff=size-f->size;
n=FRAG_NEXT(f);
if (((char*)n < (char*)qm->last_frag) &&
(n->u.nxt_free)&&((n->size+FRAG_OVERHEAD)>=diff)){
/* join */
/* detach n from the free list */
try_again:
p_id=n->id;
n_size=n->size;
if ((unlikely(p_id >=SFM_POOLS_NO))){
hash=GET_HASH(n_size);
SFM_MAIN_HASH_LOCK(qm, hash);
if (unlikely((n->u.nxt_free==0) ||
((n->size+FRAG_OVERHEAD)<diff))){
SFM_MAIN_HASH_UNLOCK(qm, hash);
goto not_found;
}
if (unlikely((n->id!=p_id) || (n->size!=n_size))){
/* fragment still free, but changed, either
* moved to another pool or has a diff. size */
SFM_MAIN_HASH_UNLOCK(qm, hash);
goto try_again;
}
pf=&(qm->free_hash[hash].first);
/* find it */
for(;(*pf)&&(*pf!=n); pf=&((*pf)->u.nxt_free));/*FIXME slow */
if (*pf==0){
SFM_MAIN_HASH_UNLOCK(qm, hash);
/* not found, bad! */
LOG(L_WARN, "WARNING: sfm_realloc: could not find %p in "
"free " "list (hash=%d)\n", n, hash);
/* somebody is in the process of changing it ? */
goto not_found;
}
/* detach */
*pf=n->u.nxt_free;
n->u.nxt_free=0; /* mark it immediately as detached */
qm->free_hash[hash].no--;
SFM_MAIN_HASH_UNLOCK(qm, hash);
/* join */
f->size+=n->size+FRAG_OVERHEAD;
/* split it if necessary */
if (f->size > size){
#ifdef DBG_F_MALLOC
sfm_split_frag(qm, f, size, file, "fragm. from "
"sfm_realloc", line);
#else
sfm_split_frag(qm, f, size);
#endif
}
}else{ /* p_id < SFM_POOLS_NO (=> in a pool )*/
hash=GET_SMALL_HASH(n_size);
pool=&qm->pool[p_id];
SFM_POOL_LOCK(pool, hash);
if (unlikely((n->u.nxt_free==0) ||
((n->size+FRAG_OVERHEAD)<diff))){
SFM_POOL_UNLOCK(pool, hash);
goto not_found;
}
if (unlikely((n->id!=p_id) || (n->size!=n_size))){
/* fragment still free, but changed, either
* moved to another pool or has a diff. size */
SFM_POOL_UNLOCK(pool, hash);
goto try_again;
}
pf=&(pool->pool_hash[hash].first);
/* find it */
for(;(*pf)&&(*pf!=n); pf=&((*pf)->u.nxt_free));/*FIXME slow */
if (*pf==0){
SFM_POOL_UNLOCK(pool, hash);
/* not found, bad! */
LOG(L_WARN, "WARNING: sfm_realloc: could not find %p in "
"free " "list (hash=%d)\n", n, hash);
/* somebody is in the process of changing it ? */
goto not_found;
}
/* detach */
*pf=n->u.nxt_free;
n->u.nxt_free=0; /* mark it immediately as detached */
pool->pool_hash[hash].no--;
SFM_POOL_UNLOCK(pool, hash);
/* join */
f->size+=n->size+FRAG_OVERHEAD;
/* split it if necessary */
if (f->size > size){
#ifdef DBG_F_MALLOC
sfm_split_frag(qm, f, size, file, "fragm. from "
"sfm_realloc", line);
#else
sfm_split_frag(qm, f, size);
#endif
}
}
}else{
not_found:
/* could not join => realloc */
#else/* SFM_REALLOC_REMALLOC */
{
#endif /* SFM_REALLOC_REMALLOC */
#ifdef DBG_F_MALLOC
ptr=sfm_malloc(qm, size, file, func, line);
#else
ptr=sfm_malloc(qm, size);
#endif
if (ptr){
/* copy, need by libssl */
memcpy(ptr, p, orig_size);
#ifdef DBG_F_MALLOC
sfm_free(qm, p, file, func, line);
#else
sfm_free(qm, p);
#endif
}
p=ptr;
}
}else{
void* sfm_malloc(struct sfm_block* qm, unsigned long size)
#endif
{
struct sfm_frag* frag;
int hash;
unsigned int i;
#ifdef DBG_F_MALLOC
MDBG("sfm_malloc(%p, %lu) called from %s: %s(%d)\n", qm, size, file, func,
line);
#endif
/*size must be a multiple of 8*/
size=ROUNDUP(size);
/* if (size>(qm->size-qm->real_used)) return 0; */
/* check if our pool id is set */
sfm_fix_pool_id(qm);
/*search for a suitable free frag*/
if (likely(size<=SF_POOL_MAX_SIZE)){
hash=GET_SMALL_HASH(size);
/* try first in our pool */
#ifdef DBG_F_MALLOC
if (likely((frag=pool_get_frag(qm, &qm->pool[pool_id], hash, size,
file, func, line))!=0))
goto found;
/* try in the "main" free hash, go through all the hash */
if (likely((frag=main_get_frag(qm, hash, size, file, func, line))!=0))
goto found;
/* really low mem , try in other pools */
for (i=(pool_id+1); i< (pool_id+SFM_POOLS_NO); i++){
if ((frag=pool_get_frag(qm, &qm->pool[i%SFM_POOLS_NO], hash, size,
file, func, line))!=0)
goto found;
}
#else
if (likely((frag=pool_get_frag(qm, &qm->pool[pool_id], hash, size))
!=0 ))
goto found;
/* try in the "main" free hash, go through all the hash */
if (likely((frag=main_get_frag(qm, hash, size))!=0))
goto found;
/* really low mem , try in other pools */
for (i=(pool_id+1); i< (pool_id+SFM_POOLS_NO); i++){
if ((frag=pool_get_frag(qm, &qm->pool[i%SFM_POOLS_NO], hash, size))
!=0 )
goto found;
}
#endif
/* not found, bad! */
return 0;
}else{
hash=GET_BIG_HASH(size);
#ifdef DBG_F_MALLOC
if ((frag=main_get_frag(qm, hash, size, file, func, line))==0)
return 0; /* not found, bad! */
#else
if ((frag=main_get_frag(qm, hash, size))==0)
return 0; /* not found, bad! */
#endif
}
found:
/* we found it!*/
#ifdef DBG_F_MALLOC
frag->file=file;
frag->func=func;
frag->line=line;
frag->check=ST_CHECK_PATTERN;
MDBG("sfm_malloc(%p, %lu) returns address %p \n", qm, size,
(char*)frag+sizeof(struct sfm_frag));
#endif
FRAG_MARK_USED(frag); /* mark it as used */
return (char*)frag+sizeof(struct sfm_frag);
}