static inline void sfm_pool_insert (struct sfm_pool* pool, int hash, struct sfm_frag* frag) { unsigned long hash_bit; SFM_POOL_LOCK(pool, hash); frag_push(&pool->pool_hash[hash].first, frag); pool->pool_hash[hash].no++; /* set it only if not already set (avoids an expensive * cache trashing atomic write op) */ hash_bit=HASH_TO_BITMAP(hash); if (!(atomic_get_long((long*)&pool->bitmap) & hash_bit)) atomic_or_long((long*)&pool->bitmap, hash_bit); SFM_POOL_UNLOCK(pool, hash); }
void* sfm_realloc(struct sfm_block* qm, void* p, unsigned long size) #endif { struct sfm_frag *f; unsigned long orig_size; void *ptr; #ifndef SFM_REALLOC_REMALLOC struct sfm_frag *n; struct sfm_frag **pf; unsigned long diff; unsigned long p_id; int hash; unsigned long n_size; struct sfm_pool * pool; #endif #ifdef DBG_F_MALLOC MDBG("sfm_realloc(%p, %p, %lu) called from %s: %s(%d)\n", qm, p, size, file, func, line); if ((p)&&(p>(void*)qm->last_frag || p<(void*)qm->first_frag)){ LOG(L_CRIT, "BUG: sfm_free: bad pointer %p (out of memory block!) - " "aborting\n", p); abort(); } #endif if (size==0) { if (p) #ifdef DBG_F_MALLOC sfm_free(qm, p, file, func, line); #else sfm_free(qm, p); #endif return 0; } if (p==0) #ifdef DBG_F_MALLOC return sfm_malloc(qm, size, file, func, line); #else return sfm_malloc(qm, size); #endif f=(struct sfm_frag*) ((char*)p-sizeof(struct sfm_frag)); #ifdef DBG_F_MALLOC MDBG("sfm_realloc: realloc'ing frag %p alloc'ed from %s: %s(%ld)\n", f, f->file, f->func, f->line); #endif size=ROUNDUP(size); orig_size=f->size; if (f->size > size){ /* shrink */ #ifdef DBG_F_MALLOC MDBG("sfm_realloc: shrinking from %lu to %lu\n", f->size, size); sfm_split_frag(qm, f, size, file, "frag. from sfm_realloc", line); #else sfm_split_frag(qm, f, size); #endif }else if (f->size<size){ /* grow */ #ifdef DBG_F_MALLOC MDBG("sfm_realloc: growing from %lu to %lu\n", f->size, size); #endif #ifndef SFM_REALLOC_REMALLOC /* should set a magic value in list head and in push/pop if magic value => * lock and wait */ #error LL_MALLOC realloc not finished yet diff=size-f->size; n=FRAG_NEXT(f); if (((char*)n < (char*)qm->last_frag) && (n->u.nxt_free)&&((n->size+FRAG_OVERHEAD)>=diff)){ /* join */ /* detach n from the free list */ try_again: p_id=n->id; n_size=n->size; if ((unlikely(p_id >=SFM_POOLS_NO))){ hash=GET_HASH(n_size); SFM_MAIN_HASH_LOCK(qm, hash); if (unlikely((n->u.nxt_free==0) || ((n->size+FRAG_OVERHEAD)<diff))){ SFM_MAIN_HASH_UNLOCK(qm, hash); goto not_found; } if (unlikely((n->id!=p_id) || (n->size!=n_size))){ /* fragment still free, but changed, either * moved to another pool or has a diff. size */ SFM_MAIN_HASH_UNLOCK(qm, hash); goto try_again; } pf=&(qm->free_hash[hash].first); /* find it */ for(;(*pf)&&(*pf!=n); pf=&((*pf)->u.nxt_free));/*FIXME slow */ if (*pf==0){ SFM_MAIN_HASH_UNLOCK(qm, hash); /* not found, bad! */ LOG(L_WARN, "WARNING: sfm_realloc: could not find %p in " "free " "list (hash=%d)\n", n, hash); /* somebody is in the process of changing it ? */ goto not_found; } /* detach */ *pf=n->u.nxt_free; n->u.nxt_free=0; /* mark it immediately as detached */ qm->free_hash[hash].no--; SFM_MAIN_HASH_UNLOCK(qm, hash); /* join */ f->size+=n->size+FRAG_OVERHEAD; /* split it if necessary */ if (f->size > size){ #ifdef DBG_F_MALLOC sfm_split_frag(qm, f, size, file, "fragm. from " "sfm_realloc", line); #else sfm_split_frag(qm, f, size); #endif } }else{ /* p_id < SFM_POOLS_NO (=> in a pool )*/ hash=GET_SMALL_HASH(n_size); pool=&qm->pool[p_id]; SFM_POOL_LOCK(pool, hash); if (unlikely((n->u.nxt_free==0) || ((n->size+FRAG_OVERHEAD)<diff))){ SFM_POOL_UNLOCK(pool, hash); goto not_found; } if (unlikely((n->id!=p_id) || (n->size!=n_size))){ /* fragment still free, but changed, either * moved to another pool or has a diff. size */ SFM_POOL_UNLOCK(pool, hash); goto try_again; } pf=&(pool->pool_hash[hash].first); /* find it */ for(;(*pf)&&(*pf!=n); pf=&((*pf)->u.nxt_free));/*FIXME slow */ if (*pf==0){ SFM_POOL_UNLOCK(pool, hash); /* not found, bad! */ LOG(L_WARN, "WARNING: sfm_realloc: could not find %p in " "free " "list (hash=%d)\n", n, hash); /* somebody is in the process of changing it ? */ goto not_found; } /* detach */ *pf=n->u.nxt_free; n->u.nxt_free=0; /* mark it immediately as detached */ pool->pool_hash[hash].no--; SFM_POOL_UNLOCK(pool, hash); /* join */ f->size+=n->size+FRAG_OVERHEAD; /* split it if necessary */ if (f->size > size){ #ifdef DBG_F_MALLOC sfm_split_frag(qm, f, size, file, "fragm. from " "sfm_realloc", line); #else sfm_split_frag(qm, f, size); #endif } } }else{ not_found: /* could not join => realloc */ #else/* SFM_REALLOC_REMALLOC */ { #endif /* SFM_REALLOC_REMALLOC */ #ifdef DBG_F_MALLOC ptr=sfm_malloc(qm, size, file, func, line); #else ptr=sfm_malloc(qm, size); #endif if (ptr){ /* copy, need by libssl */ memcpy(ptr, p, orig_size); #ifdef DBG_F_MALLOC sfm_free(qm, p, file, func, line); #else sfm_free(qm, p); #endif } p=ptr; } }else{
static inline struct sfm_frag* pool_get_frag(struct sfm_block* qm, struct sfm_pool* pool, int hash, unsigned long size) #endif { int r; int next_block; struct sfm_frag* volatile* f; struct sfm_frag* frag; unsigned long b; unsigned long eob; /* special case for r=hash */ r=hash; f=&pool->pool_hash[r].first; if (*f==0) goto not_found; SFM_POOL_LOCK(pool, r); if (unlikely(*f==0)){ SFM_POOL_UNLOCK(pool, r); goto not_found; } found: /* detach it from the free list*/ frag=frag_pop((struct sfm_frag**)f); frag->u.nxt_free=0; /* mark it as 'taken' */ frag->id=pool_id; pool->pool_hash[r].no--; SFM_POOL_UNLOCK(pool, r); #ifdef DBG_F_MALLOC sfm_split_frag(qm, frag, size, file, func, line); #else sfm_split_frag(qm, frag, size); #endif if (&qm->pool[pool_id]==pool) atomic_inc_long((long*)&pool->hits); return frag; not_found: atomic_inc_long((long*)&pool->pool_hash[r].misses); r++; b=HASH_BIT_POS(r); while(r<SF_HASH_POOL_SIZE){ b=_next_set_bit(b, &pool->bitmap); next_block=_hash_range(b, &eob); r=(r<next_block)?next_block:r; for (; r<eob; r++){ f=&pool->pool_hash[r].first; if (*f){ SFM_POOL_LOCK(pool, r); if (unlikely(*f==0)){ /* not found */ SFM_POOL_UNLOCK(pool, r); }else goto found; } atomic_inc_long((long*)&pool->pool_hash[r].misses); } b++; } #if 0 /* EXPENSIVE BUG CHECK */ for (r=hash; r<SF_HASH_POOL_SIZE; r++){ f=&pool->pool_hash[r].first; if (*f){ SFM_POOL_LOCK(pool, r); if (unlikely(*f==0)){ /* not found */ SFM_POOL_UNLOCK(pool, r); }else{ b=_next_set_bit(HASH_BIT_POS(r), &pool->bitmap); next_block=_hash_range(b, &eob); BUG("pool_get_frag: found fragm. %d at %d (bit %ld range %ld-%ld), next set bit=%ld" " bitmap %ld (%p)\n", hash, r, HASH_BIT_POS(r), next_block, eob, b, pool->bitmap, &pool->bitmap); goto found; } } } #endif atomic_inc_long((long*)&pool->missed); return 0; }