void vqm_status(struct vqm_block* qm) { struct vqm_frag* f; unsigned int i,on_list; LOG(memlog, "vqm_status (%p):\n", qm); if (!qm) return; LOG(memlog, " heap size= %d, available: %d\n", qm->core_end-qm->init_core, qm->free_core ); LOG(memlog, "dumping unfreed fragments:\n"); for (f=(struct vqm_frag*)qm->init_core, i=0;(char*)f<(char*)qm->core; f=FRAG_NEXT(f) ,i++) if ( FRAG_ISUSED(f) ) dump_frag(f, i); LOG(memlog, "dumping unfreed big fragments:\n"); for (f=(struct vqm_frag*)qm->big_chunks,i=0;(char*)f<(char*)qm->core_end; f=FRAG_NEXT(f) ,i++) if ( FRAG_ISUSED(f) ) dump_frag( f, i ); #ifdef DBG_QM_MALLOC DBG("dumping bucket statistics:\n"); for (i=0; i<=BIG_BUCKET(qm); i++) { for(on_list=0, f=qm->next_free[i]; f; f=f->u.nxt_free ) on_list++; LOG(L_DBG, " %3d. bucket: in use: %ld, on free list: %d\n", i, qm->usage[i], on_list ); } #endif LOG(memlog, "-----------------------------\n"); }
void hp_pkg_free(struct hp_block *hpb, void *p) { struct hp_frag *f, *next; if (!p) { LM_WARN("free(0) called\n"); return; } f = FRAG_OF(p); /* * for private memory, coalesce as many consecutive fragments as possible * The same operation is not performed for shared memory, because: * - performance penalties introduced by additional locking logic * - the allocator itself actually favours fragmentation and reusage */ for (;;) { next = FRAG_NEXT(f); if (next >= hpb->last_frag || !next->prev) break; hp_frag_detach(hpb, next); update_stats_pkg_frag_detach(hpb, next); f->size += next->size + FRAG_OVERHEAD; update_stats_pkg_frag_merge(hpb); } hp_frag_attach(hpb, f); update_stats_pkg_frag_attach(hpb, f); }
void set_indexes(int core_index) { struct fm_frag* f; for (f=shm_block->first_frag; (char*)f<(char*)shm_block->last_frag; f=FRAG_NEXT(f)) if (!f->is_free) f->statistic_index = core_index; }
void __shm_frag_split_unsafe(struct hp_block *hpb, struct hp_frag *frag, unsigned long size) { unsigned long rest; struct hp_frag *n; #ifdef HP_MALLOC_FAST_STATS hpb->free_hash[PEEK_HASH_RR(hpb, frag->size)].total_no--; hpb->free_hash[PEEK_HASH_RR(hpb, size)].total_no++; #endif rest = frag->size - size; frag->size = size; /* split the fragment */ n = FRAG_NEXT(frag); n->size = rest - FRAG_OVERHEAD; #ifdef HP_MALLOC_FAST_STATS hpb->free_hash[PEEK_HASH_RR(hpb, n->size)].total_no++; #endif hp_frag_attach(hpb, n); if (stats_are_ready()) update_stats_shm_frag_attach(n); else { hpb->used -= n->size; hpb->real_used -= n->size; } }
/* size should already be rounded-up */ void __shm_frag_split(struct hp_block *hpb, struct hp_frag *frag, unsigned long size, unsigned int old_hash) { unsigned long rest, hash; struct hp_frag *n; #ifdef HP_MALLOC_FAST_STATS hpb->free_hash[PEEK_HASH_RR(hpb, frag->size)].total_no--; hpb->free_hash[PEEK_HASH_RR(hpb, size)].total_no++; #endif rest = frag->size - size; frag->size = size; /* split the fragment */ n = FRAG_NEXT(frag); n->size = rest - FRAG_OVERHEAD; /* insert the newly obtained hp_frag in its free list */ hash = PEEK_HASH_RR(hpb, n->size); if (hash != old_hash) SHM_LOCK(hash); hp_frag_attach(hpb, n); #ifdef HP_MALLOC_FAST_STATS hpb->free_hash[hash].total_no++; #endif if (hash != old_hash) SHM_UNLOCK(hash); }
void qm_sums(struct vqm_block* qm) { struct vqm_frag* f; int i; mem_counter *root=0,*x; lock_get(process_lock); if (process_no!=0) LOG(memlog, "vqm_sums (%p): PKG[%s]\n", qm,pt[process_no].desc); else LOG(memlog, "vqm_sums (%p): PKG[0]/SHM \n",qm); if (!qm) return; LOG(memlog, "summarizing all alloc'ed. fragments:\n"); for (f=qm->first_frag, i=0;(char*)f<(char*)qm->last_frag_end;f=FRAG_NEXT(f),i++){ if (! f->u.is_free){ x = get_mem_counter(&root,f); x->count++; x->size+=f->size; } } x = root; while(x){ LOG(memlog, " count=%6d size=%10lu bytes from %s: %s(%ld)\n", x->count,x->size, x->file, x->func, x->line ); root = x->next; free(x); x = root; } LOG(memlog, "-----------------------------\n"); lock_release(process_lock); }
void hp_stats_core_init(struct hp_block *hp, int core_index) { struct hp_frag *f; for (f=hp->first_frag; (char*)f<(char*)hp->last_frag; f=FRAG_NEXT(f)) if (!frag_is_free(f)) f->statistic_index = core_index; }
void fm_free(struct fm_block* qm, void* p) #endif { struct fm_frag* f,*n; #ifdef DBG_F_MALLOC LM_DBG("params(%p, %p), called from %s: %s(%d)\n", qm, p, file, func, line); if (p>(void*)qm->last_frag || p<(void*)qm->first_frag){ LM_CRIT("bad pointer %p (out of memory block!) - aborting\n", p); abort(); } #endif if (p==0) { LM_DBG("free(0) called\n"); return; } f=(struct fm_frag*) ((char*)p-sizeof(struct fm_frag)); #ifdef DBG_F_MALLOC LM_DBG("freeing block alloc'ed from %s: %s(%ld)\n", f->file, f->func, f->line); f->file=file; f->func=func; f->line=line; #endif join: if( qm->large_limit < qm->large_space ) goto no_join; n = FRAG_NEXT(f); if (((char*)n < (char*)qm->last_frag) && n->prev ) { fm_remove_free(qm, n); /* join */ f->size += n->size + FRAG_OVERHEAD; #if defined(DBG_F_MALLOC) || defined(STATISTICS) //qm->real_used -= FRAG_OVERHEAD; qm->used += FRAG_OVERHEAD; #endif goto join; } no_join: fm_insert_free(qm, f); #if defined(DBG_F_MALLOC) || defined(STATISTICS) qm->fragments -= 1; #endif pkg_threshold_check(); }
void __pkg_frag_split(struct hp_block *hpb, struct hp_frag *frag, unsigned long size) { unsigned long rest; struct hp_frag *n; rest = frag->size - size; frag->size = size; /* split the fragment */ n = FRAG_NEXT(frag); n->size = rest - FRAG_OVERHEAD; hp_frag_attach(hpb, n); update_stats_pkg_frag_attach(hpb, n); }
void* fm_realloc(struct fm_block* qm, void* p, unsigned long size) #endif { struct fm_frag *f; unsigned long diff; unsigned long orig_size; struct fm_frag *n; void *ptr; #ifdef DBG_F_MALLOC LM_DBG("params(%p, %p, %lu), called from %s: %s(%d)\n", qm, p, size, file, func, line); if ((p)&&(p>(void*)qm->last_frag || p<(void*)qm->first_frag)){ LM_CRIT("bad pointer %p (out of memory block!) - aborting\n", p); abort(); } #endif if (size==0) { if (p) #ifdef DBG_F_MALLOC fm_free(qm, p, file, func, line); #else fm_free(qm, p); #endif pkg_threshold_check(); return 0; } if (p==0) #ifdef DBG_F_MALLOC return fm_malloc(qm, size, file, func, line); #else return fm_malloc(qm, size); #endif f=(struct fm_frag*) ((char*)p-sizeof(struct fm_frag)); #ifdef DBG_F_MALLOC LM_DBG("realloc'ing frag %p alloc'ed from %s: %s(%ld)\n", f, f->file, f->func, f->line); #endif size=ROUNDUP(size); orig_size=f->size; if (f->size > size){ /* shrink */ #ifdef DBG_F_MALLOC LM_DBG("shrinking from %lu to %lu\n", f->size, size); fm_split_frag(qm, f, size, file, "frag. from fm_realloc", line); #else fm_split_frag(qm, f, size); #endif }else if (f->size<size){ /* grow */ #ifdef DBG_F_MALLOC LM_DBG("growing from %lu to %lu\n", f->size, size); #endif diff=size-f->size; n=FRAG_NEXT(f); if (((char*)n < (char*)qm->last_frag) && n->prev && ((n->size+FRAG_OVERHEAD)>=diff)){ fm_remove_free(qm,n); /* join */ f->size += n->size + FRAG_OVERHEAD; #if defined(DBG_F_MALLOC) || defined(STATISTICS) //qm->real_used -= FRAG_OVERHEAD; qm->used += FRAG_OVERHEAD; #endif /* split it if necessary */ if (f->size > size){ #ifdef DBG_F_MALLOC fm_split_frag(qm, f, size, file, "fragm. from fm_realloc", line); #else fm_split_frag(qm, f, size); #endif } }else{ /* could not join => realloc */ #ifdef DBG_F_MALLOC ptr=fm_malloc(qm, size, file, func, line); #else ptr = fm_malloc(qm, size); #endif if (ptr) { /* copy, need by libssl */ memcpy(ptr, p, orig_size); #ifdef DBG_F_MALLOC fm_free(qm, p, file, func, line); #else fm_free(qm, p); #endif } p = ptr; } }else{ /* do nothing */ #ifdef DBG_F_MALLOC LM_DBG("doing nothing, same size: %lu - %lu\n", f->size, size); #endif } #ifdef DBG_F_MALLOC LM_DBG("returning %p\n", p); #endif #if defined(DBG_F_MALLOC) || defined(STATISTICS) if (qm->max_real_used<qm->real_used) qm->max_real_used=qm->real_used; #endif pkg_threshold_check(); return p; }
void* fm_malloc(struct fm_block* qm, unsigned long size) #endif { struct fm_frag* frag,*n; unsigned int hash; #ifdef DBG_F_MALLOC LM_DBG("params (%p, %lu), called from %s: %s(%d)\n", qm, size, file, func, line); #endif /*size must be a multiple of 8*/ size=ROUNDUP(size); /*search for a suitable free frag*/ for(hash=GET_HASH(size);hash<F_HASH_SIZE;hash++){ frag=qm->free_hash[hash].first; for( ; frag; frag = frag->u.nxt_free ) if ( frag->size >= size ) goto found; /* try in a bigger bucket */ } /* not found, bad! */ LM_WARN("Not enough free memory, will atempt defragmenation\n"); for( frag = qm->first_frag; (char*)frag < (char*)qm->last_frag; ) { n = FRAG_NEXT(frag); if ( ((char*)n < (char*)qm->last_frag) && n->prev && frag->prev ) { /* detach frag*/ fm_remove_free(qm, frag); do { fm_remove_free(qm, n); frag->size += n->size + FRAG_OVERHEAD; #if defined(DBG_F_MALLOC) || defined(STATISTICS) //qm->real_used -= FRAG_OVERHEAD; qm->used += FRAG_OVERHEAD; #endif if( frag->size >size ) goto solved; n = FRAG_NEXT(frag); } while ( ((char*)n < (char*)qm->last_frag) && n->prev); fm_insert_free(qm,frag); } frag = n; } pkg_threshold_check(); return 0; found: /* we found it!*/ fm_remove_free(qm,frag); /*see if we'll use full frag, or we'll split it in 2*/ #ifdef DBG_F_MALLOC fm_split_frag(qm, frag, size, file, func, line); frag->file=file; frag->func=func; frag->line=line; frag->check=ST_CHECK_PATTERN; LM_DBG("params(%p, %lu), returns address %p \n", qm, size, (char*)frag+sizeof(struct fm_frag)); #else fm_split_frag(qm, frag, size); #endif solved: #if defined(DBG_F_MALLOC) || defined(STATISTICS) if (qm->max_real_used<qm->real_used) qm->max_real_used=qm->real_used; #endif pkg_threshold_check(); return (char*)frag+sizeof(struct fm_frag); }
void fm_split_frag(struct fm_block* qm, struct fm_frag* frag, unsigned long size) #endif { unsigned long rest; struct fm_frag* n; rest=frag->size-size; #ifdef MEM_FRAG_AVOIDANCE if ((rest> (FRAG_OVERHEAD+F_MALLOC_OPTIMIZE))|| (rest>=(FRAG_OVERHEAD+size))){ /* the residue fragm. is big enough*/ #else if (rest>(FRAG_OVERHEAD+MIN_FRAG_SIZE)){ #endif frag->size=size; /*split the fragment*/ n=FRAG_NEXT(frag); n->size=rest-FRAG_OVERHEAD; /* * The real used memory does not increase, as the frag memory is not * freed from real_used. On the other hand, the used size should * decrease, because the new fragment is not "useful data" - razvanc #if defined(DBG_F_MALLOC) || defined(STATISTICS) qm->real_used+=FRAG_OVERHEAD; #endif */ #if defined(DBG_F_MALLOC) || defined(STATISTICS) qm->used-=FRAG_OVERHEAD; #endif #ifdef DBG_F_MALLOC /* frag created by malloc, mark it*/ n->file=file; n->func="frag. from fm_malloc"; n->line=line; n->check=ST_CHECK_PATTERN; #endif /* reinsert n in free list*/ fm_insert_free(qm, n); }else{ /* we cannot split this fragment any more => alloc all of it*/ } } /* init malloc and return a fm_block*/ struct fm_block* fm_malloc_init(char* address, unsigned long size) { char* start; char* end; struct fm_block* qm; unsigned long init_overhead; /* make address and size multiple of 8*/ start=(char*)ROUNDUP((unsigned long) address); LM_DBG("F_OPTIMIZE=%lu, /ROUNDTO=%lu\n", F_MALLOC_OPTIMIZE, F_MALLOC_OPTIMIZE/ROUNDTO); LM_DBG("F_HASH_SIZE=%lu, fm_block size=%lu\n", F_HASH_SIZE, (long)sizeof(struct fm_block)); LM_DBG("params (%p, %lu), start=%p\n", address, size, start); if (size<(unsigned long)(start-address)) return 0; size-=(start-address); if (size <(MIN_FRAG_SIZE+FRAG_OVERHEAD)) return 0; size=ROUNDDOWN(size); init_overhead=(ROUNDUP(sizeof(struct fm_block))+ 2 * FRAG_OVERHEAD); if (size < init_overhead) { /* not enough mem to create our control structures !!!*/ return 0; } end=start+size; qm=(struct fm_block*)start; memset(qm, 0, sizeof(struct fm_block)); qm->size=size; #if defined(DBG_F_MALLOC) || defined(STATISTICS) qm->used=size-init_overhead; qm->real_used=size; qm->max_real_used=init_overhead; #endif qm->first_frag=(struct fm_frag*)(start+ROUNDUP(sizeof(struct fm_block))); qm->last_frag=(struct fm_frag*)(end-sizeof(struct fm_frag)); /* init initial fragment*/ qm->first_frag->size=size-init_overhead; qm->last_frag->size=0; qm->last_frag->prev=NULL; qm->first_frag->prev=NULL; #ifdef DBG_F_MALLOC qm->first_frag->check=ST_CHECK_PATTERN; qm->last_frag->check=END_CHECK_PATTERN1; #endif /* link initial fragment into the free list*/ qm->large_space = 0; qm->large_limit = qm->size / 100 * F_MALLOC_DEFRAG_PERCENT; if( qm->large_limit < F_MALLOC_DEFRAG_LIMIT ) qm->large_limit = F_MALLOC_DEFRAG_LIMIT; fm_insert_free(qm, qm->first_frag); return qm; }
void *hp_pkg_realloc(struct hp_block *hpb, void *p, unsigned long size) { struct hp_frag *f; unsigned long diff; unsigned long orig_size; struct hp_frag *next; void *ptr; if (size == 0) { if (p) hp_pkg_free(hpb, p); return NULL; } if (!p) return hp_pkg_malloc(hpb, size); f = FRAG_OF(p); size = ROUNDUP(size); orig_size = f->size; /* shrink operation */ if (orig_size > size) { pkg_frag_split(hpb, f, size); /* grow operation */ } else if (orig_size < size) { diff = size - orig_size; next = FRAG_NEXT(f); /* try to join with a large enough adjacent free fragment */ if (next < hpb->last_frag && next->prev && (next->size + FRAG_OVERHEAD) >= diff) { hp_frag_detach(hpb, next); update_stats_pkg_frag_detach(hpb, next); f->size += next->size + FRAG_OVERHEAD; /* split the result if necessary */ if (f->size > size) pkg_frag_split(hpb, f, size); } else { /* could not join => realloc */ ptr = hp_pkg_malloc(hpb, size); if (ptr) { /* copy, need by libssl */ memcpy(ptr, p, orig_size); hp_pkg_free(hpb, p); } p = ptr; } if (hpb->real_used > hpb->max_real_used) hpb->max_real_used = hpb->real_used; } pkg_threshold_check(); return p; }
void fm_status(struct fm_block* qm) { struct fm_frag* f; unsigned int i,j; unsigned int h; int unused; unsigned long size; #ifdef DBG_MALLOC mem_dbg_htable_t allocd; struct mem_dbg_entry *it; #endif LM_GEN1(memdump, "fm_status (%p):\n", qm); if (!qm) return; LM_GEN1(memdump, " heap size= %ld\n", qm->size); #if defined(DBG_MALLOC) || defined(STATISTICS) LM_GEN1(memdump, " used= %lu, used+overhead=%lu, free=%lu\n", qm->used, qm->real_used, qm->size-qm->used); LM_GEN1(memdump, " max used (+overhead)= %lu\n", qm->max_real_used); #endif #if defined(DBG_MALLOC) dbg_ht_init(allocd); for (f=qm->first_frag; (char*)f<(char*)qm->last_frag; f=FRAG_NEXT(f)) if (!f->is_free) if (dbg_ht_update(allocd, f->file, f->func, f->line, f->size) < 0) { LM_ERR("Unable to update alloc'ed. memory summary\n"); dbg_ht_free(allocd); return; } LM_GEN1(memdump, " dumping summary of all alloc'ed. fragments:\n"); for(i=0; i < DBG_HASH_SIZE; i++) { it = allocd[i]; while (it) { LM_GEN1(memdump, " %10lu : %lu x [%s: %s, line %lu]\n", it->size, it->no_fragments, it->file, it->func, it->line); it = it->next; } } dbg_ht_free(allocd); #endif LM_GEN1(memdump, "dumping free list:\n"); for(h=0,i=0,size=0; h<F_HASH_SIZE; h++) { unused=0; for (f=qm->free_hash[h].first,j=0; f; size+=f->size,f=f->u.nxt_free,i++,j++) { } if (j) LM_GEN1(memdump,"hash = %3d fragments no.: %5d, unused: %5d\n\t\t" " bucket size: %9lu - %9lu (first %9lu)\n", h, j, unused, UN_HASH(h), ((h<=F_MALLOC_OPTIMIZE/ROUNDTO)?1:2)* UN_HASH(h), qm->free_hash[h].first->size ); if (j!=qm->free_hash[h].no) { LM_CRIT("different free frag. count: %d!=%ld" " for hash %3d\n", j, qm->free_hash[h].no, h); } } LM_GEN1(memdump, "TOTAL: %6d free fragments = %6lu free bytes\n", i, size); LM_GEN1(memdump, "TOTAL: %ld large bytes\n", qm->large_space ); LM_GEN1(memdump, "TOTAL: %u overhead\n", (unsigned int)FRAG_OVERHEAD ); LM_GEN1(memdump, "-----------------------------\n"); }
void* fm_malloc(struct fm_block* qm, unsigned long size) #endif { struct fm_frag* frag,*n; unsigned int hash; #ifdef DBG_MALLOC LM_GEN1(memlog, "%s_malloc(%lu), called from %s: %s(%d)\n", qm->name, size, file, func, line); #endif /*size must be a multiple of 8*/ size=ROUNDUP(size); /*search for a suitable free frag*/ for(hash=GET_HASH(size); hash<F_HASH_SIZE; hash++) { frag=qm->free_hash[hash].first; for( ; frag; frag = frag->u.nxt_free ) if ( frag->size >= size ) goto found; /* try in a bigger bucket */ } /* not found, bad! */ #if defined(DBG_MALLOC) || defined(STATISTICS) LM_ERR(oom_errorf, qm->name, qm->size - qm->real_used, qm->name[0] == 'p' ? "M" : "m"); LM_INFO("attempting defragmentation... (need %lu bytes)\n", size); #else LM_ERR(oom_nostats_errorf, qm->name, qm->name[0] == 'p' ? "M" : "m"); LM_INFO("attempting defragmentation... (need %lu bytes)\n", size); #endif for( frag = qm->first_frag; (char*)frag < (char*)qm->last_frag; ) { n = FRAG_NEXT(frag); if ( ((char*)n < (char*)qm->last_frag) && n->prev && frag->prev ) { /* detach frag*/ fm_remove_free(qm, frag); do { fm_remove_free(qm, n); frag->size += n->size + FRAG_OVERHEAD; #if defined(DBG_MALLOC) || defined(STATISTICS) //qm->real_used -= FRAG_OVERHEAD; qm->used += FRAG_OVERHEAD; #endif if( frag->size >size ) { #ifdef DBG_MALLOC /* mark it as "busy" */ frag->is_free = 0; #endif goto solved; } n = FRAG_NEXT(frag); } while ( ((char*)n < (char*)qm->last_frag) && n->prev); fm_insert_free(qm,frag); } frag = n; } LM_INFO("unable to alloc a big enough fragment!\n"); pkg_threshold_check(); return 0; found: /* we found it!*/ fm_remove_free(qm,frag); #ifdef DBG_MALLOC /* mark it as "busy" */ frag->is_free = 0; #endif /*see if we'll use full frag, or we'll split it in 2*/ #ifdef DBG_MALLOC fm_split_frag(qm, frag, size, file, func, line); frag->file=file; frag->func=func; frag->line=line; frag->check=ST_CHECK_PATTERN; LM_GEN1(memlog, "%s_malloc(%lu), returns address %p\n", qm->name, size, (char*)frag+sizeof(struct fm_frag)); #else fm_split_frag(qm, frag, size); #endif solved: #if defined(DBG_MALLOC) || defined(STATISTICS) if (qm->max_real_used<qm->real_used) qm->max_real_used=qm->real_used; qm->fragments += 1; #endif pkg_threshold_check(); return (char*)frag+sizeof(struct fm_frag); }
int split_frag(struct qm_block* qm, struct qm_frag* f, unsigned long new_size) #endif { unsigned long rest; struct qm_frag* n; struct qm_frag_end* end; rest=f->size-new_size; #ifdef MEM_FRAG_AVOIDANCE if ((rest> (FRAG_OVERHEAD+QM_MALLOC_OPTIMIZE))|| (rest>=(FRAG_OVERHEAD+new_size))){/* the residue fragm. is big enough*/ #else if (rest>(FRAG_OVERHEAD+MIN_FRAG_SIZE)){ #endif f->size=new_size; /*split the fragment*/ end=FRAG_END(f); end->size=new_size; n=(struct qm_frag*)((char*)end+sizeof(struct qm_frag_end)); n->size=rest-FRAG_OVERHEAD; FRAG_END(n)->size=n->size; FRAG_CLEAR_USED(n); /* never used */ qm->real_used+=FRAG_OVERHEAD; #ifdef DBG_QM_MALLOC end->check1=END_CHECK_PATTERN1; end->check2=END_CHECK_PATTERN2; /* frag created by malloc, mark it*/ n->file=file; n->func=func; n->line=line; n->check=ST_CHECK_PATTERN; #endif /* reinsert n in free list*/ qm_insert_free(qm, n); return 0; }else{ /* we cannot split this fragment any more */ return -1; } } #ifdef DBG_QM_MALLOC void* qm_malloc(struct qm_block* qm, unsigned long size, const char* file, const char* func, unsigned int line) #else void* qm_malloc(struct qm_block* qm, unsigned long size) #endif { struct qm_frag* f; int hash; #ifdef DBG_QM_MALLOC unsigned int list_cntr; list_cntr = 0; MDBG("qm_malloc(%p, %lu) called from %s: %s(%d)\n", qm, size, file, func, line); #endif /*size must be a multiple of 8*/ size=ROUNDUP(size); if (size>(qm->size-qm->real_used)) return 0; /*search for a suitable free frag*/ #ifdef DBG_QM_MALLOC if ((f=qm_find_free(qm, size, &hash, &list_cntr))!=0){ #else if ((f=qm_find_free(qm, size, &hash))!=0){ #endif /* we found it!*/ /*detach it from the free list*/ #ifdef DBG_QM_MALLOC qm_debug_frag(qm, f); #endif qm_detach_free(qm, f); /*mark it as "busy"*/ f->u.is_free=0; qm->free_hash[hash].no--; /* we ignore split return */ #ifdef DBG_QM_MALLOC split_frag(qm, f, size, file, "fragm. from qm_malloc", line); #else split_frag(qm, f, size); #endif qm->real_used+=f->size; qm->used+=f->size; if (qm->max_real_used<qm->real_used) qm->max_real_used=qm->real_used; #ifdef DBG_QM_MALLOC f->file=file; f->func=func; f->line=line; f->check=ST_CHECK_PATTERN; /* FRAG_END(f)->check1=END_CHECK_PATTERN1; FRAG_END(f)->check2=END_CHECK_PATTERN2;*/ MDBG("qm_malloc(%p, %lu) returns address %p frag. %p (size=%lu) on %d" " -th hit\n", qm, size, (char*)f+sizeof(struct qm_frag), f, f->size, list_cntr ); #endif return (char*)f+sizeof(struct qm_frag); } return 0; } #ifdef DBG_QM_MALLOC void qm_free(struct qm_block* qm, void* p, const char* file, const char* func, unsigned int line) #else void qm_free(struct qm_block* qm, void* p) #endif { struct qm_frag* f; struct qm_frag* prev; struct qm_frag* next; unsigned long size; #ifdef DBG_QM_MALLOC MDBG("qm_free(%p, %p), called from %s: %s(%d)\n", qm, p, file, func, line); if (p>(void*)qm->last_frag_end || p<(void*)qm->first_frag){ LOG(L_CRIT, "BUG: qm_free: bad pointer %p (out of memory block!) - " "aborting\n", p); abort(); } #endif if (p==0) { LOG(L_WARN, "WARNING:qm_free: free(0) called\n"); return; } prev=next=0; f=(struct qm_frag*) ((char*)p-sizeof(struct qm_frag)); #ifdef DBG_QM_MALLOC qm_debug_frag(qm, f); if (f->u.is_free){ LOG(L_CRIT, "BUG: qm_free: freeing already freed pointer," " first free: %s: %s(%ld) - aborting\n", f->file, f->func, f->line); abort(); } MDBG("qm_free: freeing frag. %p alloc'ed from %s: %s(%ld)\n", f, f->file, f->func, f->line); #endif size=f->size; qm->used-=size; qm->real_used-=size; #ifdef QM_JOIN_FREE /* join packets if possible*/ next=FRAG_NEXT(f); if (((char*)next < (char*)qm->last_frag_end) &&( next->u.is_free)){ /* join */ #ifdef DBG_QM_MALLOC qm_debug_frag(qm, next); #endif qm_detach_free(qm, next); size+=next->size+FRAG_OVERHEAD; qm->real_used-=FRAG_OVERHEAD; qm->free_hash[GET_HASH(next->size)].no--; /* FIXME slow */ } if (f > qm->first_frag){ prev=FRAG_PREV(f); /* (struct qm_frag*)((char*)f - (struct qm_frag_end*)((char*)f- sizeof(struct qm_frag_end))->size);*/ #ifdef DBG_QM_MALLOC qm_debug_frag(qm, prev); #endif if (prev->u.is_free){ /*join*/ qm_detach_free(qm, prev); size+=prev->size+FRAG_OVERHEAD; qm->real_used-=FRAG_OVERHEAD; qm->free_hash[GET_HASH(prev->size)].no--; /* FIXME slow */ f=prev; } } f->size=size; FRAG_END(f)->size=f->size; #endif /* QM_JOIN_FREE*/ #ifdef DBG_QM_MALLOC f->file=file; f->func=func; f->line=line; #endif qm_insert_free(qm, f); } #ifdef DBG_QM_MALLOC void* qm_realloc(struct qm_block* qm, void* p, unsigned long size, const char* file, const char* func, unsigned int line) #else void* qm_realloc(struct qm_block* qm, void* p, unsigned long size) #endif { struct qm_frag* f; unsigned long diff; unsigned long orig_size; struct qm_frag* n; void* ptr; #ifdef DBG_QM_MALLOC MDBG("qm_realloc(%p, %p, %lu) called from %s: %s(%d)\n", qm, p, size, file, func, line); if ((p)&&(p>(void*)qm->last_frag_end || p<(void*)qm->first_frag)){ LOG(L_CRIT, "BUG: qm_free: bad pointer %p (out of memory block!) - " "aborting\n", p); abort(); } #endif if (size==0) { if (p) #ifdef DBG_QM_MALLOC qm_free(qm, p, file, func, line); #else qm_free(qm, p); #endif return 0; } if (p==0) #ifdef DBG_QM_MALLOC return qm_malloc(qm, size, file, func, line); #else return qm_malloc(qm, size); #endif f=(struct qm_frag*) ((char*)p-sizeof(struct qm_frag)); #ifdef DBG_QM_MALLOC qm_debug_frag(qm, f); MDBG("qm_realloc: realloc'ing frag %p alloc'ed from %s: %s(%ld)\n", f, f->file, f->func, f->line); if (f->u.is_free){ LOG(L_CRIT, "BUG:qm_realloc: trying to realloc an already freed " "pointer %p , fragment %p -- aborting\n", p, f); abort(); } #endif /* find first acceptable size */ size=ROUNDUP(size); if (f->size > size){ orig_size=f->size; /* shrink */ #ifdef DBG_QM_MALLOC MDBG("qm_realloc: shrinking from %lu to %lu\n", f->size, size); if(split_frag(qm, f, size, file, "fragm. from qm_realloc", line)!=0){ MDBG("qm_realloc : shrinked successful\n"); #else if(split_frag(qm, f, size)!=0){ #endif /* update used sizes: freed the spitted frag */ qm->real_used-=(orig_size-f->size-FRAG_OVERHEAD); qm->used-=(orig_size-f->size); } }else if (f->size < size){ /* grow */ #ifdef DBG_QM_MALLOC MDBG("qm_realloc: growing from %lu to %lu\n", f->size, size); #endif orig_size=f->size; diff=size-f->size; n=FRAG_NEXT(f); if (((char*)n < (char*)qm->last_frag_end) && (n->u.is_free)&&((n->size+FRAG_OVERHEAD)>=diff)){ /* join */ qm_detach_free(qm, n); qm->free_hash[GET_HASH(n->size)].no--; /*FIXME: slow*/ f->size+=n->size+FRAG_OVERHEAD; qm->real_used-=FRAG_OVERHEAD; FRAG_END(f)->size=f->size; /* end checks should be ok */ /* split it if necessary */ if (f->size > size ){ #ifdef DBG_QM_MALLOC split_frag(qm, f, size, file, "fragm. from qm_realloc", line); #else split_frag(qm, f, size); #endif } qm->real_used+=(f->size-orig_size); qm->used+=(f->size-orig_size); }else{ /* could not join => realloc */ #ifdef DBG_QM_MALLOC ptr=qm_malloc(qm, size, file, func, line); #else ptr=qm_malloc(qm, size); #endif if (ptr){ /* copy, need by libssl */ memcpy(ptr, p, orig_size); #ifdef DBG_QM_MALLOC qm_free(qm, p, file, func, line); #else qm_free(qm, p); #endif } p=ptr; } }else{ /* do nothing */ #ifdef DBG_QM_MALLOC MDBG("qm_realloc: doing nothing, same size: %lu - %lu\n", f->size, size); #endif } #ifdef DBG_QM_MALLOC MDBG("qm_realloc: returning %p\n", p); #endif return p; } void qm_status(struct qm_block* qm) { struct qm_frag* f; int i,j; int h; int unused; LOG(memlog, "qm_status (%p):\n", qm); if (!qm) return; LOG(memlog, " heap size= %lu\n", qm->size); LOG(memlog, " used= %lu, used+overhead=%lu, free=%lu\n", qm->used, qm->real_used, qm->size-qm->real_used); LOG(memlog, " max used (+overhead)= %lu\n", qm->max_real_used); LOG(memlog, "dumping all alloc'ed. fragments:\n"); for (f=qm->first_frag, i=0;(char*)f<(char*)qm->last_frag_end;f=FRAG_NEXT(f) ,i++){ if (! f->u.is_free){ LOG(memlog, " %3d. %c address=%p frag=%p size=%lu used=%d\n", i, (f->u.is_free)?'a':'N', (char*)f+sizeof(struct qm_frag), f, f->size, FRAG_WAS_USED(f)); #ifdef DBG_QM_MALLOC LOG(memlog, " %s from %s: %s(%ld)\n", (f->u.is_free)?"freed":"alloc'd", f->file, f->func, f->line); LOG(memlog, " start check=%lx, end check= %lx, %lx\n", f->check, FRAG_END(f)->check1, FRAG_END(f)->check2); #endif } } LOG(memlog, "dumping free list stats :\n"); for(h=0,i=0;h<QM_HASH_SIZE;h++){ unused=0; for (f=qm->free_hash[h].head.u.nxt_free,j=0; f!=&(qm->free_hash[h].head); f=f->u.nxt_free, i++, j++){ if (!FRAG_WAS_USED(f)){ unused++; #ifdef DBG_QM_MALLOC LOG(memlog, "unused fragm.: hash = %3d, fragment %p," " address %p size %lu, created from %s: %s(%lu)\n", h, f, (char*)f+sizeof(struct qm_frag), f->size, f->file, f->func, f->line); #endif } } if (j) LOG(memlog, "hash= %3d. fragments no.: %5d, unused: %5d\n" "\t\t bucket size: %9lu - %9ld (first %9lu)\n", h, j, unused, UN_HASH(h), ((h<=QM_MALLOC_OPTIMIZE/ROUNDTO)?1:2)*UN_HASH(h), qm->free_hash[h].head.u.nxt_free->size ); if (j!=qm->free_hash[h].no){ LOG(L_CRIT, "BUG: qm_status: different free frag. count: %d!=%lu" " for hash %3d\n", j, qm->free_hash[h].no, h); } } LOG(memlog, "-----------------------------\n"); } /* fills a malloc info structure with info about the block * if a parameter is not supported, it will be filled with 0 */ void qm_info(struct qm_block* qm, struct mem_info* info) { int r; long total_frags; total_frags=0; memset(info,0, sizeof(*info)); info->total_size=qm->size; info->min_frag=MIN_FRAG_SIZE; info->free=qm->size-qm->real_used; info->used=qm->used; info->real_used=qm->real_used; info->max_used=qm->max_real_used; for(r=0;r<QM_HASH_SIZE; r++){ total_frags+=qm->free_hash[r].no; } info->total_frags=total_frags; } /* returns how much free memory is available * it never returns an error (unlike fm_available) */ unsigned long qm_available(struct qm_block* qm) { return qm->size-qm->real_used; } #ifdef DBG_QM_MALLOC typedef struct _mem_counter{ const char *file; const char *func; unsigned long line; unsigned long size; int count; struct _mem_counter *next; } mem_counter; mem_counter* get_mem_counter(mem_counter **root,struct qm_frag* f) { mem_counter *x; if (!*root) goto make_new; for(x=*root;x;x=x->next) if (x->file == f->file && x->func == f->func && x->line == f->line) return x; make_new: x = malloc(sizeof(mem_counter)); x->file = f->file; x->func = f->func; x->line = f->line; x->count = 0; x->size = 0; x->next = *root; *root = x; return x; } #include "../locking.h" #include "../pt.h" extern gen_lock_t* process_lock; extern struct process_table *pt; extern int process_no; void qm_sums(struct qm_block* qm) { struct qm_frag* f; int i; int total_count=0; long unsigned int total_size=0; int memlog=L_ERR; mem_counter *root=0,*x; //lock_get(process_lock); if (process_no!=0) LOG(memlog, "qm_sums (%p): PKG[%s]\n", qm,pt[process_no].desc); else LOG(memlog, "qm_sums (%p): PKG[0]/SHM \n",qm); if (!qm) { //lock_release(process_lock); return; } LOG(memlog, "summarizing all alloc'ed. fragments:\n"); for (f=qm->first_frag, i=0;(char*)f<(char*)qm->last_frag_end;f=FRAG_NEXT(f),i++){ if (! f->u.is_free){ x = get_mem_counter(&root,f); x->count++; x->size+=f->size; } } x = root; while(x){ LOG(memlog, " count=%6d size=%10lu bytes from %s: %s(%ld)\n", x->count,x->size, x->file, x->func, x->line ); total_count+=x->count;total_size+=x->size; root = x->next; free(x); x = root; } LOG(memlog, " count=%6d size=%10lu bytes in total\n",total_count,total_size); LOG(memlog, "-----------------------------\n"); //lock_release(process_lock); }
void vqm_free(struct vqm_block* qm, void* p) #endif { struct vqm_frag *f, *next, *prev, *first_big; unsigned char b; #ifdef DBG_QM_MALLOC DBG("vqm_free(%p, %p), called from %s: %s(%d)\n", qm, p, file, func, line); if (p>(void *)qm->core_end || p<(void*)qm->init_core){ LOG(L_CRIT, "BUG: vqm_free: bad pointer %p (out of memory block!) - " "aborting\n", p); abort(); } #endif if (p==0) { DBG("WARNING:vqm_free: free(0) called\n"); return; } f=(struct vqm_frag*) ((char*)p-sizeof(struct vqm_frag)); b=f->u.inuse.bucket; #ifdef DBG_QM_MALLOC VQM_DEBUG_FRAG(qm, f); if ( ! FRAG_ISUSED(f) ) { LOG(L_CRIT, "BUG: vqm_free: freeing already freed pointer," " first freed: %s: %s(%d) - aborting\n", f->file, f->func, f->line); abort(); } if ( b>MAX_BUCKET ) { LOG(L_CRIT, "BUG: vqm_free: fragment with too high bucket nr: " "%d, allocated: %s: %s(%d) - aborting\n", b, f->file, f->func, f->line); abort(); } DBG("vqm_free: freeing %d bucket block alloc'ed from %s: %s(%d)\n", f->u.inuse.bucket, f->file, f->func, f->line); f->file=file; f->func=func; f->line=line; qm->usage[ f->u.inuse.bucket ]--; #endif if (IS_BIGBUCKET(qm,b)) { next=FRAG_NEXT(f); if ((char *)next +sizeof( struct vqm_frag) < qm->core_end) { VQM_DEBUG_FRAG(qm, next); if (! FRAG_ISUSED(next)) { /* coalesce with next fragment */ DBG("vqm_free: coalesced with next\n"); vqm_detach_free(qm, next); f->size+=next->size; FRAG_END(f)->size=f->size; } } first_big = qm->next_free[b]; if (first_big && f>first_big) { prev=FRAG_PREV(f); VQM_DEBUG_FRAG(qm, prev); if (!FRAG_ISUSED(prev)) { /* coalesce with prev fragment */ DBG("vqm_free: coalesced with prev\n"); vqm_detach_free(qm, prev ); prev->size+=f->size; f=prev; FRAG_END(f)->size=f->size; } } if ((char *)f==qm->big_chunks) { /* release unused core */ DBG("vqm_free: big chunk released\n"); qm->free_core+=f->size; qm->big_chunks+=f->size; return; } first_big = qm->next_free[b]; /* fix reverse link (used only for BIG_BUCKET */ if (first_big) FRAG_END(first_big)->prv_free=f; FRAG_END(f)->prv_free=0; } else first_big = qm->next_free[b]; f->u.nxt_free = first_big; /* also clobbers magic */ qm->next_free[b] = f; }
/** * on-demand memory fragmentation, based on an input pattern file */ int hp_mem_warming(struct hp_block *hpb) { struct size_fraction { int hash_index; double amount; unsigned long fragments; struct size_fraction *next; }; struct size_fraction *sf, *it, *sorted_sf = NULL; FILE *f; size_t rc; unsigned long roundto, hash_size; long long bucket_mem; int i, c = 0; unsigned int current_frag_size; struct hp_frag *big_frag; unsigned int optimized_buckets; f = fopen(mem_warming_pattern_file, "r"); if (!f) { LM_ERR("failed to open pattern file %s: %d - %s\n", mem_warming_pattern_file, errno, strerror(errno)); return -1; } rc = fscanf(f, "%lu %lu\n", &roundto, &hash_size); if (rc != 2) { LM_ERR("failed to read from %s: bad file format\n", mem_warming_pattern_file); goto out; } rc = 0; if (roundto != ROUNDTO || hash_size != HP_HASH_SIZE) { LM_ERR("incompatible pattern file data: [HP_HASH_SIZE: %lu-%lu] " "[ROUNDTO: %lu-%lu]\n", hash_size, HP_HASH_SIZE, roundto, ROUNDTO); rc = -1; goto out; } /* read bucket usage percentages and sort them by number of fragments */ for (i = 0; i < HP_LINEAR_HASH_SIZE; i++) { sf = malloc(sizeof *sf); if (!sf) { LM_INFO("malloc failed, skipping shm warming\n"); rc = -1; goto out_free; } sf->hash_index = i; sf->next = NULL; if (fscanf(f, "%lf", &sf->amount) != 1) { LM_CRIT("%s appears to be corrupt. Please remove it first\n", mem_warming_pattern_file); abort(); } if (i == 0) sf->fragments = 0; else sf->fragments = sf->amount * hpb->size / (ROUNDTO * i); if (!sorted_sf) sorted_sf = sf; else { for (it = sorted_sf; it->next && it->next->fragments > sf->fragments; it = it->next) ; if (it->fragments < sf->fragments) { sf->next = sorted_sf; sorted_sf = sf; } else { sf->next = it->next; it->next = sf; } } } /* only optimize the configured number of buckets */ optimized_buckets = (float)shm_hash_split_percentage / 100 * HP_LINEAR_HASH_SIZE; LM_INFO("Optimizing %u / %lu mem buckets\n", optimized_buckets, HP_LINEAR_HASH_SIZE); sf = sorted_sf; for (i = 0; i < optimized_buckets; i++) { hpb->free_hash[sf->hash_index].is_optimized = 1; sf = sf->next; } big_frag = hpb->first_frag; /* populate each free hash bucket with proper number of fragments */ for (sf = sorted_sf; sf; sf = sf->next) { LM_INFO("[%d][%s] fraction: %.12lf total mem: %llu, %lu\n", sf->hash_index, hpb->free_hash[sf->hash_index].is_optimized ? "X" : " ", sf->amount, (unsigned long long) (sf->amount * hpb->size * mem_warming_percentage / 100), ROUNDTO * sf->hash_index); current_frag_size = ROUNDTO * sf->hash_index; bucket_mem = sf->amount * hpb->size * mem_warming_percentage / 100; /* create free fragments worth of 'bucket_mem' memory */ while (bucket_mem >= FRAG_OVERHEAD + current_frag_size) { hp_frag_detach(hpb, big_frag); if (stats_are_ready()) update_stats_shm_frag_detach(big_frag); else { hpb->used += big_frag->size; hpb->real_used += big_frag->size + FRAG_OVERHEAD; } /* trim-insert operation on the big free fragment */ shm_frag_split_unsafe(hpb, big_frag, current_frag_size); /* * "big_frag" now points to a smaller, free and detached frag. * * With optimized buckets, inserts will be automagically * balanced within their dedicated hashes */ hp_frag_attach(hpb, big_frag); if (stats_are_ready()) update_stats_shm_frag_attach(big_frag); else { hpb->used -= big_frag->size; hpb->real_used -= big_frag->size + FRAG_OVERHEAD; } big_frag = FRAG_NEXT(big_frag); bucket_mem -= FRAG_OVERHEAD + current_frag_size; if (c % 1000000 == 0) LM_INFO("%d| %lld %p\n", c, bucket_mem, big_frag); c++; } } out_free: while (sorted_sf) { sf = sorted_sf; sorted_sf = sorted_sf->next; free(sf); } out: fclose(f); return rc; }
void sfm_split_frag(struct sfm_block* qm, struct sfm_frag* frag, unsigned long size) #endif { unsigned long rest; struct sfm_frag* n; int bigger_rest; rest=frag->size-size; #ifdef MEM_FRAG_AVOIDANCE if ((rest> (FRAG_OVERHEAD+SF_MALLOC_OPTIMIZE))|| (rest>=(FRAG_OVERHEAD+size))){ /* the residue fragm. is big enough*/ bigger_rest=1; #else if (rest>(FRAG_OVERHEAD+SF_MIN_FRAG_SIZE)){ bigger_rest=rest>=(size+FRAG_OVERHEAD); #endif frag->size=size; /*split the fragment*/ n=FRAG_NEXT(frag); n->size=rest-FRAG_OVERHEAD; n->id=pool_id; FRAG_CLEAR_USED(n); /* never used */ #ifdef DBG_F_MALLOC /* frag created by malloc, mark it*/ n->file=file; n->func="frag. from sfm_malloc"; n->line=line; n->check=ST_CHECK_PATTERN; #endif /* reinsert n in free list*/ sfm_insert_free(qm, n, bigger_rest); }else{ /* we cannot split this fragment any more => alloc all of it*/ } } /* init malloc and return a sfm_block*/ struct sfm_block* sfm_malloc_init(char* address, unsigned long size, int type) { char* start; char* end; struct sfm_block* qm; unsigned long init_overhead; int r; #ifdef SFM_LOCK_PER_BUCKET int i; #endif /* make address and size multiple of 8*/ start=(char*)ROUNDUP((unsigned long) address); DBG("sfm_malloc_init: SF_OPTIMIZE=%lu, /SF_ROUNDTO=%lu\n", SF_MALLOC_OPTIMIZE, SF_MALLOC_OPTIMIZE/SF_ROUNDTO); DBG("sfm_malloc_init: SF_HASH_SIZE=%lu, sfm_block size=%lu\n", SF_HASH_SIZE, (long)sizeof(struct sfm_block)); DBG("sfm_malloc_init(%p, %lu), start=%p\n", address, size, start); if (size<start-address) return 0; size-=(start-address); if (size <(SF_MIN_FRAG_SIZE+FRAG_OVERHEAD)) return 0; size=ROUNDDOWN(size); init_overhead=INIT_OVERHEAD; if (size < init_overhead) { /* not enough mem to create our control structures !!!*/ return 0; } end=start+size; qm=(struct sfm_block*)start; memset(qm, 0, sizeof(struct sfm_block)); qm->size=size; qm->type = type; size-=init_overhead; qm->first_frag=(struct sfm_frag*)(start+ROUNDUP(sizeof(struct sfm_block))); qm->last_frag=(struct sfm_frag*)(end-sizeof(struct sfm_frag)); /* init initial fragment*/ qm->first_frag->size=size; qm->first_frag->id=(unsigned long)-1; /* not in a pool */ qm->last_frag->size=0; #ifdef DBG_F_MALLOC qm->first_frag->check=ST_CHECK_PATTERN; qm->last_frag->check=END_CHECK_PATTERN1; #endif /* link initial fragment into the free list*/ sfm_insert_free(qm, qm->first_frag, 0); sfm_max_hash=GET_HASH(size); /* init locks */ if (lock_init(&qm->get_and_split)==0) goto error; #ifdef SFM_ONE_LOCK if (lock_init(&qm->lock)==0){ lock_destroy(&qm->get_and_split); goto error; } for (r=0; r<SFM_POOLS_NO; r++){ if (lock_init(&qm->pool[r].lock)==0){ for (;r>0; r--) lock_destroy(&qm->pool[r-1].lock); lock_destroy(&qm->lock); lock_destroy(&qm->get_and_split); goto error; } } #elif defined(SFM_LOCK_PER_BUCKET) for (r=0; r<SF_HASH_SIZE; r++) if (lock_init(&qm->free_hash[r].lock)==0){ for(;r>0; r--) lock_destroy(&qm->free_hash[r-1].lock); lock_destroy(&qm->get_and_split); goto error; } for (i=0; i<SFM_POOLS_NO; i++){ for (r=0; r<SF_HASH_POOL_SIZE; r++) if (lock_init(&qm->pool[i].pool_hash[r].lock)==0){ for(;r>0; r--) lock_destroy(&qm->pool[i].poo_hash[r].lock); for(; i>0; i--){ for (r=0; r<SF_HASH_POOL_SIZE; r++) lock_destroy(&qm->pool[i].pool_hash[r].lock); } for (r=0; r<SF_HASH_SIZE; r++) lock_destroy(&qm->free_hash[r].lock); lock_destroy(&qm->get_and_split); goto error; } } #endif qm->is_init=1; return qm; error: return 0; }
int split_frag(struct qm_block* qm, struct qm_frag* f, unsigned long new_size) #endif { unsigned long rest; struct qm_frag* n; struct qm_frag_end* end; rest=f->size-new_size; #ifdef MEM_FRAG_AVOIDANCE if ((rest> (FRAG_OVERHEAD+QM_MALLOC_OPTIMIZE))|| (rest>=(FRAG_OVERHEAD+new_size))){/* the residue fragm. is big enough*/ #else if (rest>(FRAG_OVERHEAD+MIN_FRAG_SIZE)){ #endif f->size=new_size; /*split the fragment*/ end=FRAG_END(f); end->size=new_size; n=(struct qm_frag*)((char*)end+sizeof(struct qm_frag_end)); n->size=rest-FRAG_OVERHEAD; FRAG_END(n)->size=n->size; FRAG_CLEAR_USED(n); /* never used */ qm->real_used+=FRAG_OVERHEAD; #ifdef DBG_QM_MALLOC end->check1=END_CHECK_PATTERN1; end->check2=END_CHECK_PATTERN2; /* frag created by malloc, mark it*/ n->file=file; n->func=func; n->line=line; n->check=ST_CHECK_PATTERN; #endif /* reinsert n in free list*/ qm_insert_free(qm, n); return 0; }else{ /* we cannot split this fragment any more */ return -1; } } #ifdef DBG_QM_MALLOC void* qm_malloc(void* qmp, unsigned long size, const char* file, const char* func, unsigned int line) #else void* qm_malloc(void* qmp, unsigned long size) #endif { struct qm_block* qm; struct qm_frag* f; int hash; #ifdef DBG_QM_MALLOC unsigned int list_cntr; #endif qm = (struct qm_block*)qmp; #ifdef DBG_QM_MALLOC list_cntr = 0; MDBG("qm_malloc(%p, %lu) called from %s: %s(%d)\n", qm, size, file, func, line); #endif /*malloc(0) should return a valid pointer according to specs*/ if(unlikely(size==0)) size=4; /*size must be a multiple of 8*/ size=ROUNDUP(size); if (size>(qm->size-qm->real_used)) return 0; /*search for a suitable free frag*/ #ifdef DBG_QM_MALLOC if ((f=qm_find_free(qm, size, &hash, &list_cntr))!=0){ #else if ((f=qm_find_free(qm, size, &hash))!=0){ #endif /* we found it!*/ /*detach it from the free list*/ #ifdef DBG_QM_MALLOC qm_debug_frag(qm, f); #endif qm_detach_free(qm, f); /*mark it as "busy"*/ f->u.is_free=0; qm->free_hash[hash].no--; qm->ffrags--; /* we ignore split return */ #ifdef DBG_QM_MALLOC split_frag(qm, f, size, file, "fragm. from qm_malloc", line); #else split_frag(qm, f, size); #endif qm->real_used+=f->size; qm->used+=f->size; if (qm->max_real_used<qm->real_used) qm->max_real_used=qm->real_used; #ifdef DBG_QM_MALLOC f->file=file; f->func=func; f->line=line; f->check=ST_CHECK_PATTERN; /* FRAG_END(f)->check1=END_CHECK_PATTERN1; FRAG_END(f)->check2=END_CHECK_PATTERN2;*/ MDBG("qm_malloc(%p, %lu) returns address %p frag. %p (size=%lu) on %d" " -th hit\n", qm, size, (char*)f+sizeof(struct qm_frag), f, f->size, list_cntr ); #endif #ifdef MALLOC_STATS if(qm->type==MEM_TYPE_PKG) { sr_event_exec(SREV_PKG_UPDATE_STATS, 0); } #endif return (char*)f+sizeof(struct qm_frag); } return 0; } #ifdef DBG_QM_MALLOC void qm_free(void* qmp, void* p, const char* file, const char* func, unsigned int line) #else void qm_free(void* qmp, void* p) #endif { struct qm_block* qm; struct qm_frag* f; unsigned long size; #ifdef MEM_JOIN_FREE struct qm_frag* next; struct qm_frag* prev; #endif /* MEM_JOIN_FREE*/ qm = (struct qm_block*)qmp; #ifdef DBG_QM_MALLOC MDBG("qm_free(%p, %p), called from %s: %s(%d)\n", qm, p, file, func, line); #endif if (p==0) { #ifdef DBG_QM_MALLOC LOG(L_WARN, "WARNING:qm_free: free(0) called from %s: %s(%d)\n", file, func, line); #else LOG(L_WARN, "WARNING:qm_free: free(0) called\n"); #endif return; } #ifdef DBG_QM_MALLOC if (p>(void*)qm->last_frag_end || p<(void*)qm->first_frag){ LOG(L_CRIT, "BUG: qm_free: bad pointer %p (out of memory block!)" " called from %s: %s(%d) - aborting\n", p, file, func, line); if(likely(cfg_get(core, core_cfg, mem_safety)==0)) abort(); else return; } #endif f=(struct qm_frag*) ((char*)p-sizeof(struct qm_frag)); #ifdef DBG_QM_MALLOC qm_debug_frag(qm, f); if (f->u.is_free){ LOG(L_CRIT, "BUG: qm_free: freeing already freed pointer (%p)," " called from %s: %s(%d), first free %s: %s(%ld) - aborting\n", p, file, func, line, f->file, f->func, f->line); if(likely(cfg_get(core, core_cfg, mem_safety)==0)) abort(); else return; } MDBG("qm_free: freeing frag. %p alloc'ed from %s: %s(%ld)\n", f, f->file, f->func, f->line); #endif if (unlikely(f->u.is_free)){ LM_INFO("freeing a free fragment (%p/%p) - ignore\n", f, p); return; } size=f->size; qm->used-=size; qm->real_used-=size; #ifdef MEM_JOIN_FREE if(unlikely(cfg_get(core, core_cfg, mem_join)!=0)) { next=prev=0; /* mark this fragment as used (might fall into the middle of joined frags) to give us an extra chance of detecting a double free call (if the joined fragment has not yet been reused) */ f->u.nxt_free=(void*)0x1L; /* bogus value, just to mark it as free */ /* join packets if possible*/ next=FRAG_NEXT(f); if (((char*)next < (char*)qm->last_frag_end) && (next->u.is_free)){ /* join next packet */ #ifdef DBG_QM_MALLOC qm_debug_frag(qm, next); #endif qm_detach_free(qm, next); size+=next->size+FRAG_OVERHEAD; qm->real_used-=FRAG_OVERHEAD; qm->free_hash[GET_HASH(next->size)].no--; /* FIXME slow */ qm->ffrags--; } if (f > qm->first_frag){ prev=FRAG_PREV(f); /* (struct qm_frag*)((char*)f - (struct qm_frag_end*)((char*)f- sizeof(struct qm_frag_end))->size);*/ if (prev->u.is_free){ /* join prev packet */ #ifdef DBG_QM_MALLOC qm_debug_frag(qm, prev); #endif qm_detach_free(qm, prev); size+=prev->size+FRAG_OVERHEAD; qm->real_used-=FRAG_OVERHEAD; qm->free_hash[GET_HASH(prev->size)].no--; /* FIXME slow */ qm->ffrags--; f=prev; } } f->size=size; FRAG_END(f)->size=f->size; } /* if cfg_core->mem_join */ #endif /* MEM_JOIN_FREE*/ #ifdef DBG_QM_MALLOC f->file=file; f->func=func; f->line=line; #endif qm_insert_free(qm, f); #ifdef MALLOC_STATS if(qm->type==MEM_TYPE_PKG) { sr_event_exec(SREV_PKG_UPDATE_STATS, 0); } #endif } #ifdef DBG_QM_MALLOC void* qm_realloc(void* qmp, void* p, unsigned long size, const char* file, const char* func, unsigned int line) #else void* qm_realloc(void* qmp, void* p, unsigned long size) #endif { struct qm_block* qm; struct qm_frag* f; unsigned long diff; unsigned long orig_size; struct qm_frag* n; void* ptr; qm = (struct qm_block*)qmp; #ifdef DBG_QM_MALLOC MDBG("qm_realloc(%p, %p, %lu) called from %s: %s(%d)\n", qm, p, size, file, func, line); if ((p)&&(p>(void*)qm->last_frag_end || p<(void*)qm->first_frag)){ LOG(L_CRIT, "BUG: qm_free: bad pointer %p (out of memory block!) - " "aborting\n", p); abort(); } #endif if (size==0) { if (p) #ifdef DBG_QM_MALLOC qm_free(qm, p, file, func, line); #else qm_free(qm, p); #endif return 0; } if (p==0) #ifdef DBG_QM_MALLOC return qm_malloc(qm, size, file, func, line); #else return qm_malloc(qm, size); #endif f=(struct qm_frag*) ((char*)p-sizeof(struct qm_frag)); #ifdef DBG_QM_MALLOC qm_debug_frag(qm, f); MDBG("qm_realloc: realloc'ing frag %p alloc'ed from %s: %s(%ld)\n", f, f->file, f->func, f->line); if (f->u.is_free){ LOG(L_CRIT, "BUG:qm_realloc: trying to realloc an already freed " "pointer %p , fragment %p -- aborting\n", p, f); abort(); } #endif /* find first acceptable size */ size=ROUNDUP(size); if (f->size > size){ orig_size=f->size; /* shrink */ #ifdef DBG_QM_MALLOC MDBG("qm_realloc: shrinking from %lu to %lu\n", f->size, size); if(split_frag(qm, f, size, file, "fragm. from qm_realloc", line)!=0){ MDBG("qm_realloc : shrinked successful\n"); #else if(split_frag(qm, f, size)!=0){ #endif /* update used sizes: freed the splited frag */ /* split frag already adds FRAG_OVERHEAD for the newly created free frag, so here we only need orig_size-f->size for real used */ qm->real_used-=(orig_size-f->size); qm->used-=(orig_size-f->size); } }else if (f->size < size){ /* grow */ #ifdef DBG_QM_MALLOC MDBG("qm_realloc: growing from %lu to %lu\n", f->size, size); #endif orig_size=f->size; diff=size-f->size; n=FRAG_NEXT(f); if (((char*)n < (char*)qm->last_frag_end) && (n->u.is_free)&&((n->size+FRAG_OVERHEAD)>=diff)){ /* join */ qm_detach_free(qm, n); qm->free_hash[GET_HASH(n->size)].no--; /*FIXME: slow*/ qm->ffrags--; f->size+=n->size+FRAG_OVERHEAD; qm->real_used-=FRAG_OVERHEAD; FRAG_END(f)->size=f->size; /* end checks should be ok */ /* split it if necessary */ if (f->size > size ){ #ifdef DBG_QM_MALLOC split_frag(qm, f, size, file, "fragm. from qm_realloc", line); #else split_frag(qm, f, size); #endif } qm->real_used+=(f->size-orig_size); qm->used+=(f->size-orig_size); }else{ /* could not join => realloc */ #ifdef DBG_QM_MALLOC ptr=qm_malloc(qm, size, file, func, line); #else ptr=qm_malloc(qm, size); #endif if (ptr){ /* copy, need by libssl */ memcpy(ptr, p, orig_size); } #ifdef DBG_QM_MALLOC qm_free(qm, p, file, func, line); #else qm_free(qm, p); #endif p=ptr; } }else{ /* do nothing */ #ifdef DBG_QM_MALLOC MDBG("qm_realloc: doing nothing, same size: %lu - %lu\n", f->size, size); #endif } #ifdef DBG_QM_MALLOC MDBG("qm_realloc: returning %p\n", p); #endif #ifdef MALLOC_STATS if(qm->type==MEM_TYPE_PKG) { sr_event_exec(SREV_PKG_UPDATE_STATS, 0); } #endif return p; } void qm_check(struct qm_block* qm) { struct qm_frag* f; long fcount = 0; int memlog; memlog=cfg_get(core, core_cfg, memlog); LOG(memlog, "DEBUG: qm_check()\n"); f = qm->first_frag; while ((char*)f < (char*)qm->last_frag_end) { fcount++; /* check struct qm_frag */ #ifdef DBG_QM_MALLOC if (f->check!=ST_CHECK_PATTERN){ LOG(L_CRIT, "BUG: qm_*: fragm. %p (address %p) " "beginning overwritten(%lx)!\n", f, (char*)f + sizeof(struct qm_frag), f->check); qm_status(qm); abort(); }; #endif if (f + sizeof(struct qm_frag) + f->size + sizeof(struct qm_frag_end) > qm->first_frag + qm->size) { LOG(L_CRIT, "BUG: qm_*: fragm. %p (address %p) " "bad size: %lu (frag end: %p > end of block: %p)\n", f, (char*)f + sizeof(struct qm_frag) + sizeof(struct qm_frag_end), f->size, f + sizeof(struct qm_frag) + f->size, qm->first_frag + qm->size); qm_status(qm); abort(); } /* check struct qm_frag_end */ if (FRAG_END(f)->size != f->size) { LOG(L_CRIT, "BUG: qm_*: fragm. %p (address %p) " "size in qm_frag and qm_frag_end does not match: frag->size=%lu, frag_end->size=%lu)\n", f, (char*)f + sizeof(struct qm_frag), f->size, FRAG_END(f)->size); qm_status(qm); abort(); } #ifdef DBG_QM_MALLOC if ((FRAG_END(f)->check1 != END_CHECK_PATTERN1) || (FRAG_END(f)->check2 != END_CHECK_PATTERN2)) { LOG(L_CRIT, "BUG: qm_*: fragm. %p (address %p)" " end overwritten(%lx, %lx)!\n", f, (char*)f + sizeof(struct qm_frag), FRAG_END(f)->check1, FRAG_END(f)->check2); qm_status(qm); abort(); } #endif f = FRAG_NEXT(f); } LOG(memlog, "DEBUG: qm_check: %lu fragments OK\n", fcount); } void qm_status(void* qmp) { struct qm_block* qm; struct qm_frag* f; int i,j; int h; int unused; int memlog; int mem_summary; qm = (struct qm_block*)qmp; memlog=cfg_get(core, core_cfg, memlog); mem_summary=cfg_get(core, core_cfg, mem_summary); LOG_(DEFAULT_FACILITY, memlog, "qm_status: ", "(%p):\n", qm); if (!qm) return; LOG_(DEFAULT_FACILITY, memlog, "qm_status: ", "heap size= %lu\n", qm->size); LOG_(DEFAULT_FACILITY, memlog, "qm_status: ", "used= %lu, used+overhead=%lu, free=%lu\n", qm->used, qm->real_used, qm->size-qm->real_used); LOG_(DEFAULT_FACILITY, memlog, "qm_status: ", "max used (+overhead)= %lu\n", qm->max_real_used); if (mem_summary & 16) return; LOG_(DEFAULT_FACILITY, memlog, "qm_status: ", "dumping all alloc'ed. fragments:\n"); for (f=qm->first_frag, i=0;(char*)f<(char*)qm->last_frag_end;f=FRAG_NEXT(f) ,i++){ if (! f->u.is_free){ LOG_(DEFAULT_FACILITY, memlog, "qm_status: ", " %3d. %c address=%p frag=%p size=%lu used=%d\n", i, (f->u.is_free)?'a':'N', (char*)f+sizeof(struct qm_frag), f, f->size, FRAG_WAS_USED(f)); #ifdef DBG_QM_MALLOC LOG_(DEFAULT_FACILITY, memlog, "qm_status: ", " %s from %s: %s(%ld)\n", (f->u.is_free)?"freed":"alloc'd", f->file, f->func, f->line); LOG_(DEFAULT_FACILITY, memlog, "qm_status: ", " start check=%lx, end check= %lx, %lx\n", f->check, FRAG_END(f)->check1, FRAG_END(f)->check2); #endif } } LOG_(DEFAULT_FACILITY, memlog, "qm_status: ", "dumping free list stats :\n"); for(h=0,i=0;h<QM_HASH_SIZE;h++){ unused=0; for (f=qm->free_hash[h].head.u.nxt_free,j=0; f!=&(qm->free_hash[h].head); f=f->u.nxt_free, i++, j++){ if (!FRAG_WAS_USED(f)){ unused++; #ifdef DBG_QM_MALLOC LOG_(DEFAULT_FACILITY, memlog, "qm_status: ", "unused fragm.: hash = %3d, fragment %p," " address %p size %lu, created from %s: %s(%lu)\n", h, f, (char*)f+sizeof(struct qm_frag), f->size, f->file, f->func, f->line); #endif } } if (j) LOG_(DEFAULT_FACILITY, memlog, "qm_status: ", "hash= %3d. fragments no.: %5d, unused: %5d\n" "\t\t bucket size: %9lu - %9ld (first %9lu)\n", h, j, unused, UN_HASH(h), ((h<=QM_MALLOC_OPTIMIZE/ROUNDTO)?1:2)*UN_HASH(h), qm->free_hash[h].head.u.nxt_free->size ); if (j!=qm->free_hash[h].no){ LOG(L_CRIT, "BUG: qm_status: different free frag. count: %d!=%lu" " for hash %3d\n", j, qm->free_hash[h].no, h); } } LOG_(DEFAULT_FACILITY, memlog, "qm_status: ", "-----------------------------\n"); } /* fills a malloc info structure with info about the block * if a parameter is not supported, it will be filled with 0 */ void qm_info(void* qmp, struct mem_info* info) { struct qm_block* qm; qm = (struct qm_block*)qmp; memset(info,0, sizeof(*info)); info->total_size=qm->size; info->min_frag=MIN_FRAG_SIZE; info->free=qm->size-qm->real_used; info->used=qm->used; info->real_used=qm->real_used; info->max_used=qm->max_real_used; info->total_frags=qm->ffrags; } /* returns how much free memory is available * it never returns an error (unlike fm_available) */ unsigned long qm_available(void* qmp) { struct qm_block* qm; qm = (struct qm_block*)qmp; return qm->size-qm->real_used; } #ifdef DBG_QM_MALLOC typedef struct _mem_counter{ const char *file; const char *func; unsigned long line; unsigned long size; int count; struct _mem_counter *next; } mem_counter; static mem_counter* get_mem_counter(mem_counter **root, struct qm_frag* f) { mem_counter *x; if (!*root) goto make_new; for(x=*root;x;x=x->next) if (x->file == f->file && x->func == f->func && x->line == f->line) return x; make_new: x = malloc(sizeof(mem_counter)); x->file = f->file; x->func = f->func; x->line = f->line; x->count = 0; x->size = 0; x->next = *root; *root = x; return x; } void qm_sums(void* qmp) { struct qm_block* qm; struct qm_frag* f; int i; mem_counter *root, *x; int memlog; qm = (struct qm_block*)qmp; root=0; if (!qm) return; memlog=cfg_get(core, core_cfg, memlog); LOG_(DEFAULT_FACILITY, memlog, "qm_sums: ", "summarizing all alloc'ed. fragments:\n"); for (f=qm->first_frag, i=0;(char*)f<(char*)qm->last_frag_end; f=FRAG_NEXT(f),i++){ if (! f->u.is_free){ x = get_mem_counter(&root,f); x->count++; x->size+=f->size; } } x = root; while(x){ LOG_(DEFAULT_FACILITY, memlog, "qm_sums: ", " count=%6d size=%10lu bytes from %s: %s(%ld)\n", x->count,x->size, x->file, x->func, x->line ); root = x->next; free(x); x = root; } LOG_(DEFAULT_FACILITY, memlog, "qm_sums: ", "-----------------------------\n"); } #else void qm_sums(void* qm) { return; } #endif /* DBG_QM_MALLOC */ /*memory manager core api*/ static char *_qm_mem_name = "q_malloc"; /* PKG - private memory API*/ static char *_qm_pkg_pool = 0; static struct qm_block *_qm_pkg_block = 0; /** * \brief Destroy memory pool */ void qm_malloc_destroy_pkg_manager(void) { if (_qm_pkg_pool) { free(_qm_pkg_pool); _qm_pkg_pool = 0; } _qm_pkg_block = 0; } /** * \brief Init memory pool */ int qm_malloc_init_pkg_manager(void) { sr_pkg_api_t ma; _qm_pkg_pool = malloc(pkg_mem_size); if (_qm_pkg_pool) _qm_pkg_block=qm_malloc_init(_qm_pkg_pool, pkg_mem_size, MEM_TYPE_PKG); if (_qm_pkg_block==0){ LOG(L_CRIT, "could not initialize qm memory pool\n"); fprintf(stderr, "Too much qm pkg memory demanded: %ld bytes\n", pkg_mem_size); return -1; } memset(&ma, 0, sizeof(sr_pkg_api_t)); ma.mname = _qm_mem_name; ma.mem_pool = _qm_pkg_pool; ma.mem_block = _qm_pkg_block; ma.xmalloc = qm_malloc; ma.xfree = qm_free; ma.xrealloc = qm_realloc; ma.xstatus = qm_status; ma.xinfo = qm_info; ma.xavailable = qm_available; ma.xsums = qm_sums; ma.xdestroy = qm_malloc_destroy_pkg_manager; return pkg_init_api(&ma); } /* SHM - shared memory API*/ static void *_qm_shm_pool = 0; static struct qm_block *_qm_shm_block = 0; /*SHM wrappers to sync the access to memory block*/ #ifdef DBG_QM_MALLOC void* qm_shm_malloc(void* qmp, unsigned long size, const char* file, const char* func, unsigned int line) { void *r; shm_lock(); r = qm_malloc(qmp, size, file, func, line); shm_unlock(); return r; }
int split_frag(struct qm_block* qm, struct qm_frag* f, unsigned long new_size) #endif { unsigned long rest; struct qm_frag* n; struct qm_frag_end* end; rest=f->size-new_size; #ifdef MEM_FRAG_AVOIDANCE if ((rest> (FRAG_OVERHEAD+QM_MALLOC_OPTIMIZE))|| (rest>=(FRAG_OVERHEAD+new_size))){/* the residue fragm. is big enough*/ #else if (rest>(FRAG_OVERHEAD+MIN_FRAG_SIZE)){ #endif f->size=new_size; /*split the fragment*/ end=FRAG_END(f); end->size=new_size; n=(struct qm_frag*)((char*)end+sizeof(struct qm_frag_end)); n->size=rest-FRAG_OVERHEAD; FRAG_END(n)->size=n->size; FRAG_CLEAR_USED(n); /* never used */ qm->real_used+=FRAG_OVERHEAD; #ifdef DBG_QM_MALLOC end->check1=END_CHECK_PATTERN1; end->check2=END_CHECK_PATTERN2; /* frag created by malloc, mark it*/ n->file=file; n->func=func; n->line=line; n->check=ST_CHECK_PATTERN; #endif /* reinsert n in free list*/ qm_insert_free(qm, n); return 0; }else{ /* we cannot split this fragment any more */ return -1; } } #ifdef DBG_QM_MALLOC void* qm_malloc(struct qm_block* qm, unsigned long size, const char* file, const char* func, unsigned int line) #else void* qm_malloc(struct qm_block* qm, unsigned long size) #endif { struct qm_frag* f; int hash; #ifdef DBG_QM_MALLOC unsigned int list_cntr; list_cntr = 0; LM_GEN1( memlog, "params (%p, %lu), called from %s: %s(%d)\n", qm, size, file, func, line); #endif /*size must be a multiple of 8*/ size=ROUNDUP(size); if (size>(qm->size-qm->real_used)) return 0; /*search for a suitable free frag*/ #ifdef DBG_QM_MALLOC if ((f=qm_find_free(qm, size, &hash, &list_cntr))!=0){ #else if ((f=qm_find_free(qm, size, &hash))!=0){ #endif /* we found it!*/ /*detach it from the free list*/ #ifdef DBG_QM_MALLOC qm_debug_frag(qm, f); #endif qm_detach_free(qm, f); /*mark it as "busy"*/ f->u.is_free=0; qm->free_hash[hash].no--; /* we ignore split return */ #ifdef DBG_QM_MALLOC split_frag(qm, f, size, file, "fragm. from qm_malloc", line); #else split_frag(qm, f, size); #endif qm->real_used+=f->size; qm->used+=f->size; if (qm->max_real_used<qm->real_used) qm->max_real_used=qm->real_used; #ifdef DBG_QM_MALLOC f->file=file; f->func=func; f->line=line; f->check=ST_CHECK_PATTERN; /* FRAG_END(f)->check1=END_CHECK_PATTERN1; FRAG_END(f)->check2=END_CHECK_PATTERN2;*/ LM_GEN1( memlog, "params (%p, %lu), returns address %p frag. %p " "(size=%lu) on %d -th hit\n", qm, size, (char*)f+sizeof(struct qm_frag), f, f->size, list_cntr ); #endif return (char*)f+sizeof(struct qm_frag); } return 0; } #ifdef DBG_QM_MALLOC void qm_free(struct qm_block* qm, void* p, const char* file, const char* func, unsigned int line) #else void qm_free(struct qm_block* qm, void* p) #endif { struct qm_frag* f; unsigned long size; #ifdef QM_JOIN_FREE struct qm_frag* next; struct qm_frag* prev; #endif #ifdef DBG_QM_MALLOC LM_GEN1( memlog, "params(%p, %p), called from %s: %s(%d)\n", qm, p, file, func, line); if (p>(void*)qm->last_frag_end || p<(void*)qm->first_frag){ LM_CRIT("bad pointer %p (out of memory block!) - aborting\n", p); abort(); } #endif if (p==0) { LM_WARN("free(0) called\n"); return; } #ifdef QM_JOIN_FREE prev=next=0; #endif f=(struct qm_frag*) ((char*)p-sizeof(struct qm_frag)); #ifdef DBG_QM_MALLOC qm_debug_frag(qm, f); if (f->u.is_free){ LM_CRIT("freeing already freed pointer," " first free: %s: %s(%ld) - aborting\n", f->file, f->func, f->line); abort(); } LM_GEN1( memlog, "freeing frag. %p alloc'ed from %s: %s(%ld)\n", f, f->file, f->func, f->line); #endif size=f->size; qm->used-=size; qm->real_used-=size; #ifdef QM_JOIN_FREE /* join packets if possible*/ next=FRAG_NEXT(f); if (((char*)next < (char*)qm->last_frag_end) &&( next->u.is_free)){ /* join */ #ifdef DBG_QM_MALLOC qm_debug_frag(qm, next); #endif qm_detach_free(qm, next); size+=next->size+FRAG_OVERHEAD; qm->real_used-=FRAG_OVERHEAD; qm->free_hash[GET_HASH(next->size)].no--; /* FIXME slow */ } if (f > qm->first_frag){ prev=FRAG_PREV(f); /* (struct qm_frag*)((char*)f - (struct qm_frag_end*)((char*)f- sizeof(struct qm_frag_end))->size);*/ #ifdef DBG_QM_MALLOC qm_debug_frag(qm, prev); #endif if (prev->u.is_free){ /*join*/ qm_detach_free(qm, prev); size+=prev->size+FRAG_OVERHEAD; qm->real_used-=FRAG_OVERHEAD; qm->free_hash[GET_HASH(prev->size)].no--; /* FIXME slow */ f=prev; } } f->size=size; FRAG_END(f)->size=f->size; #endif /* QM_JOIN_FREE*/ #ifdef DBG_QM_MALLOC f->file=file; f->func=func; f->line=line; #endif qm_insert_free(qm, f); } #ifdef DBG_QM_MALLOC void* qm_realloc(struct qm_block* qm, void* p, unsigned long size, const char* file, const char* func, unsigned int line) #else void* qm_realloc(struct qm_block* qm, void* p, unsigned long size) #endif { struct qm_frag* f; unsigned long diff; unsigned long orig_size; struct qm_frag* n; void* ptr; #ifdef DBG_QM_MALLOC LM_GEN1( memlog, "params (%p, %p, %lu), called from %s: %s(%d)\n", qm, p, size, file, func, line); if ((p)&&(p>(void*)qm->last_frag_end || p<(void*)qm->first_frag)){ LM_CRIT("bad pointer %p (out of memory block!) - aborting\n", p); abort(); } #endif if (size==0) { if (p) #ifdef DBG_QM_MALLOC qm_free(qm, p, file, func, line); #else qm_free(qm, p); #endif return 0; } if (p==0) #ifdef DBG_QM_MALLOC return qm_malloc(qm, size, file, func, line); #else return qm_malloc(qm, size); #endif f=(struct qm_frag*) ((char*)p-sizeof(struct qm_frag)); #ifdef DBG_QM_MALLOC qm_debug_frag(qm, f); LM_GEN1( memlog, "realloc'ing frag %p alloc'ed from %s: %s(%ld)\n", f, f->file, f->func, f->line); if (f->u.is_free){ LM_CRIT("trying to realloc an already freed " "pointer %p , fragment %p -- aborting\n", p, f); abort(); } #endif /* find first acceptable size */ size=ROUNDUP(size); if (f->size > size){ orig_size=f->size; /* shrink */ #ifdef DBG_QM_MALLOC LM_GEN1(memlog,"shrinking from %lu to %lu\n", f->size, size); if(split_frag(qm, f, size, file, "fragm. from qm_realloc", line)!=0){ LM_GEN1(memlog,"shrinked successful\n"); #else if(split_frag(qm, f, size)!=0){ #endif /* update used sizes: freed the spitted frag */ qm->real_used-=(orig_size-f->size-FRAG_OVERHEAD); qm->used-=(orig_size-f->size); } }else if (f->size < size){ /* grow */ #ifdef DBG_QM_MALLOC LM_GEN1( memlog, "growing from %lu to %lu\n", f->size, size); #endif orig_size=f->size; diff=size-f->size; n=FRAG_NEXT(f); if (((char*)n < (char*)qm->last_frag_end) && (n->u.is_free)&&((n->size+FRAG_OVERHEAD)>=diff)){ /* join */ qm_detach_free(qm, n); qm->free_hash[GET_HASH(n->size)].no--; /*FIXME: slow*/ f->size+=n->size+FRAG_OVERHEAD; qm->real_used-=FRAG_OVERHEAD; FRAG_END(f)->size=f->size; /* end checks should be ok */ /* split it if necessary */ if (f->size > size ){ #ifdef DBG_QM_MALLOC split_frag(qm, f, size, file, "fragm. from qm_realloc", line); #else split_frag(qm, f, size); #endif } qm->real_used+=(f->size-orig_size); qm->used+=(f->size-orig_size); }else{ /* could not join => realloc */ #ifdef DBG_QM_MALLOC ptr=qm_malloc(qm, size, file, func, line); #else ptr=qm_malloc(qm, size); #endif if (ptr) { /* copy, need by libssl */ memcpy(ptr, p, orig_size); #ifdef DBG_QM_MALLOC qm_free(qm, p, file, func, line); #else qm_free(qm, p); #endif } p=ptr; } }else{ /* do nothing */ #ifdef DBG_QM_MALLOC LM_GEN1(memlog,"doing nothing, same size: %lu - %lu\n", f->size, size); #endif } #ifdef DBG_QM_MALLOC LM_GEN1(memlog,"returning %p\n", p); #endif return p; } void qm_status(struct qm_block* qm) { struct qm_frag* f; int i,j; int h; int unused; LM_GEN1(memdump, "qm_status (%p):\n", qm); if (!qm) return; LM_GEN1(memdump, " heap size= %lu\n", qm->size); LM_GEN1(memdump, " used= %lu, used+overhead=%lu, free=%lu\n", qm->used, qm->real_used, qm->size-qm->real_used); LM_GEN1(memdump, " max used (+overhead)= %lu\n", qm->max_real_used); LM_GEN1(memdump, "dumping all alloc'ed. fragments:\n"); for (f=qm->first_frag, i=0;(char*)f<(char*)qm->last_frag_end;f=FRAG_NEXT(f) ,i++){ if (! f->u.is_free){ LM_GEN1(memdump," %3d. %c address=%p frag=%p size=%lu used=%d\n", i, (f->u.is_free)?'a':'N', (char*)f+sizeof(struct qm_frag), f, f->size, FRAG_WAS_USED(f)); #ifdef DBG_QM_MALLOC LM_GEN1(memdump, " %s from %s: %s(%ld)\n", (f->u.is_free)?"freed":"alloc'd", f->file, f->func, f->line); LM_GEN1(memdump, " start check=%lx, end check= %lx, %lx\n", f->check, FRAG_END(f)->check1, FRAG_END(f)->check2); #endif } } LM_GEN1(memdump, "dumping free list stats :\n"); for(h=0,i=0;h<QM_HASH_SIZE;h++){ unused=0; for (f=qm->free_hash[h].head.u.nxt_free,j=0; f!=&(qm->free_hash[h].head); f=f->u.nxt_free, i++, j++){ if (!FRAG_WAS_USED(f)){ unused++; #ifdef DBG_QM_MALLOC LM_GEN1(memdump, "unused fragm.: hash = %3d, fragment %p," " address %p size %lu, created from %s: %s(%lu)\n", h, f, (char*)f+sizeof(struct qm_frag), f->size, f->file, f->func, f->line); #endif } } if (j) LM_GEN1(memdump, "hash= %3d. fragments no.: %5d, unused: %5d\n" "\t\t bucket size: %9lu - %9ld (first %9lu)\n", h, j, unused, UN_HASH(h), ((h<=QM_MALLOC_OPTIMIZE/ROUNDTO)?1:2)*UN_HASH(h), qm->free_hash[h].head.u.nxt_free->size ); if (j!=qm->free_hash[h].no){ LM_CRIT("different free frag. count: %d!=%lu" " for hash %3d\n", j, qm->free_hash[h].no, h); } } LM_GEN1(memdump, "-----------------------------\n"); } /* fills a malloc info structure with info about the block * if a parameter is not supported, it will be filled with 0 */ void qm_info(struct qm_block* qm, struct mem_info* info) { int r; long total_frags; total_frags=0; memset(info,0, sizeof(*info)); info->total_size=qm->size; info->min_frag=MIN_FRAG_SIZE; info->free=qm->size-qm->real_used; info->used=qm->used; info->real_used=qm->real_used; info->max_used=qm->max_real_used; for(r=0;r<QM_HASH_SIZE; r++){ total_frags+=qm->free_hash[r].no; } info->total_frags=total_frags; }
void* sfm_realloc(struct sfm_block* qm, void* p, unsigned long size) #endif { struct sfm_frag *f; unsigned long orig_size; void *ptr; #ifndef SFM_REALLOC_REMALLOC struct sfm_frag *n; struct sfm_frag **pf; unsigned long diff; unsigned long p_id; int hash; unsigned long n_size; struct sfm_pool * pool; #endif #ifdef DBG_F_MALLOC MDBG("sfm_realloc(%p, %p, %lu) called from %s: %s(%d)\n", qm, p, size, file, func, line); if ((p)&&(p>(void*)qm->last_frag || p<(void*)qm->first_frag)){ LOG(L_CRIT, "BUG: sfm_free: bad pointer %p (out of memory block!) - " "aborting\n", p); abort(); } #endif if (size==0) { if (p) #ifdef DBG_F_MALLOC sfm_free(qm, p, file, func, line); #else sfm_free(qm, p); #endif return 0; } if (p==0) #ifdef DBG_F_MALLOC return sfm_malloc(qm, size, file, func, line); #else return sfm_malloc(qm, size); #endif f=(struct sfm_frag*) ((char*)p-sizeof(struct sfm_frag)); #ifdef DBG_F_MALLOC MDBG("sfm_realloc: realloc'ing frag %p alloc'ed from %s: %s(%ld)\n", f, f->file, f->func, f->line); #endif size=ROUNDUP(size); orig_size=f->size; if (f->size > size){ /* shrink */ #ifdef DBG_F_MALLOC MDBG("sfm_realloc: shrinking from %lu to %lu\n", f->size, size); sfm_split_frag(qm, f, size, file, "frag. from sfm_realloc", line); #else sfm_split_frag(qm, f, size); #endif }else if (f->size<size){ /* grow */ #ifdef DBG_F_MALLOC MDBG("sfm_realloc: growing from %lu to %lu\n", f->size, size); #endif #ifndef SFM_REALLOC_REMALLOC /* should set a magic value in list head and in push/pop if magic value => * lock and wait */ #error LL_MALLOC realloc not finished yet diff=size-f->size; n=FRAG_NEXT(f); if (((char*)n < (char*)qm->last_frag) && (n->u.nxt_free)&&((n->size+FRAG_OVERHEAD)>=diff)){ /* join */ /* detach n from the free list */ try_again: p_id=n->id; n_size=n->size; if ((unlikely(p_id >=SFM_POOLS_NO))){ hash=GET_HASH(n_size); SFM_MAIN_HASH_LOCK(qm, hash); if (unlikely((n->u.nxt_free==0) || ((n->size+FRAG_OVERHEAD)<diff))){ SFM_MAIN_HASH_UNLOCK(qm, hash); goto not_found; } if (unlikely((n->id!=p_id) || (n->size!=n_size))){ /* fragment still free, but changed, either * moved to another pool or has a diff. size */ SFM_MAIN_HASH_UNLOCK(qm, hash); goto try_again; } pf=&(qm->free_hash[hash].first); /* find it */ for(;(*pf)&&(*pf!=n); pf=&((*pf)->u.nxt_free));/*FIXME slow */ if (*pf==0){ SFM_MAIN_HASH_UNLOCK(qm, hash); /* not found, bad! */ LOG(L_WARN, "WARNING: sfm_realloc: could not find %p in " "free " "list (hash=%d)\n", n, hash); /* somebody is in the process of changing it ? */ goto not_found; } /* detach */ *pf=n->u.nxt_free; n->u.nxt_free=0; /* mark it immediately as detached */ qm->free_hash[hash].no--; SFM_MAIN_HASH_UNLOCK(qm, hash); /* join */ f->size+=n->size+FRAG_OVERHEAD; /* split it if necessary */ if (f->size > size){ #ifdef DBG_F_MALLOC sfm_split_frag(qm, f, size, file, "fragm. from " "sfm_realloc", line); #else sfm_split_frag(qm, f, size); #endif } }else{ /* p_id < SFM_POOLS_NO (=> in a pool )*/ hash=GET_SMALL_HASH(n_size); pool=&qm->pool[p_id]; SFM_POOL_LOCK(pool, hash); if (unlikely((n->u.nxt_free==0) || ((n->size+FRAG_OVERHEAD)<diff))){ SFM_POOL_UNLOCK(pool, hash); goto not_found; } if (unlikely((n->id!=p_id) || (n->size!=n_size))){ /* fragment still free, but changed, either * moved to another pool or has a diff. size */ SFM_POOL_UNLOCK(pool, hash); goto try_again; } pf=&(pool->pool_hash[hash].first); /* find it */ for(;(*pf)&&(*pf!=n); pf=&((*pf)->u.nxt_free));/*FIXME slow */ if (*pf==0){ SFM_POOL_UNLOCK(pool, hash); /* not found, bad! */ LOG(L_WARN, "WARNING: sfm_realloc: could not find %p in " "free " "list (hash=%d)\n", n, hash); /* somebody is in the process of changing it ? */ goto not_found; } /* detach */ *pf=n->u.nxt_free; n->u.nxt_free=0; /* mark it immediately as detached */ pool->pool_hash[hash].no--; SFM_POOL_UNLOCK(pool, hash); /* join */ f->size+=n->size+FRAG_OVERHEAD; /* split it if necessary */ if (f->size > size){ #ifdef DBG_F_MALLOC sfm_split_frag(qm, f, size, file, "fragm. from " "sfm_realloc", line); #else sfm_split_frag(qm, f, size); #endif } } }else{ not_found: /* could not join => realloc */ #else/* SFM_REALLOC_REMALLOC */ { #endif /* SFM_REALLOC_REMALLOC */ #ifdef DBG_F_MALLOC ptr=sfm_malloc(qm, size, file, func, line); #else ptr=sfm_malloc(qm, size); #endif if (ptr){ /* copy, need by libssl */ memcpy(ptr, p, orig_size); #ifdef DBG_F_MALLOC sfm_free(qm, p, file, func, line); #else sfm_free(qm, p); #endif } p=ptr; } }else{