void *Mem::gbmalloc ( size_t size , const char *note ) {
logTrace( g_conf.m_logTraceMem, "size=%zu note='%s'", size, note );
// don't let electric fence zap us
if ( size == 0 ) return (void *)0x7fffffff;
if ( allocationShouldFailRandomly() ) {
g_errno = ENOMEM;
log( LOG_WARN, "mem: malloc-fake(%zu,%s): %s",size,note, mstrerror(g_errno));
return NULL;
}
retry:
size_t max = g_conf.m_maxMem;
// don't go over max
if ( g_mem.getUsedMem() + size + UNDERPAD + OVERPAD >= max ) {
// try to free temp mem. returns true if it freed some.
if ( freeCacheMem() ) goto retry;
g_errno = ENOMEM;
log( LOG_WARN, "mem: malloc(%zu): Out of memory", size );
return NULL;
}
void *mem;
mem = (void *)sysmalloc ( size + UNDERPAD + OVERPAD );
int32_t memLoop = 0;
mallocmemloop:
if ( ! mem && size > 0 ) {
g_mem.m_outOfMems++;
// try to free temp mem. returns true if it freed some.
if ( freeCacheMem() ) goto retry;
g_errno = errno;
static int64_t s_lastTime;
static int32_t s_missed = 0;
int64_t now = gettimeofdayInMillisecondsLocal();
int64_t avail = (int64_t)g_conf.m_maxMem - (int64_t)m_used;
if ( now - s_lastTime >= 1000LL ) {
log(LOG_WARN, "mem: system malloc(%zu,%s) availShouldBe=%" PRId64": "
"%s (%s) (ooms suppressed since last log msg = %" PRId32")",
size+UNDERPAD+OVERPAD,
note,
avail,
mstrerror(g_errno),
note,
s_missed);
s_lastTime = now;
s_missed = 0;
} else {
s_missed++;
}
return NULL;
}
if ( (PTRTYPE)mem < 0x00010000 ) {
void *remem = sysmalloc(size);
log( LOG_WARN, "mem: Caught low memory allocation "
"at %08" PTRFMT", "
"reallocated to %08" PTRFMT"",
(PTRTYPE)mem, (PTRTYPE)remem );
sysfree(mem);
mem = remem;
memLoop++;
if ( memLoop > 100 ) {
log( LOG_WARN, "mem: Attempted to reallocate low "
"memory allocation 100 times, "
"aborting and returning NOMEM." );
g_errno = ENOMEM;
return NULL;
}
goto mallocmemloop;
}
logTrace( g_conf.m_logTraceMem, "mem=%p size=%zu note='%s'", mem, size, note );
addMem ( (char *)mem + UNDERPAD , size , note , 0 );
return (char *)mem + UNDERPAD;
}
void *Mem::gbrealloc ( void *ptr , size_t oldSize , size_t newSize , const char *note ) {
logTrace( g_conf.m_logTraceMem, "ptr=%p oldSize=%zu newSize=%zu note='%s'", ptr, oldSize, newSize, note );
// return dummy values since realloc() returns NULL if failed
if ( oldSize == 0 && newSize == 0 ) return (void *)0x7fffffff;
// do nothing if size is same
if ( oldSize == newSize ) return ptr;
// if newSize is 0...
if ( newSize == 0 ) {
gbfree(ptr, note, oldSize, true);
return (void *)0x7fffffff;
}
retry:
// hack so hostid #0 can use more mem
size_t max = g_conf.m_maxMem;
//if ( g_hostdb.m_hostId == 0 ) max += 2000000000;
// don't go over max
if ( g_mem.getUsedMem() + newSize - oldSize >= max ) {
// try to free temp mem. returns true if it freed some.
if ( freeCacheMem() ) goto retry;
g_errno = ENOMEM;
log( LOG_WARN, "mem: realloc(%zu,%zu): Out of memory.",oldSize,newSize);
return NULL;
}
// if oldSize is 0, use our malloc() instead
if ( oldSize == 0 ) {
return gbmalloc ( newSize , note );
}
// assume it will be successful. we can't call rmMem() after
// calling sysrealloc() because it will mess up our MAGICCHAR buf
rmMem(ptr, oldSize, note, true);
// . do the actual realloc
// . CAUTION: don't pass in 0x7fffffff in as "ptr"
// . this was causing problems
char *mem = (char *)sysrealloc ( (char *)ptr - UNDERPAD , newSize + UNDERPAD + OVERPAD );
// remove old guy on sucess
if ( mem ) {
addMem ( (char *)mem + UNDERPAD , newSize , note , 0 );
char *returnMem = mem + UNDERPAD;
// set magic char bytes for mem
for ( int32_t i = 0 ; i < UNDERPAD ; i++ )
returnMem[0-i-1] = MAGICCHAR;
for ( int32_t i = 0 ; i < OVERPAD ; i++ )
returnMem[0+newSize+i] = MAGICCHAR;
return returnMem;
}
// ok, just try using malloc then!
mem = (char *)mmalloc ( newSize , note );
// bail on error
if ( ! mem ) {
g_mem.m_outOfMems++;
// restore the original buf we tried to grow
addMem ( ptr , oldSize , note , 0 );
errno = g_errno = ENOMEM;
return NULL;
}
// log a note
log(LOG_INFO,"mem: had to use malloc+memcpy instead of realloc.");
// copy over to it
memcpy ( mem, ptr, oldSize );
// we already called rmMem() so don't double call
sysfree ( (char *)ptr - UNDERPAD );
return mem;
}
void *Mem::gbmalloc ( int size , const char *note ) {
logTrace( g_conf.m_logTraceMem, "size=%d note='%s'", size, note );
// don't let electric fence zap us
if ( size == 0 ) return (void *)0x7fffffff;
// random oom testing
//static int32_t s_mcount = 0;
//s_mcount++;
if ( g_conf.m_testMem && (rand() % 100) < 2 ) {
//if ( s_mcount > 1055 && (rand() % 1000) < 2 ) {
g_errno = ENOMEM;
log( LOG_WARN, "mem: malloc-fake(%i,%s): %s",size,note, mstrerror(g_errno));
return NULL;
}
retry:
int64_t max = g_conf.m_maxMem;
// don't go over max
if ( m_used + size + UNDERPAD + OVERPAD >= max ) {
// try to free temp mem. returns true if it freed some.
if ( freeCacheMem() ) goto retry;
g_errno = ENOMEM;
log( LOG_WARN, "mem: malloc(%i): Out of memory", size );
return NULL;
}
if ( size < 0 ) {
g_errno = EBADENGINEER;
log( LOG_ERROR, "mem: malloc(%i): Bad value.", size );
char *xx = NULL; *xx = 0;
return NULL;
}
void *mem;
g_inMemFunction = true;
mem = (void *)sysmalloc ( size + UNDERPAD + OVERPAD );
g_inMemFunction = false;
int32_t memLoop = 0;
mallocmemloop:
if ( ! mem && size > 0 ) {
g_mem.m_outOfMems++;
// try to free temp mem. returns true if it freed some.
if ( freeCacheMem() ) goto retry;
g_errno = errno;
static int64_t s_lastTime;
static int32_t s_missed = 0;
int64_t now = gettimeofdayInMillisecondsLocal();
int64_t avail = (int64_t)g_conf.m_maxMem -
(int64_t)m_used;
if ( now - s_lastTime >= 1000LL ) {
log(LOG_WARN, "mem: system malloc(%i,%s) availShouldBe=%" PRId64": "
"%s (%s) (ooms suppressed since last log msg = %" PRId32")",
size+UNDERPAD+OVERPAD,
note,
avail,
mstrerror(g_errno),
note,
s_missed);
s_lastTime = now;
s_missed = 0;
} else {
s_missed++;
}
// to debug oom issues:
//char *xx=NULL;*xx=0;
// send an email alert if this happens! it is a sign of "memory fragmentation"
//static bool s_sentEmail = false;
// stop sending these now... seems to be problematic. says
// 160MB is avail and can't alloc 20MB...
static bool s_sentEmail = true;
// assume only 90% is really available because of
// inefficient mallocing
avail = (int64_t)((float)avail * 0.80);
// but if it is within about 15MB of what is theoretically
// available, don't send an email, because there is always some
// minor fragmentation
if ( ! s_sentEmail && avail > size ) {
s_sentEmail = true;
char msgbuf[1024];
Host *h = g_hostdb.m_myHost;
snprintf(msgbuf, 1024,
"Possible memory fragmentation "
"on host #%" PRId32" %s",
h->m_hostId,h->m_note);
log(LOG_WARN, "query: %s",msgbuf);
g_pingServer.sendEmail(NULL, msgbuf,true,true);
}
return NULL;
}
if ( (PTRTYPE)mem < 0x00010000 ) {
void *remem = sysmalloc(size);
log ( LOG_WARN, "mem: Caught low memory allocation "
"at %08" PTRFMT", "
"reallocated to %08" PTRFMT"",
(PTRTYPE)mem, (PTRTYPE)remem );
sysfree(mem);
mem = remem;
memLoop++;
if ( memLoop > 100 ) {
log ( LOG_WARN, "mem: Attempted to reallocate low "
"memory allocation 100 times, "
"aborting and returning NOMEM." );
g_errno = ENOMEM;
return NULL;
}
goto mallocmemloop;
}
logTrace( g_conf.m_logTraceMem, "mem=%p size=%d note='%s'", mem, size, note );
addMem ( (char *)mem + UNDERPAD , size , note , 0 );
return (char *)mem + UNDERPAD;
}
