/*
* Single-lined comments can now be logged safely, together with
* process, thread and user ID, and the current time.
*/
void
GDKlog(const char *format, ...)
{
va_list ap;
char *p = 0, buf[1024];
int mustopen = GDKgetHome();
time_t tm = time(0);
if (MT_pagesize() == 0)
return;
va_start(ap, format);
vsprintf(buf, format, ap);
va_end(ap);
/* remove forbidden characters from message */
for (p = buf; (p = strchr(p, '\n')) != NULL; *p = ' ')
;
for (p = buf; (p = strchr(p, '@')) != NULL; *p = ' ')
;
fseek(GDKlockFile, 0, SEEK_END);
#ifndef HAVE_GETUID
#define getuid() 0
#endif
fprintf(GDKlockFile, "USR=%d PID=%d TIME=%.24s @ %s\n", (int) getuid(), (int) getpid(), ctime(&tm), buf);
fflush(GDKlockFile);
if (mustopen)
GDKunlockHome();
}
static void
GDKvmtrim(void *limit)
{
int highload = 0;
ssize_t prevmem = 0, prevrss = 0;
(void) limit;
do {
int t;
size_t rss;
ssize_t rssdiff, memdiff;
size_t cursize;
/* sleep using catnaps so we can exit in a timely fashion */
for (t = highload ? 500 : 5000; t > 0; t -= CATNAP) {
MT_sleep_ms(CATNAP);
if (GDKexiting())
return;
}
rss = MT_getrss();
rssdiff = (ssize_t) rss - (ssize_t) prevrss;
cursize = GDKvm_cursize();
memdiff = (ssize_t) cursize - (ssize_t) prevmem;
MEMDEBUG THRprintf(GDKstdout, "alloc = " SZFMT " %+zd rss = " SZFMT " %+zd\n", cursize, memdiff, rss, rssdiff);
prevmem = cursize;
prevrss = rss;
if (memdiff >= 0 && rssdiff < -32 * (ssize_t) MT_pagesize()) {
BBPtrim(rss);
highload = 1;
} else {
highload = 0;
}
} while (!GDKexiting());
}
str
BKCgetSize(lng *tot, const bat *bid){
BAT *b;
lng size = 0;
lng blksize = (lng) MT_pagesize();
if ((b = BATdescriptor(*bid)) == NULL) {
throw(MAL, "bat.getDiskSize", RUNTIME_OBJECT_MISSING);
}
size = sizeof (bat);
if ( !isVIEW(b)) {
BUN cnt = BATcapacity(b);
size += ROUND_UP(b->H->heap.free, blksize);
size += ROUND_UP(b->T->heap.free, blksize);
if (b->H->vheap)
size += ROUND_UP(b->H->vheap->free, blksize);
if (b->T->vheap)
size += ROUND_UP(b->T->vheap->free, blksize);
if (b->H->hash)
size += ROUND_UP(sizeof(BUN) * cnt, blksize);
if (b->T->hash)
size += ROUND_UP(sizeof(BUN) * cnt, blksize);
size += IMPSimprintsize(b);
}
*tot = size;
BBPunfix(*bid);
return MAL_SUCCEED;
}
/*
* Really really get the lock. Now!!
*/
static int
GDKgetHome(void)
{
if (MT_pagesize() == 0 || GDKlockFile)
return 0;
while ((GDKlockFile = fopen(GDKLOCK, "r+")) == NULL) {
GDKerror("GDKgetHome: PANIC on open %s. sleep(1)\n", GDKLOCK);
MT_sleep_ms(1000);
}
if (MT_lockf(GDKLOCK, F_TLOCK, 4, 1) < 0) {
IODEBUG THRprintf(GDKstdout, "#GDKgetHome: blocking on lock '%s'.\n", GDKLOCK);
MT_lockf(GDKLOCK, F_LOCK, 4, 1);
}
return 1;
}
static void
monet_hello(void)
{
#ifdef MONETDB_STATIC
char *linkinfo = "statically";
#else
char *linkinfo = "dynamically";
#endif
dbl sz_mem_h;
char *qc = " kMGTPE";
int qi = 0;
monet_memory = MT_npages() * MT_pagesize();
sz_mem_h = (dbl) monet_memory;
while (sz_mem_h >= 1000.0 && qi < 6) {
sz_mem_h /= 1024.0;
qi++;
}
printf("# MonetDB 5 server v" VERSION);
if (strcmp(MONETDB_RELEASE, "unreleased") == 0)
printf("\n# This is an unreleased version");
else
printf(" \"%s\"", MONETDB_RELEASE);
printf("\n# Serving database '%s', using %d thread%s\n",
GDKgetenv("gdk_dbname"),
GDKnr_threads, (GDKnr_threads != 1) ? "s" : "");
printf("# Compiled for %s/" SZFMT "bit with " SZFMT "bit OIDs %s linked\n",
HOST, sizeof(ptr) * 8, sizeof(oid) * 8, linkinfo);
printf("# Found %.3f %ciB available main-memory.\n",
sz_mem_h, qc[qi]);
#ifdef MONET_GLOBAL_DEBUG
printf("# Database path:%s\n", GDKgetenv("gdk_dbpath"));
printf("# Module path:%s\n", GDKgetenv("monet_mod_path"));
#endif
printf("# Copyright (c) 1993-July 2008 CWI.\n");
printf("# Copyright (c) August 2008-2013 MonetDB B.V., all rights reserved\n");
printf("# Visit http://www.monetdb.org/ for further information\n");
}
void
addOptimizers(Client c, MalBlkPtr mb, char *pipe)
{
int i;
InstrPtr q;
backend *be;
str msg;
lng space;
be = (backend *) c->sqlcontext;
assert(be && be->mvc); /* SQL clients should always have their state set */
space = SQLgetSpace(be->mvc, mb);
if(space && (pipe == NULL || strcmp(pipe,"default_pipe")== 0)){
if( space > (lng)(0.8 * MT_npages() * MT_pagesize()) && GDKnr_threads > 1){
pipe = "volcano_pipe";
//mnstr_printf(GDKout, "#use volcano optimizer pipeline? "SZFMT"\n", space);
}else
pipe = "default_pipe";
} else
pipe = pipe? pipe: "default_pipe";
msg = addOptimizerPipe(c, mb, pipe);
if (msg)
GDKfree(msg); /* what to do with an error? */
if (be->mvc->no_mitosis) {
for (i = mb->stop - 1; i > 0; i--) {
q = getInstrPtr(mb, i);
if (q->token == ENDsymbol)
break;
if (getFunctionId(q) == mitosisRef || getFunctionId(q) == dataflowRef)
q->token = REMsymbol; /* they are ignored */
}
}
if (be->mvc->emod & mod_debug)
addtoMalBlkHistory(mb, "getStatistics");
}
int
GDKinit(opt *set, int setlen)
{
char *dbpath = mo_find_option(set, setlen, "gdk_dbpath");
char *p;
opt *n;
int i, j, nlen = 0;
char buf[16];
/* some sanity checks (should also find if symbols are not defined) */
assert(sizeof(char) == SIZEOF_CHAR);
assert(sizeof(short) == SIZEOF_SHORT);
assert(sizeof(int) == SIZEOF_INT);
assert(sizeof(long) == SIZEOF_LONG);
assert(sizeof(lng) == SIZEOF_LNG);
assert(sizeof(oid) == SIZEOF_OID);
assert(sizeof(void *) == SIZEOF_VOID_P);
assert(sizeof(wrd) == SIZEOF_WRD);
assert(sizeof(size_t) == SIZEOF_SIZE_T);
assert(sizeof(ptrdiff_t) == SIZEOF_PTRDIFF_T);
assert(SIZEOF_OID == SIZEOF_INT || SIZEOF_OID == SIZEOF_LNG);
#ifdef NEED_MT_LOCK_INIT
MT_lock_init(&MT_system_lock,"MT_system_lock");
ATOMIC_INIT(GDKstoppedLock, "GDKstoppedLock");
ATOMIC_INIT(mbyteslock, "mbyteslock");
MT_lock_init(&GDKnameLock, "GDKnameLock");
MT_lock_init(&GDKthreadLock, "GDKthreadLock");
MT_lock_init(&GDKtmLock, "GDKtmLock");
#endif
for (i = 0; i <= BBP_BATMASK; i++) {
MT_lock_init(&GDKbatLock[i].swap, "GDKswapLock");
MT_lock_init(&GDKbatLock[i].hash, "GDKhashLock");
MT_lock_init(&GDKbatLock[i].imprints, "GDKimprintsLock");
}
for (i = 0; i <= BBP_THREADMASK; i++) {
MT_lock_init(&GDKbbpLock[i].alloc, "GDKcacheLock");
MT_lock_init(&GDKbbpLock[i].trim, "GDKtrimLock");
GDKbbpLock[i].free = 0;
}
errno = 0;
if (!GDKenvironment(dbpath))
return 0;
if ((p = mo_find_option(set, setlen, "gdk_debug")))
GDKdebug = strtol(p, NULL, 10);
if ((p = mo_find_option(set, setlen, "gdk_mem_pagebits")))
GDK_mem_pagebits = (int) strtol(p, NULL, 10);
mnstr_init();
MT_init_posix();
THRinit();
#ifndef NATIVE_WIN32
BATSIGinit();
#endif
#ifdef WIN32
(void) signal(SIGABRT, BATSIGabort);
_set_abort_behavior(0, _CALL_REPORTFAULT | _WRITE_ABORT_MSG);
_set_error_mode(_OUT_TO_STDERR);
#endif
GDKlockHome();
/* Mserver by default takes 80% of all memory as a default */
GDK_mem_maxsize = GDK_mem_maxsize_max = (size_t) ((double) MT_npages() * (double) MT_pagesize() * 0.815);
#ifdef NATIVE_WIN32
GDK_mmap_minsize = GDK_mem_maxsize_max;
#else
GDK_mmap_minsize = MIN( 1<<30 , GDK_mem_maxsize_max/6 );
/* per op: 2 args + 1 res, each with head & tail => (2+1)*2 = 6 ^ */
#endif
GDK_mem_bigsize = 1024*1024;
GDKremovedir(DELDIR);
BBPinit();
HEAPcacheInit();
GDKkey = BATnew(TYPE_void, TYPE_str, 100);
GDKval = BATnew(TYPE_void, TYPE_str, 100);
if (GDKkey == NULL)
GDKfatal("GDKinit: Could not create environment BAT");
if (GDKval == NULL)
GDKfatal("GDKinit: Could not create environment BAT");
BATseqbase(GDKkey,0);
BATkey(GDKkey, BOUND2BTRUE);
BATrename(GDKkey, "environment_key");
BATmode(GDKkey, TRANSIENT);
BATseqbase(GDKval,0);
BATkey(GDKval, BOUND2BTRUE);
BATrename(GDKval, "environment_val");
BATmode(GDKval, TRANSIENT);
n = (opt *) malloc(setlen * sizeof(opt));
for (i = 0; i < setlen; i++) {
int done = 0;
for (j = 0; j < nlen; j++) {
if (strcmp(n[j].name, set[i].name) == 0) {
if (n[j].kind < set[i].kind) {
n[j] = set[i];
}
done = 1;
break;
}
}
if (!done) {
n[nlen] = set[i];
nlen++;
}
}
for (i = 0; i < nlen; i++)
GDKsetenv(n[i].name, n[i].value);
free(n);
if ((p = GDKgetenv("gdk_dbpath")) != NULL &&
(p = strrchr(p, DIR_SEP)) != NULL) {
GDKsetenv("gdk_dbname", p + 1);
#if DIR_SEP != '/' /* on Windows look for different separator */
} else if ((p = GDKgetenv("gdk_dbpath")) != NULL &&
(p = strrchr(p, '/')) != NULL) {
GDKsetenv("gdk_dbname", p + 1);
#endif
}
if ((p = GDKgetenv("gdk_mem_maxsize"))) {
GDK_mem_maxsize = MAX(1 << 26, (size_t) strtoll(p, NULL, 10));
}
if ((p = GDKgetenv("gdk_vm_maxsize"))) {
GDK_vm_maxsize = MAX(1 << 30, (size_t) strtoll(p, NULL, 10));
}
if ((p = GDKgetenv("gdk_mem_bigsize"))) {
/* when allocating >6% of all RAM; do so using
* vmalloc() iso malloc() */
lng max_mem_bigsize = GDK_mem_maxsize_max / 16;
/* sanity check to avoid memory fragmentation */
GDK_mem_bigsize = (size_t) MIN(max_mem_bigsize, strtoll(p, NULL, 10));
}
if ((p = GDKgetenv("gdk_mmap_minsize"))) {
GDK_mmap_minsize = MAX(REMAP_PAGE_MAXSIZE, (size_t) strtoll(p, NULL, 10));
}
if (GDKgetenv("gdk_mem_pagebits") == NULL) {
snprintf(buf, sizeof(buf), "%d", GDK_mem_pagebits);
GDKsetenv("gdk_mem_pagebits", buf);
}
if (GDKgetenv("gdk_mem_bigsize") == NULL) {
snprintf(buf, sizeof(buf), SZFMT, GDK_mem_bigsize);
GDKsetenv("gdk_mem_bigsize", buf);
}
if (GDKgetenv("monet_pid") == NULL) {
snprintf(buf, sizeof(buf), "%d", (int) getpid());
GDKsetenv("monet_pid", buf);
}
GDKnr_threads = GDKgetenv_int("gdk_nr_threads", 0);
if (GDKnr_threads == 0)
GDKnr_threads = MT_check_nr_cores();
#ifdef NATIVE_WIN32
GDK_mmap_minsize /= (GDKnr_threads ? GDKnr_threads : 1);
#else
/* WARNING: This unconditionally overwrites above settings, */
/* incl. setting via MonetDB env. var. "gdk_mmap_minsize" ! */
GDK_mmap_minsize = MIN( 1<<30 , (GDK_mem_maxsize_max/6) / (GDKnr_threads ? GDKnr_threads : 1) );
/* per op: 2 args + 1 res, each with head & tail => (2+1)*2 = 6 ^ */
#endif
if ((p = mo_find_option(set, setlen, "gdk_vmtrim")) == NULL ||
strcasecmp(p, "yes") == 0)
MT_create_thread(&GDKvmtrim_id, GDKvmtrim, &GDK_mem_maxsize,
MT_THR_JOINABLE);
return 1;
}
/*
* Collect some global system properties to relate performance results later
*/
static void
monet_hello(void)
{
dbl sz_mem_h;
char *qc = " kMGTPE";
int qi = 0;
monet_memory = MT_npages() * MT_pagesize();
sz_mem_h = (dbl) monet_memory;
while (sz_mem_h >= 1000.0 && qi < 6) {
sz_mem_h /= 1024.0;
qi++;
}
printf("# MonetDB 5 server v%s", GDKversion());
{
#ifdef MONETDB_RELEASE
printf(" (%s)", MONETDB_RELEASE);
#else
const char *rev = mercurial_revision();
if (strcmp(rev, "Unknown") != 0)
printf(" (hg id: %s)", rev);
#endif
}
#ifndef MONETDB_RELEASE
printf("\n# This is an unreleased version");
#endif
printf("\n# Serving database '%s', using %d thread%s\n",
GDKgetenv("gdk_dbname"),
GDKnr_threads, (GDKnr_threads != 1) ? "s" : "");
printf("# Compiled for %s/%zubit%s\n",
HOST, sizeof(ptr) * 8,
#ifdef HAVE_HGE
" with 128bit integers"
#else
""
#endif
);
printf("# Found %.3f %ciB available main-memory.\n",
sz_mem_h, qc[qi]);
#ifdef MONET_GLOBAL_DEBUG
printf("# Database path:%s\n", GDKgetenv("gdk_dbpath"));
printf("# Module path:%s\n", GDKgetenv("monet_mod_path"));
#endif
printf("# Copyright (c) 1993 - July 2008 CWI.\n");
printf("# Copyright (c) August 2008 - 2019 MonetDB B.V., all rights reserved\n");
printf("# Visit https://www.monetdb.org/ for further information\n");
// The properties shipped through the performance profiler
(void) snprintf(monet_characteristics, sizeof(monet_characteristics),
"{\n"
"\"version\":\"%s\",\n"
"\"release\":\"%s\",\n"
"\"host\":\"%s\",\n"
"\"threads\":\"%d\",\n"
"\"memory\":\"%.3f %cB\",\n"
"\"oid\":\"%zu\",\n"
"\"packages\":["
#ifdef HAVE_HGE
"\"huge\""
#endif
"]\n}",
GDKversion(),
#ifdef MONETDB_RELEASE
MONETDB_RELEASE,
#else
"unreleased",
#endif
HOST, GDKnr_threads,
sz_mem_h, qc[qi], sizeof(oid) * 8);
}
str
CMDgetPageSize(int *ret)
{
*ret= (int) MT_pagesize();
return MAL_SUCCEED;
}
int mal_init(void){
#ifdef NEED_MT_LOCK_INIT
MT_lock_init( &mal_contextLock, "mal_contextLock");
MT_lock_init( &mal_namespaceLock, "mal_namespaceLock");
MT_lock_init( &mal_remoteLock, "mal_remoteLock");
MT_lock_init( &mal_profileLock, "mal_profileLock");
MT_lock_init( &mal_copyLock, "mal_copyLock");
MT_lock_init( &mal_delayLock, "mal_delayLock");
#endif
/* "/2" is arbitrarily used / chosen, as on systems with
* hyper-threading enabled, using all hardware threads rather than
* "only" all physical cores does not necessarily yield a linear
* performance benefit */
MT_sema_init( &mal_parallelism, (GDKnr_threads > 1 ? GDKnr_threads/2: 1), "mal_parallelism");
tstAligned();
MCinit();
if (mdbInit())
return -1;
if (monet_memory == 0)
monet_memory = MT_npages() * MT_pagesize();
initNamespace();
initParser();
initHeartbeat();
initResource();
#ifdef HAVE_JSONSTORE
startHttpdaemon();
#endif
RECYCLEinit();
if( malBootstrap() == 0)
return -1;
/* set up the profiler if needed, output sent to console */
/* Use the same shortcuts as stethoscope */
if ( mal_trace && *mal_trace) {
char *s;
setFilterAll();
openProfilerStream(mal_clients[0].fdout);
for ( s= mal_trace; *s; s++)
switch(*s){
case 'a': activateCounter("aggregate");break;
case 'b': activateCounter("rbytes");
activateCounter("wbytes");break;
case 'c': activateCounter("cpu");break;
case 'e': activateCounter("event");break;
case 'f': activateCounter("function");break;
case 'i': activateCounter("pc");break;
case 'm': activateCounter("memory");break;
case 'p': activateCounter("process");break;
case 'r': activateCounter("reads");break;
case 's': activateCounter("stmt");break;
case 't': activateCounter("ticks");break;
case 'u': activateCounter("user");break;
case 'w': activateCounter("writes");break;
case 'y': activateCounter("type");break;
case 'D': activateCounter("dot");break;
case 'I': activateCounter("thread");break;
case 'T': activateCounter("time");break;
case 'S': activateCounter("start");
}
startProfiling();
} else mal_trace =0;
return 0;
}