static str
SQLinit(void)
{
char *debug_str = GDKgetenv("sql_debug"), *msg = MAL_SUCCEED;
int readonly = GDKgetenv_isyes("gdk_readonly");
int single_user = GDKgetenv_isyes("gdk_single_user");
const char *gmt = "GMT";
tzone tz;
#ifdef _SQL_SCENARIO_DEBUG
mnstr_printf(GDKout, "#SQLinit Monet 5\n");
#endif
if (SQLinitialized)
return MAL_SUCCEED;
#ifdef NEED_MT_LOCK_INIT
MT_lock_init(&sql_contextLock, "sql_contextLock");
#endif
MT_lock_set(&sql_contextLock);
memset((char *) &be_funcs, 0, sizeof(backend_functions));
be_funcs.fstack = &monet5_freestack;
be_funcs.fcode = &monet5_freecode;
be_funcs.fresolve_function = &monet5_resolve_function;
monet5_user_init(&be_funcs);
msg = MTIMEtimezone(&tz, &gmt);
if (msg)
return msg;
(void) tz;
if (debug_str)
SQLdebug = strtol(debug_str, NULL, 10);
if (single_user)
SQLdebug |= 64;
if (readonly)
SQLdebug |= 32;
if ((SQLnewcatalog = mvc_init(SQLdebug, store_bat, readonly, single_user, 0)) < 0)
throw(SQL, "SQLinit", "Catalogue initialization failed");
SQLinitialized = TRUE;
MT_lock_unset(&sql_contextLock);
if (MT_create_thread(&sqllogthread, (void (*)(void *)) mvc_logmanager, NULL, MT_THR_DETACHED) != 0) {
throw(SQL, "SQLinit", "Starting log manager failed");
}
#if 0
if (MT_create_thread(&minmaxthread, (void (*)(void *)) mvc_minmaxmanager, NULL, MT_THR_DETACHED) != 0) {
throw(SQL, "SQLinit", "Starting minmax manager failed");
}
#endif
return MAL_SUCCEED;
}
/*
* Create an interpreter pool.
* One worker will adaptively be available for each client.
* The remainder are taken from the GDKnr_threads argument and
* typically is equal to the number of cores
* The workers are assembled in a local table to enable debugging.
*/
static int
DFLOWinitialize(void)
{
int i, limit;
int created = 0;
MT_lock_set(&mal_contextLock, "DFLOWinitialize");
if (todo) {
/* somebody else beat us to it */
MT_lock_unset(&mal_contextLock, "DFLOWinitialize");
return 0;
}
todo = q_create(2048, "todo");
if (todo == NULL) {
MT_lock_unset(&mal_contextLock, "DFLOWinitialize");
return -1;
}
for (i = 0; i < THREADS; i++)
MT_sema_init(&workers[i].s, 0, "DFLOWinitialize");
limit = GDKnr_threads ? GDKnr_threads - 1 : 0;
#ifdef NEED_MT_LOCK_INIT
ATOMIC_INIT(exitingLock, "exitingLock");
MT_lock_init(&dataflowLock, "dataflowLock");
#endif
MT_lock_set(&dataflowLock, "DFLOWinitialize");
for (i = 0; i < limit; i++) {
workers[i].flag = RUNNING;
workers[i].cntxt = NULL;
if (MT_create_thread(&workers[i].id, DFLOWworker, (void *) &workers[i], MT_THR_JOINABLE) < 0)
workers[i].flag = IDLE;
else
created++;
}
MT_lock_unset(&dataflowLock, "DFLOWinitialize");
if (created == 0) {
/* no threads created */
q_destroy(todo);
todo = NULL;
MT_lock_unset(&mal_contextLock, "DFLOWinitialize");
return -1;
}
MT_lock_unset(&mal_contextLock, "DFLOWinitialize");
return 0;
}
int
GDKinit(opt *set, int setlen)
{
char *dbpath = mo_find_option(set, setlen, "gdk_dbpath");
char *p;
opt *n;
int i, j, nlen = 0;
char buf[16];
/* some sanity checks (should also find if symbols are not defined) */
assert(sizeof(char) == SIZEOF_CHAR);
assert(sizeof(short) == SIZEOF_SHORT);
assert(sizeof(int) == SIZEOF_INT);
assert(sizeof(long) == SIZEOF_LONG);
assert(sizeof(lng) == SIZEOF_LNG);
assert(sizeof(oid) == SIZEOF_OID);
assert(sizeof(void *) == SIZEOF_VOID_P);
assert(sizeof(wrd) == SIZEOF_WRD);
assert(sizeof(size_t) == SIZEOF_SIZE_T);
assert(sizeof(ptrdiff_t) == SIZEOF_PTRDIFF_T);
assert(SIZEOF_OID == SIZEOF_INT || SIZEOF_OID == SIZEOF_LNG);
#ifdef NEED_MT_LOCK_INIT
MT_lock_init(&MT_system_lock,"MT_system_lock");
ATOMIC_INIT(GDKstoppedLock, "GDKstoppedLock");
ATOMIC_INIT(mbyteslock, "mbyteslock");
MT_lock_init(&GDKnameLock, "GDKnameLock");
MT_lock_init(&GDKthreadLock, "GDKthreadLock");
MT_lock_init(&GDKtmLock, "GDKtmLock");
#endif
for (i = 0; i <= BBP_BATMASK; i++) {
MT_lock_init(&GDKbatLock[i].swap, "GDKswapLock");
MT_lock_init(&GDKbatLock[i].hash, "GDKhashLock");
MT_lock_init(&GDKbatLock[i].imprints, "GDKimprintsLock");
}
for (i = 0; i <= BBP_THREADMASK; i++) {
MT_lock_init(&GDKbbpLock[i].alloc, "GDKcacheLock");
MT_lock_init(&GDKbbpLock[i].trim, "GDKtrimLock");
GDKbbpLock[i].free = 0;
}
errno = 0;
if (!GDKenvironment(dbpath))
return 0;
if ((p = mo_find_option(set, setlen, "gdk_debug")))
GDKdebug = strtol(p, NULL, 10);
if ((p = mo_find_option(set, setlen, "gdk_mem_pagebits")))
GDK_mem_pagebits = (int) strtol(p, NULL, 10);
mnstr_init();
MT_init_posix();
THRinit();
#ifndef NATIVE_WIN32
BATSIGinit();
#endif
#ifdef WIN32
(void) signal(SIGABRT, BATSIGabort);
_set_abort_behavior(0, _CALL_REPORTFAULT | _WRITE_ABORT_MSG);
_set_error_mode(_OUT_TO_STDERR);
#endif
GDKlockHome();
/* Mserver by default takes 80% of all memory as a default */
GDK_mem_maxsize = GDK_mem_maxsize_max = (size_t) ((double) MT_npages() * (double) MT_pagesize() * 0.815);
#ifdef NATIVE_WIN32
GDK_mmap_minsize = GDK_mem_maxsize_max;
#else
GDK_mmap_minsize = MIN( 1<<30 , GDK_mem_maxsize_max/6 );
/* per op: 2 args + 1 res, each with head & tail => (2+1)*2 = 6 ^ */
#endif
GDK_mem_bigsize = 1024*1024;
GDKremovedir(DELDIR);
BBPinit();
HEAPcacheInit();
GDKkey = BATnew(TYPE_void, TYPE_str, 100);
GDKval = BATnew(TYPE_void, TYPE_str, 100);
if (GDKkey == NULL)
GDKfatal("GDKinit: Could not create environment BAT");
if (GDKval == NULL)
GDKfatal("GDKinit: Could not create environment BAT");
BATseqbase(GDKkey,0);
BATkey(GDKkey, BOUND2BTRUE);
BATrename(GDKkey, "environment_key");
BATmode(GDKkey, TRANSIENT);
BATseqbase(GDKval,0);
BATkey(GDKval, BOUND2BTRUE);
BATrename(GDKval, "environment_val");
BATmode(GDKval, TRANSIENT);
n = (opt *) malloc(setlen * sizeof(opt));
for (i = 0; i < setlen; i++) {
int done = 0;
for (j = 0; j < nlen; j++) {
if (strcmp(n[j].name, set[i].name) == 0) {
if (n[j].kind < set[i].kind) {
n[j] = set[i];
}
done = 1;
break;
}
}
if (!done) {
n[nlen] = set[i];
nlen++;
}
}
for (i = 0; i < nlen; i++)
GDKsetenv(n[i].name, n[i].value);
free(n);
if ((p = GDKgetenv("gdk_dbpath")) != NULL &&
(p = strrchr(p, DIR_SEP)) != NULL) {
GDKsetenv("gdk_dbname", p + 1);
#if DIR_SEP != '/' /* on Windows look for different separator */
} else if ((p = GDKgetenv("gdk_dbpath")) != NULL &&
(p = strrchr(p, '/')) != NULL) {
GDKsetenv("gdk_dbname", p + 1);
#endif
}
if ((p = GDKgetenv("gdk_mem_maxsize"))) {
GDK_mem_maxsize = MAX(1 << 26, (size_t) strtoll(p, NULL, 10));
}
if ((p = GDKgetenv("gdk_vm_maxsize"))) {
GDK_vm_maxsize = MAX(1 << 30, (size_t) strtoll(p, NULL, 10));
}
if ((p = GDKgetenv("gdk_mem_bigsize"))) {
/* when allocating >6% of all RAM; do so using
* vmalloc() iso malloc() */
lng max_mem_bigsize = GDK_mem_maxsize_max / 16;
/* sanity check to avoid memory fragmentation */
GDK_mem_bigsize = (size_t) MIN(max_mem_bigsize, strtoll(p, NULL, 10));
}
if ((p = GDKgetenv("gdk_mmap_minsize"))) {
GDK_mmap_minsize = MAX(REMAP_PAGE_MAXSIZE, (size_t) strtoll(p, NULL, 10));
}
if (GDKgetenv("gdk_mem_pagebits") == NULL) {
snprintf(buf, sizeof(buf), "%d", GDK_mem_pagebits);
GDKsetenv("gdk_mem_pagebits", buf);
}
if (GDKgetenv("gdk_mem_bigsize") == NULL) {
snprintf(buf, sizeof(buf), SZFMT, GDK_mem_bigsize);
GDKsetenv("gdk_mem_bigsize", buf);
}
if (GDKgetenv("monet_pid") == NULL) {
snprintf(buf, sizeof(buf), "%d", (int) getpid());
GDKsetenv("monet_pid", buf);
}
GDKnr_threads = GDKgetenv_int("gdk_nr_threads", 0);
if (GDKnr_threads == 0)
GDKnr_threads = MT_check_nr_cores();
#ifdef NATIVE_WIN32
GDK_mmap_minsize /= (GDKnr_threads ? GDKnr_threads : 1);
#else
/* WARNING: This unconditionally overwrites above settings, */
/* incl. setting via MonetDB env. var. "gdk_mmap_minsize" ! */
GDK_mmap_minsize = MIN( 1<<30 , (GDK_mem_maxsize_max/6) / (GDKnr_threads ? GDKnr_threads : 1) );
/* per op: 2 args + 1 res, each with head & tail => (2+1)*2 = 6 ^ */
#endif
if ((p = mo_find_option(set, setlen, "gdk_vmtrim")) == NULL ||
strcasecmp(p, "yes") == 0)
MT_create_thread(&GDKvmtrim_id, GDKvmtrim, &GDK_mem_maxsize,
MT_THR_JOINABLE);
return 1;
}
/**
* Generic function that loads a given list of BAM files according to
* the values of the other parameters. It starts with creating the
* bam schema and the header tables if these do not exist yet. Then
* it initializes bam_wrapper structs for all bam files and reads the
* headers for all BAM files. If the pairwise storage schema has to
* be used, It then creates a thread for every file that reads all
* alignments for this file.
*
*/
static str
bam_loader(Client cntxt, MalBlkPtr mb, str * filenames, int nr_files,
sht dbschema, sht nr_threads)
{
bam_wrapper *bws = NULL;
MT_Id *reader_threads = NULL;
reader_thread_data *r_thread_data = NULL;
mvc *m = NULL;
sql_schema *s = NULL;
sql_table *files_table = NULL;
lng cur_file_id;
char buf_threads_msg[4096] = "There were reader threads that contained errors:\n";
int threads_msg_len = strlen(buf_threads_msg);
int i, errnr;
str msg = MAL_SUCCEED;
TO_LOG("<bam_loader>: Loader started for %d BAM file%s...\n",
nr_files, (nr_files != 1 ? "s" : ""));
/* Check sanity of input */
if (dbschema != 0 && dbschema != 1) {
msg = createException(MAL, "bam_loader",
"Wrong value for dbschema: '%d' (0=straightforward storage schema, 1=pairwise storage schema)",
dbschema);
goto cleanup;
}
if (nr_threads <= 0) {
nr_threads = 1;
} else if(nr_threads > 4) {
nr_threads = 4;
}
/* Get SQL context */
if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED) {
/* Here, and in multiple other locations in this code,
* new message is stored in tmp var, since the old msg
* needs to be freed after construction of the new
* msg */
REUSE_EXCEPTION(msg, MAL, "bam_loader",
"Could not retrieve SQLContext: %s", msg);
goto cleanup;
}
/* Start with binding bam schema and the files table */
if ((msg =
bind_bam_schema(m, &s)) != MAL_SUCCEED)
goto cleanup;
if((msg =
bind_table(m, s, "files", &files_table)) != MAL_SUCCEED)
goto cleanup;
/* Get next file id from files table */
TO_LOG("<bam_loader> Retrieving next file id...\n");
if ((msg = next_file_id(m, files_table, &cur_file_id)) != MAL_SUCCEED) {
goto cleanup;
}
/* Init bam_wrapper structs */
if ((bws =
(bam_wrapper *) GDKmalloc(nr_files * sizeof(bam_wrapper))) ==
NULL) {
msg = createException(MAL, "bam_loader", MAL_MALLOC_FAIL);
goto cleanup;
}
/* Enables cleanup to check which bam_wrappers to clear */
memset(bws, 0, nr_files * sizeof(bam_wrapper));
for (i = 0; i < nr_files; ++i) {
int fln = strlen(filenames[i]);
TO_LOG("<bam_loader> Initializing BAM wrapper for file '%s'...\n", filenames[i]);
if ((msg =
init_bam_wrapper(bws + i, (IS_BAM(filenames[i], fln) ? BAM : SAM),
filenames[i], cur_file_id++, dbschema)) != MAL_SUCCEED) {
goto cleanup;
}
}
/* Parse all headers */
for (i = 0; i < nr_files; ++i) {
TO_LOG("<bam_loader> Parsing header for file '%s'...\n",
filenames[i]);
if ((msg = process_header(bws + i)) != MAL_SUCCEED) {
goto cleanup;
}
}
/* If we have to load the BAM data into the pairwise storage
* schema, make sure that all input BAM files are sorted on
* QNAME */
if (dbschema == 1) {
for (i = 0; i < nr_files; ++i) {
TO_LOG("<bam_loader> Checking sortedness for BAM file '%s'...\n", filenames[i]);
if (bws[i].ord != ORDERING_QUERYNAME) {
msg = createException(MAL, "bam_loader",
"Only BAM files that are sorted on queryname can be inserted into the pairwise storage schema; "
"BAM file '%s' has ordering '%s'",
bws[i].file_location,
ordering_str(bws[i].
ord));
goto cleanup;
}
}
}
/* Create alignment storage */
for (i = 0; i < nr_files; ++i) {
TO_LOG("<bam_loader> Creating alignment tables for file '%s'...\n", filenames[i]);
if ((dbschema == 0
&& (msg = create_alignment_storage_0(cntxt,
"bam.create_storage_0",
bws + i)) != MAL_SUCCEED)
|| (dbschema == 1
&& (msg = create_alignment_storage_1(cntxt,
"bam.create_storage_1",
bws + i)) != MAL_SUCCEED)) {
goto cleanup;
}
}
/* Now create threads to read alignment data of different files */
TO_LOG("<bam_loader> Creating reader threads...\n");
if ((reader_threads =
(MT_Id *) GDKmalloc(nr_threads * sizeof(MT_Id))) == NULL) {
msg = createException(MAL, "bam_loader", MAL_MALLOC_FAIL);
goto cleanup;
}
if ((r_thread_data =
create_reader_thread_data(bws, nr_files, nr_threads)) == NULL) {
msg = createException(MAL, "bam_loader", MAL_MALLOC_FAIL);
goto cleanup;
}
for (i = 0; i < nr_threads; ++i) {
if ((errnr =
MT_create_thread(&reader_threads[i],
run_process_bam_alignments,
&r_thread_data[i],
MT_THR_JOINABLE)) != 0) {
msg = createException(MAL, "bam_loader",
"Could not create thread to process alignments (errnr %d)",
errnr);
goto cleanup;
}
}
TO_LOG("<bam_loader> Waiting for reader threads to finish...\n");
/* Wait until all threads finish and collect their
* messages. Though it is not very likely, it could be the
* case that more than 1 thread generates an error message (not
* likely because threads exit once they notice that another
* thread has failed). Therefore, we collect all error
* messages in one big error string
*/
for (i = 0; i < nr_threads; ++i) {
if ((errnr = MT_join_thread(reader_threads[i])) != 0) {
msg = createException(MAL, "bam_loader",
"Could not join alignment processing thread (errnr %d)",
errnr);
goto cleanup;
}
/* Thread finished ok, append its error message, if any */
if (r_thread_data[i].msg != MAL_SUCCEED) {
int step;
if (msg == MAL_SUCCEED) {
/* First encountered thread error,
* indicate this by pointing to error
* buf */
msg = buf_threads_msg;
}
/* snprintf returns -1 on failure; since we
* don't want to fail when snprintf fails, we
* use MAX to make sure we don't add a
* negative amount to threads_msg_len */
step = snprintf(msg + threads_msg_len,
4096 - threads_msg_len, "* %s\n",
r_thread_data[i].msg);
threads_msg_len += MAX(0, step);
GDKfree(r_thread_data[i].msg);
}
}
/* Fail if any thread has failed */
if (msg != MAL_SUCCEED) {
/* Do not use REUSE_EXCEPTION here, since msg was not
* malloced. Instead, just copy buffer contents to
* malloced buffer */
msg = GDKstrdup(msg);
goto cleanup;
}
TO_LOG("<bam_loader> Copying data into DB...\n");
/* All threads finished succesfully, copy all data into DB */
for (i = 0; i < nr_files; ++i) {
if ((msg = copy_into_db(cntxt, bws + i)) != MAL_SUCCEED) {
goto cleanup;
}
}
cleanup:
if (bws) {
for (i = 0; i < nr_files; ++i) {
if (bws + i)
clear_bam_wrapper(bws + i);
}
GDKfree(bws);
}
if (reader_threads)
GDKfree(reader_threads);
if (r_thread_data)
destroy_reader_thread_data(r_thread_data);
if (msg != MAL_SUCCEED) {
TO_LOG("<bam_loader> Error on processing BAM files: %s\n",
msg);
}
TO_LOG("<bam_loader>: Loader finished processing %d BAM file%s...\n",
nr_files, (nr_files != 1 ? "s" : ""));
return msg;
}
int main(int argc, char **argv)
{
MT_Id pid;
int i, j = 0;
char *err = NULL;
char name[MYBUFSIZ + 1];
char hostname[1024];
Sensor se = NULL;
static SOCKET sockfd;
static struct option long_options[18] = {
{ "increment", 0, 0, 'i' },
{ "batch", 1, 0, 'b' },
{ "columns", 1, 0, 'c' },
{ "client", 0, 0, 'c' },
{ "port", 1, 0, 'p' },
{ "protocol", 1, 0, 'p' },
{ "timestamp", 0, 0, 't' },
{ "time", 1, 0, 't' },
{ "events", 1, 0, 'e' },
{ "sensor", 1, 0, 's' },
{ "server", 0, 0, 's' },
{ "replay", 0, 0, 'r' },
{ "delay", 1, 0, 'd' },
{ "file", 1, 0, 'f' },
{ "host", 1, 0, 'h' },
{ "trace", 0, 0, 't' },
{ "help", 1, 0, '?' },
{ 0, 0, 0, 0 }
};
THRdata[0] = (void *) file_wastream(stdout, "stdout");
THRdata[1] = (void *) file_rastream(stdin, "stdin");
for (i = 0; i < THREADS; i++) {
GDKthreads[i].tid = i + 1;
}
for (;;) {
int option_index = 0;
int c = getopt_long(argc, argv, "i:b:c:c:p:p:t:t:e:s:s:r:d:f:h:t:?:0",
long_options, &option_index);
if (c == -1)
break;
switch (c) {
case 'b':
batchsize = optarg ? atol(optarg) : -1;
if (batchsize <= 0) {
mnstr_printf(SEout, "Illegal batch %d\n", batchsize);
exit(0);
}
break;
case 'c':
if (strcmp(long_options[option_index].name, "client") == 0) {
server = 0;
break;
}
columns = optarg ? atol(optarg) : -1;
if (columns <= 0) {
mnstr_printf(SEout, "Illegal columns %d\n", columns);
exit(0);
}
break;
case 'd':
delay = optarg ? atol(optarg) : -1;
if (delay < 0) {
mnstr_printf(SEout, "Illegal delay %d\n", delay);
exit(0);
}
break;
case 'i':
autoincrement = optarg ? atol(optarg) : 0;
if (autoincrement < 0) {
mnstr_printf(SEout, "Illegal increment %d\n", autoincrement);
exit(0);
}
break;
case 't':
if (strcmp(long_options[option_index].name, "timestamp") == 0) {
timestamp = 1;
break;
}
if (strcmp(long_options[option_index].name, "time") == 0) {
timecolumn = optarg ? atol(optarg) : 0;
break;
}
if (strcmp(long_options[option_index].name, "trace") == 0) {
trace = optarg ? atol(optarg) : 1;
} else {
usage();
exit(0);
}
break;
case 'f':
datafile = optarg && *optarg? optarg:0;
break;
case 'e':
if (strcmp(long_options[option_index].name, "events") == 0) {
events = optarg ? atol(optarg) : -1;
if (events < -1) {
mnstr_printf(SEout, "illegal events value, reset to -1\n");
events = -1;
} else if (events == 0) {
mnstr_printf(SEout, "Illegal events value %d\n", events);
exit(0);
}
break;
} else {
usage();
exit(0);
}
break;
case 'r':
replay= 1;
break;
case 's':
if (strcmp(long_options[option_index].name, "sensor") == 0) {
sensor = optarg;
break;
}
if (strcmp(long_options[option_index].name, "server") == 0) {
server = 1;
break;
} else {
usage();
exit(0);
}
break;
case 'p':
if (strcmp(long_options[option_index].name, "protocol") == 0) {
char *name= optarg? optarg: "xyz";
if (strcmp(name, "TCP") == 0 || strcmp(name, "tcp") == 0) {
protocol = TCP;
break;
}
if (strcmp(name, "UDP") == 0 || strcmp(name, "udp") == 0) {
protocol = UDP;
break;
}
if (strcmp(name, "CSV") == 0 || strcmp(name, "csv") == 0) {
protocol = CSV;
break;
}
if (strcmp(name, "debug") == 0) {
protocol = DEB;
break;
}
}
if (strcmp(long_options[option_index].name, "port") == 0) {
port = optarg ? atol(optarg) : -1;
#ifdef SENSOR_DEBUG
mnstr_printf(SEout, "#PORT : %d\n", port);
#endif
break;
} else {
usage();
exit(0);
}
break;
case 'h':
host = optarg;
break;
case '?':
default:
usage();
exit(0);
}
}
signal(SIGABRT, stopSend);
#ifdef SIGPIPE
signal(SIGPIPE, stopSend);
#endif
#ifdef SIGHUP
signal(SIGHUP, stopSend);
#endif
signal(SIGTERM, stopSend);
signal(SIGINT, stopSend);
/* display properties */
if (trace) {
mnstr_printf(SEout, "--host=%s\n", host);
mnstr_printf(SEout, "--port=%d\n", port);
mnstr_printf(SEout, "--sensor=%s\n", sensor);
mnstr_printf(SEout, "--columns=%d\n", columns);
mnstr_printf(SEout, "--autoincrement=%d\n", autoincrement);
mnstr_printf(SEout, "--timestamp=%d\n", timestamp);
mnstr_printf(SEout, "--time=%d\n", timecolumn);
mnstr_printf(SEout, "--events=%d\n", events);
mnstr_printf(SEout, "--batch=%d\n", batchsize);
mnstr_printf(SEout, "--replay=%d\n", replay);
mnstr_printf(SEout, "--delay=%d\n", delay);
mnstr_printf(SEout, "--protocol %s\n", protocolname[protocol]);
mnstr_printf(SEout, "--trace=%d\n", trace);
mnstr_printf(SEout, "--server=%d\n", server);
mnstr_printf(SEout, "--client=%d\n", server);
if (datafile)
mnstr_printf(SEout, "--input=%s\n", datafile);
}
estimateOverhead();
strncpy(hostname, host, 1024);
if (strcmp(host, "localhost") == 0)
gethostname(hostname, 1024);
host = hostname;
/*
* We limit the protocols for the time being to what can be
* considered a safe method.
*/
if (protocol == DEB) {
/* save event stream in a file */
Sensor se = SEnew(sensor);
if (events == -1 || batchsize != 1) {
printf("Provide an event limit using --events=<nr> and --batch=1\n");
return 0;
}
if (datafile)
se->toServer = open_wastream(datafile);
else
se->toServer = file_wastream(stdout, "stdout");
produceStream(se);
}
if (protocol == UDP) {
Sensor se = SEnew(sensor);
se->toServer = udp_wastream(host, port, sensor);
if (se->toServer == NULL) {
perror("Sensor: Could not open stream");
mnstr_printf(SEout, "#stream %s.%d.%s\n", host, port, sensor);
return 0;
}
produceStream(se);
}
if (protocol == TCP) {
if (server && (err = socket_server_connect(&sockfd, port))) {
mnstr_printf(SEout, "#SENSOR:start server:%s\n", err);
return 0;
}
do {
int createThread = 0;
snprintf(name, MYBUFSIZ - (strlen(sensor) + sizeof(j)), "%s%d", sensor, j++);
se = SEnew(name);
name[0] = 0;
err = NULL;
if (server) {
#ifdef SENSOR_DEBUG
mnstr_printf(SEout, "#listen %s as server is %d \n", se->name, server);
#endif
err = socket_server_listen(sockfd, &(se->newsockfd));
if (err) {
mnstr_printf(SEout, "#SENSOR:server listen:%s\n", err);
break;
}
} else {
#ifdef SENSOR_DEBUG
mnstr_printf(SEout, "#%s is client \n", se->name);
#endif
err = socket_client_connect(&(se->newsockfd), host, port);
if (err) {
mnstr_printf(SEout, "#SENSOR:client start:%s\n", err);
break;
}
}
se->toServer = socket_wastream(se->newsockfd, se->name);
if (se->toServer == NULL) {
perror("Sensor: Could not open stream");
mnstr_printf(SEout, "#stream %s.%d.%s\n", host, port, sensor);
socket_close(se->newsockfd);
return 0;
}
if (server) {
createThread = MT_create_thread(&pid, (void (*)(void *))produceServerStream, se, MT_THR_DETACHED);
#ifdef SENSOR_DEBUG
if (createThread)
mnstr_printf(SEout, "#Create thread is : %d \n", createThread);
#else
(void) createThread;
#endif
} else { /* client part */
produceServerStream(se);
}
} while (server);
if (server)
socket_close(sockfd);
if (se)
shutdown(se->newsockfd, SHUT_RDWR);
}
return 0;
}
/* We create a pool of GDKnr_threads-1 generic workers, that is,
* workers that will take on jobs from any clients. In addition, we
* create a single specific worker per client (i.e. each time we enter
* here). This specific worker will only do work for the client for
* which it was started. In this way we can guarantee that there will
* always be progress for the client, even if all other workers are
* doing something big.
*
* When all jobs for a client have been done (there are no more
* entries for the client in the queue), the specific worker turns
* itself into a generic worker. At the same time, we signal that one
* generic worker should exit and this function returns. In this way
* we make sure that there are once again GDKnr_threads-1 generic
* workers. */
str
runMALdataflow(Client cntxt, MalBlkPtr mb, int startpc, int stoppc, MalStkPtr stk)
{
DataFlow flow = NULL;
str msg = MAL_SUCCEED;
int size;
int *ret;
int i;
#ifdef DEBUG_FLOW
fprintf(stderr, "#runMALdataflow for block %d - %d\n", startpc, stoppc);
printFunction(GDKstdout, mb, 0, LIST_MAL_STMT | LIST_MAPI);
#endif
/* in debugging mode we should not start multiple threads */
if (stk == NULL)
throw(MAL, "dataflow", "runMALdataflow(): Called with stk == NULL");
ret = (int*) getArgReference(stk,getInstrPtr(mb,startpc),0);
*ret = FALSE;
if (stk->cmd) {
*ret = TRUE;
return MAL_SUCCEED;
}
assert(stoppc > startpc);
/* check existence of workers */
if (todo == NULL) {
/* create thread pool */
if (GDKnr_threads <= 1 || DFLOWinitialize() < 0) {
/* no threads created, run serially */
*ret = TRUE;
return MAL_SUCCEED;
}
i = THREADS; /* we didn't create an extra thread */
}
assert(todo);
/* in addition, create one more worker that will only execute
* tasks for the current client to compensate for our waiting
* until all work is done */
MT_lock_set(&dataflowLock, "runMALdataflow");
/* join with already exited threads */
{
int joined;
do {
joined = 0;
for (i = 0; i < THREADS; i++) {
if (workers[i].flag == EXITED) {
workers[i].flag = JOINING;
workers[i].cntxt = NULL;
joined = 1;
MT_lock_unset(&dataflowLock, "runMALdataflow");
MT_join_thread(workers[i].id);
MT_lock_set(&dataflowLock, "runMALdataflow");
workers[i].flag = IDLE;
}
}
} while (joined);
}
for (i = 0; i < THREADS; i++) {
if (workers[i].flag == IDLE) {
/* only create specific worker if we are not doing a
* recursive call */
if (stk->calldepth > 1) {
int j;
MT_Id pid = MT_getpid();
/* doing a recursive call: copy specificity from
* current worker to new worker */
workers[i].cntxt = NULL;
for (j = 0; j < THREADS; j++) {
if (workers[j].flag == RUNNING && workers[j].id == pid) {
workers[i].cntxt = workers[j].cntxt;
break;
}
}
} else {
/* not doing a recursive call: create specific worker */
workers[i].cntxt = cntxt;
}
workers[i].flag = RUNNING;
if (MT_create_thread(&workers[i].id, DFLOWworker, (void *) &workers[i], MT_THR_JOINABLE) < 0) {
/* cannot start new thread, run serially */
*ret = TRUE;
workers[i].flag = IDLE;
MT_lock_unset(&dataflowLock, "runMALdataflow");
return MAL_SUCCEED;
}
break;
}
}
MT_lock_unset(&dataflowLock, "runMALdataflow");
if (i == THREADS) {
/* no empty thread slots found, run serially */
*ret = TRUE;
return MAL_SUCCEED;
}
flow = (DataFlow)GDKzalloc(sizeof(DataFlowRec));
if (flow == NULL)
throw(MAL, "dataflow", "runMALdataflow(): Failed to allocate flow");
flow->cntxt = cntxt;
flow->mb = mb;
flow->stk = stk;
flow->error = 0;
/* keep real block count, exclude brackets */
flow->start = startpc + 1;
flow->stop = stoppc;
MT_lock_init(&flow->flowlock, "flow->flowlock");
flow->done = q_create(stoppc- startpc+1, "flow->done");
if (flow->done == NULL) {
MT_lock_destroy(&flow->flowlock);
GDKfree(flow);
throw(MAL, "dataflow", "runMALdataflow(): Failed to create flow->done queue");
}
flow->status = (FlowEvent)GDKzalloc((stoppc - startpc + 1) * sizeof(FlowEventRec));
if (flow->status == NULL) {
q_destroy(flow->done);
MT_lock_destroy(&flow->flowlock);
GDKfree(flow);
throw(MAL, "dataflow", "runMALdataflow(): Failed to allocate flow->status");
}
size = DFLOWgraphSize(mb, startpc, stoppc);
size += stoppc - startpc;
flow->nodes = (int*)GDKzalloc(sizeof(int) * size);
if (flow->nodes == NULL) {
GDKfree(flow->status);
q_destroy(flow->done);
MT_lock_destroy(&flow->flowlock);
GDKfree(flow);
throw(MAL, "dataflow", "runMALdataflow(): Failed to allocate flow->nodes");
}
flow->edges = (int*)GDKzalloc(sizeof(int) * size);
if (flow->edges == NULL) {
GDKfree(flow->nodes);
GDKfree(flow->status);
q_destroy(flow->done);
MT_lock_destroy(&flow->flowlock);
GDKfree(flow);
throw(MAL, "dataflow", "runMALdataflow(): Failed to allocate flow->edges");
}
msg = DFLOWinitBlk(flow, mb, size);
if (msg == MAL_SUCCEED)
msg = DFLOWscheduler(flow, &workers[i]);
GDKfree(flow->status);
GDKfree(flow->edges);
GDKfree(flow->nodes);
q_destroy(flow->done);
MT_lock_destroy(&flow->flowlock);
GDKfree(flow);
if (i != THREADS) {
/* we created one worker, now tell one worker to exit again */
MT_lock_set(&todo->l, "runMALdataflow");
todo->exitcount++;
MT_lock_unset(&todo->l, "runMALdataflow");
MT_sema_up(&todo->s, "runMALdataflow");
}
return msg;
}
