static void QOTstatisticsInit(void){
oid o=0;
int i,j;
if (qotStat[QOTnames]) return;
#ifdef NEED_MT_LOCK_INIT
MT_lock_init(&qotlock,"QOT statistics");
#endif
MT_lock_set(&qotlock, "QOT statistics");
qotStat[QOTnames]= QOT_create("opt","names",TYPE_str);
BATseqbase(qotStat[QOTnames],o);
qotStat[QOTcalls]= QOT_create("opt","calls",TYPE_int);
BATseqbase(qotStat[QOTcalls],o);
qotStat[QOTactions]= QOT_create("opt","actions",TYPE_int);
BATseqbase(qotStat[QOTactions],o);
qotStat[QOTtimings]= QOT_create("opt","timings",TYPE_lng);
BATseqbase(qotStat[QOTtimings],o);
/* recover from errors */
for ( i=0; i<4; i++)
if ( qotStat[i] == NULL){
for (j= 0; j < i; j++)
BBPclear(qotStat[j]->batCacheid);
MT_lock_unset(&qotlock, "QOT statistics");
return;
}
MT_lock_unset(&qotlock, "QOT statistics");
/* save them at least once */
QOTstatisticsExit();
}
str
defaultScenario(Client c)
{
#ifdef NEED_MT_LOCK_INIT
if (c == mal_clients)
MT_lock_init(&scenarioLock, "scenarioLock");
#endif
return initScenario(c, scenarioRec);
}
list *
list_new(sql_allocator *sa, fdestroy destroy)
{
list *l = (sa)?SA_ZNEW(sa, list):ZNEW(list);
l->sa = sa;
l->destroy = destroy;
l->h = l->t = NULL;
l->cnt = 0;
l->ht = NULL;
MT_lock_init(&l->ht_lock, "sa_ht_lock");
return l;
}
list *
list_create(fdestroy destroy)
{
list *l = MNEW(list);
l->sa = NULL;
l->destroy = destroy;
l->h = l->t = NULL;
l->cnt = 0;
l->expected_cnt = 0;
l->ht = NULL;
MT_lock_init(&l->ht_lock, "sa_ht_lock");
return l;
}
static str
SQLinit(void)
{
char *debug_str = GDKgetenv("sql_debug"), *msg = MAL_SUCCEED;
int readonly = GDKgetenv_isyes("gdk_readonly");
int single_user = GDKgetenv_isyes("gdk_single_user");
const char *gmt = "GMT";
tzone tz;
#ifdef _SQL_SCENARIO_DEBUG
mnstr_printf(GDKout, "#SQLinit Monet 5\n");
#endif
if (SQLinitialized)
return MAL_SUCCEED;
#ifdef NEED_MT_LOCK_INIT
MT_lock_init(&sql_contextLock, "sql_contextLock");
#endif
MT_lock_set(&sql_contextLock);
memset((char *) &be_funcs, 0, sizeof(backend_functions));
be_funcs.fstack = &monet5_freestack;
be_funcs.fcode = &monet5_freecode;
be_funcs.fresolve_function = &monet5_resolve_function;
monet5_user_init(&be_funcs);
msg = MTIMEtimezone(&tz, &gmt);
if (msg)
return msg;
(void) tz;
if (debug_str)
SQLdebug = strtol(debug_str, NULL, 10);
if (single_user)
SQLdebug |= 64;
if (readonly)
SQLdebug |= 32;
if ((SQLnewcatalog = mvc_init(SQLdebug, store_bat, readonly, single_user, 0)) < 0)
throw(SQL, "SQLinit", "Catalogue initialization failed");
SQLinitialized = TRUE;
MT_lock_unset(&sql_contextLock);
if (MT_create_thread(&sqllogthread, (void (*)(void *)) mvc_logmanager, NULL, MT_THR_DETACHED) != 0) {
throw(SQL, "SQLinit", "Starting log manager failed");
}
#if 0
if (MT_create_thread(&minmaxthread, (void (*)(void *)) mvc_minmaxmanager, NULL, MT_THR_DETACHED) != 0) {
throw(SQL, "SQLinit", "Starting minmax manager failed");
}
#endif
return MAL_SUCCEED;
}
/*
* Create an interpreter pool.
* One worker will adaptively be available for each client.
* The remainder are taken from the GDKnr_threads argument and
* typically is equal to the number of cores
* The workers are assembled in a local table to enable debugging.
*/
static int
DFLOWinitialize(void)
{
int i, limit;
int created = 0;
MT_lock_set(&mal_contextLock, "DFLOWinitialize");
if (todo) {
/* somebody else beat us to it */
MT_lock_unset(&mal_contextLock, "DFLOWinitialize");
return 0;
}
todo = q_create(2048, "todo");
if (todo == NULL) {
MT_lock_unset(&mal_contextLock, "DFLOWinitialize");
return -1;
}
for (i = 0; i < THREADS; i++)
MT_sema_init(&workers[i].s, 0, "DFLOWinitialize");
limit = GDKnr_threads ? GDKnr_threads - 1 : 0;
#ifdef NEED_MT_LOCK_INIT
ATOMIC_INIT(exitingLock, "exitingLock");
MT_lock_init(&dataflowLock, "dataflowLock");
#endif
MT_lock_set(&dataflowLock, "DFLOWinitialize");
for (i = 0; i < limit; i++) {
workers[i].flag = RUNNING;
workers[i].cntxt = NULL;
if (MT_create_thread(&workers[i].id, DFLOWworker, (void *) &workers[i], MT_THR_JOINABLE) < 0)
workers[i].flag = IDLE;
else
created++;
}
MT_lock_unset(&dataflowLock, "DFLOWinitialize");
if (created == 0) {
/* no threads created */
q_destroy(todo);
todo = NULL;
MT_lock_unset(&mal_contextLock, "DFLOWinitialize");
return -1;
}
MT_lock_unset(&mal_contextLock, "DFLOWinitialize");
return 0;
}
static Queue*
q_create(int sz, const char *name)
{
Queue *q = (Queue*)GDKmalloc(sizeof(Queue));
if (q == NULL)
return NULL;
q->size = ((sz << 1) >> 1); /* we want a multiple of 2 */
q->last = 0;
q->data = (FlowEvent*) GDKmalloc(sizeof(FlowEvent) * q->size);
if (q->data == NULL) {
GDKfree(q);
return NULL;
}
q->exitcount = 0;
(void) name; /* in case MT_LOCK_TRACE is not enabled in gdk_system.h */
MT_lock_init(&q->l, name);
MT_sema_init(&q->s, 0, name);
return q;
}
static reader_thread_data *
create_reader_thread_data(bam_wrapper * bws, int nr_files, sht nr_threads)
{
reader_thread_data *d =
(reader_thread_data *) GDKmalloc(nr_threads *
sizeof(reader_thread_data));
MT_Lock *reader_lock = (MT_Lock *) GDKmalloc(sizeof(MT_Lock));
int *cur_file = (int *) GDKmalloc(sizeof(int));
str reader_lock_ref = "bamReaderLock";
bit *failure = (bit *) GDKmalloc(sizeof(bit));
sht i;
assert(nr_threads > 0);
if (d == NULL || reader_lock == NULL || cur_file == NULL
|| failure == NULL) {
GDKfree(d);
GDKfree(reader_lock);
GDKfree(cur_file);
GDKfree(failure);
return NULL;
}
MT_lock_init(reader_lock, reader_lock_ref);
*cur_file = -1;
*failure = FALSE;
for (i = 0; i < nr_threads; ++i) {
d[i].thread_id = i;
d[i].bws = bws;
d[i].reader_lock = reader_lock;
d[i].reader_lock_ref = reader_lock_ref;
d[i].cur_file = cur_file;
d[i].nr_files = nr_files;
d[i].msg = MAL_SUCCEED;
d[i].failure = failure;
}
return d;
}
int
GDKinit(opt *set, int setlen)
{
char *dbpath = mo_find_option(set, setlen, "gdk_dbpath");
char *p;
opt *n;
int i, j, nlen = 0;
char buf[16];
/* some sanity checks (should also find if symbols are not defined) */
assert(sizeof(char) == SIZEOF_CHAR);
assert(sizeof(short) == SIZEOF_SHORT);
assert(sizeof(int) == SIZEOF_INT);
assert(sizeof(long) == SIZEOF_LONG);
assert(sizeof(lng) == SIZEOF_LNG);
assert(sizeof(oid) == SIZEOF_OID);
assert(sizeof(void *) == SIZEOF_VOID_P);
assert(sizeof(wrd) == SIZEOF_WRD);
assert(sizeof(size_t) == SIZEOF_SIZE_T);
assert(sizeof(ptrdiff_t) == SIZEOF_PTRDIFF_T);
assert(SIZEOF_OID == SIZEOF_INT || SIZEOF_OID == SIZEOF_LNG);
#ifdef NEED_MT_LOCK_INIT
MT_lock_init(&MT_system_lock,"MT_system_lock");
ATOMIC_INIT(GDKstoppedLock, "GDKstoppedLock");
ATOMIC_INIT(mbyteslock, "mbyteslock");
MT_lock_init(&GDKnameLock, "GDKnameLock");
MT_lock_init(&GDKthreadLock, "GDKthreadLock");
MT_lock_init(&GDKtmLock, "GDKtmLock");
#endif
for (i = 0; i <= BBP_BATMASK; i++) {
MT_lock_init(&GDKbatLock[i].swap, "GDKswapLock");
MT_lock_init(&GDKbatLock[i].hash, "GDKhashLock");
MT_lock_init(&GDKbatLock[i].imprints, "GDKimprintsLock");
}
for (i = 0; i <= BBP_THREADMASK; i++) {
MT_lock_init(&GDKbbpLock[i].alloc, "GDKcacheLock");
MT_lock_init(&GDKbbpLock[i].trim, "GDKtrimLock");
GDKbbpLock[i].free = 0;
}
errno = 0;
if (!GDKenvironment(dbpath))
return 0;
if ((p = mo_find_option(set, setlen, "gdk_debug")))
GDKdebug = strtol(p, NULL, 10);
if ((p = mo_find_option(set, setlen, "gdk_mem_pagebits")))
GDK_mem_pagebits = (int) strtol(p, NULL, 10);
mnstr_init();
MT_init_posix();
THRinit();
#ifndef NATIVE_WIN32
BATSIGinit();
#endif
#ifdef WIN32
(void) signal(SIGABRT, BATSIGabort);
_set_abort_behavior(0, _CALL_REPORTFAULT | _WRITE_ABORT_MSG);
_set_error_mode(_OUT_TO_STDERR);
#endif
GDKlockHome();
/* Mserver by default takes 80% of all memory as a default */
GDK_mem_maxsize = GDK_mem_maxsize_max = (size_t) ((double) MT_npages() * (double) MT_pagesize() * 0.815);
#ifdef NATIVE_WIN32
GDK_mmap_minsize = GDK_mem_maxsize_max;
#else
GDK_mmap_minsize = MIN( 1<<30 , GDK_mem_maxsize_max/6 );
/* per op: 2 args + 1 res, each with head & tail => (2+1)*2 = 6 ^ */
#endif
GDK_mem_bigsize = 1024*1024;
GDKremovedir(DELDIR);
BBPinit();
HEAPcacheInit();
GDKkey = BATnew(TYPE_void, TYPE_str, 100);
GDKval = BATnew(TYPE_void, TYPE_str, 100);
if (GDKkey == NULL)
GDKfatal("GDKinit: Could not create environment BAT");
if (GDKval == NULL)
GDKfatal("GDKinit: Could not create environment BAT");
BATseqbase(GDKkey,0);
BATkey(GDKkey, BOUND2BTRUE);
BATrename(GDKkey, "environment_key");
BATmode(GDKkey, TRANSIENT);
BATseqbase(GDKval,0);
BATkey(GDKval, BOUND2BTRUE);
BATrename(GDKval, "environment_val");
BATmode(GDKval, TRANSIENT);
n = (opt *) malloc(setlen * sizeof(opt));
for (i = 0; i < setlen; i++) {
int done = 0;
for (j = 0; j < nlen; j++) {
if (strcmp(n[j].name, set[i].name) == 0) {
if (n[j].kind < set[i].kind) {
n[j] = set[i];
}
done = 1;
break;
}
}
if (!done) {
n[nlen] = set[i];
nlen++;
}
}
for (i = 0; i < nlen; i++)
GDKsetenv(n[i].name, n[i].value);
free(n);
if ((p = GDKgetenv("gdk_dbpath")) != NULL &&
(p = strrchr(p, DIR_SEP)) != NULL) {
GDKsetenv("gdk_dbname", p + 1);
#if DIR_SEP != '/' /* on Windows look for different separator */
} else if ((p = GDKgetenv("gdk_dbpath")) != NULL &&
(p = strrchr(p, '/')) != NULL) {
GDKsetenv("gdk_dbname", p + 1);
#endif
}
if ((p = GDKgetenv("gdk_mem_maxsize"))) {
GDK_mem_maxsize = MAX(1 << 26, (size_t) strtoll(p, NULL, 10));
}
if ((p = GDKgetenv("gdk_vm_maxsize"))) {
GDK_vm_maxsize = MAX(1 << 30, (size_t) strtoll(p, NULL, 10));
}
if ((p = GDKgetenv("gdk_mem_bigsize"))) {
/* when allocating >6% of all RAM; do so using
* vmalloc() iso malloc() */
lng max_mem_bigsize = GDK_mem_maxsize_max / 16;
/* sanity check to avoid memory fragmentation */
GDK_mem_bigsize = (size_t) MIN(max_mem_bigsize, strtoll(p, NULL, 10));
}
if ((p = GDKgetenv("gdk_mmap_minsize"))) {
GDK_mmap_minsize = MAX(REMAP_PAGE_MAXSIZE, (size_t) strtoll(p, NULL, 10));
}
if (GDKgetenv("gdk_mem_pagebits") == NULL) {
snprintf(buf, sizeof(buf), "%d", GDK_mem_pagebits);
GDKsetenv("gdk_mem_pagebits", buf);
}
if (GDKgetenv("gdk_mem_bigsize") == NULL) {
snprintf(buf, sizeof(buf), SZFMT, GDK_mem_bigsize);
GDKsetenv("gdk_mem_bigsize", buf);
}
if (GDKgetenv("monet_pid") == NULL) {
snprintf(buf, sizeof(buf), "%d", (int) getpid());
GDKsetenv("monet_pid", buf);
}
GDKnr_threads = GDKgetenv_int("gdk_nr_threads", 0);
if (GDKnr_threads == 0)
GDKnr_threads = MT_check_nr_cores();
#ifdef NATIVE_WIN32
GDK_mmap_minsize /= (GDKnr_threads ? GDKnr_threads : 1);
#else
/* WARNING: This unconditionally overwrites above settings, */
/* incl. setting via MonetDB env. var. "gdk_mmap_minsize" ! */
GDK_mmap_minsize = MIN( 1<<30 , (GDK_mem_maxsize_max/6) / (GDKnr_threads ? GDKnr_threads : 1) );
/* per op: 2 args + 1 res, each with head & tail => (2+1)*2 = 6 ^ */
#endif
if ((p = mo_find_option(set, setlen, "gdk_vmtrim")) == NULL ||
strcasecmp(p, "yes") == 0)
MT_create_thread(&GDKvmtrim_id, GDKvmtrim, &GDK_mem_maxsize,
MT_THR_JOINABLE);
return 1;
}
int mal_init(void){
#ifdef NEED_MT_LOCK_INIT
MT_lock_init( &mal_contextLock, "mal_contextLock");
MT_lock_init( &mal_namespaceLock, "mal_namespaceLock");
MT_lock_init( &mal_remoteLock, "mal_remoteLock");
MT_lock_init( &mal_profileLock, "mal_profileLock");
MT_lock_init( &mal_copyLock, "mal_copyLock");
MT_lock_init( &mal_delayLock, "mal_delayLock");
#endif
/* "/2" is arbitrarily used / chosen, as on systems with
* hyper-threading enabled, using all hardware threads rather than
* "only" all physical cores does not necessarily yield a linear
* performance benefit */
MT_sema_init( &mal_parallelism, (GDKnr_threads > 1 ? GDKnr_threads/2: 1), "mal_parallelism");
tstAligned();
MCinit();
if (mdbInit())
return -1;
if (monet_memory == 0)
monet_memory = MT_npages() * MT_pagesize();
initNamespace();
initParser();
initHeartbeat();
initResource();
#ifdef HAVE_JSONSTORE
startHttpdaemon();
#endif
RECYCLEinit();
if( malBootstrap() == 0)
return -1;
/* set up the profiler if needed, output sent to console */
/* Use the same shortcuts as stethoscope */
if ( mal_trace && *mal_trace) {
char *s;
setFilterAll();
openProfilerStream(mal_clients[0].fdout);
for ( s= mal_trace; *s; s++)
switch(*s){
case 'a': activateCounter("aggregate");break;
case 'b': activateCounter("rbytes");
activateCounter("wbytes");break;
case 'c': activateCounter("cpu");break;
case 'e': activateCounter("event");break;
case 'f': activateCounter("function");break;
case 'i': activateCounter("pc");break;
case 'm': activateCounter("memory");break;
case 'p': activateCounter("process");break;
case 'r': activateCounter("reads");break;
case 's': activateCounter("stmt");break;
case 't': activateCounter("ticks");break;
case 'u': activateCounter("user");break;
case 'w': activateCounter("writes");break;
case 'y': activateCounter("type");break;
case 'D': activateCounter("dot");break;
case 'I': activateCounter("thread");break;
case 'T': activateCounter("time");break;
case 'S': activateCounter("start");
}
startProfiling();
} else mal_trace =0;
return 0;
}
int main(int argc, char *argv[])
{
/*Initialize the catalog locks*/
MT_lock_init(&dataLock);
MT_cond_init(&mainCond);
MT_cond_init(&writeTCond);
MT_cond_init(&readCond);
char* file_name_in = 0;
char* file_name_out = 0;
char separator_sign = ' ';
char* parse_string = "xyz";
char* buffer;
char printstring[256];
LASReaderH reader = NULL;
LASHeaderH header = NULL;
LASPointH p = NULL;
FILE** files_out = NULL;
int len, j;
int64_t mortonkey = 0;
unsigned int index = 0;
int num_files_in = 0, num_files_out = 0, num_files, num_of_entries=0, check = 0, num_read_threads = DEFAULT_NUM_READ_THREADS;
int i;
pthread_t *writeThreads = NULL;
pthread_t *readThreads = NULL;
struct readThreadArgs *dataRead = NULL;
boolean input_file = FALSE;
int64_t global_offset_x = 0;
int64_t global_offset_y = 0;
double scale_x;
double scale_y;
if (argc == 1) {
usage();
exit(0);
}
/*Allocate space for input files*/
files_name_in = (char**) malloc(sizeof(char*)*DEFAULT_NUM_INPUT_FILES);
for (i = 1; i < argc; i++)
{
if ( strcmp(argv[i],"-h") == 0 ||
strcmp(argv[i],"--help") == 0
)
{
usage();
exit(0);
}
else if ( strcmp(argv[i],"-v") == 0 ||
strcmp(argv[i],"--verbose") == 0
)
{
verbose = TRUE;
}
else if ( strcmp(argv[i],"--num_read_threads") == 0)
{
num_read_threads = atoi(argv[++i]);
}
else if ( strcmp(argv[i],"-s") == 0 ||
strcmp(argv[i],"--skip_invalid") == 0
)
{
skip_invalid = TRUE;
}
else if ( strcmp(argv[i], "--parse") == 0 ||
strcmp(argv[i], "-parse") == 0
)
{
i++;
parse_string = argv[i];
}
else if ( strcmp(argv[i], "--moffset") == 0 ||
strcmp(argv[i], "-moffset") == 0
)
{
i++;
buffer = strtok (argv[i], ",");
j = 0;
while (buffer) {
if (j == 0) {
global_offset_x = S64(buffer);
}
else if (j == 1) {
global_offset_y = S64(buffer);
}
j++;
buffer = strtok (NULL, ",");
while (buffer && *buffer == '\040')
buffer++;
}
if (j != 2){
fprintf(stderr, "Only two int64_t are required in moffset option!\n");
exit(1);
}
}
else if ( strcmp(argv[i], "--check") == 0 ||
strcmp(argv[i], "-check") == 0
)
{
i++;
check = 1;
buffer = strtok (argv[i], ",");
j = 0;
while (buffer) {
if (j == 0) {
sscanf(buffer, "%lf", &scale_x);
}
else if (j == 1) {
sscanf(buffer, "%lf", &scale_y);
}
j++;
buffer = strtok (NULL, ",");
while (buffer && *buffer == '\040')
buffer++;
}
if (j != 2){
fprintf(stderr, "Only two doubles are required in moffset option!\n");
exit(1);
}
}
else if ( strcmp(argv[i],"--input") == 0 ||
strcmp(argv[i],"-input") == 0 ||
strcmp(argv[i],"-i") == 0 ||
strcmp(argv[i],"-in") == 0
)
{
i++;
files_name_in[num_files_in++] = argv[i];
if (num_files_in % DEFAULT_NUM_INPUT_FILES)
files_name_in = (char**) realloc(files_name_in, (num_files_in*2)*sizeof(char*));
}
else if (strcmp(argv[i],"--file") == 0 ||
strcmp(argv[i],"-file") == 0 ||
strcmp(argv[i],"-f") == 0
)
{
i++;
int read;
char line_buffer[BUFSIZ];
FILE* in = NULL;
in = fopen(argv[i], "r");
if (!in) {
fprintf(stderr, "ERROR: the path for file containing the input files is invalid %s\n", argv[i]);
exit(1);
}
while (fgets(line_buffer, sizeof(line_buffer), in)) {
line_buffer[strlen(line_buffer)-1]='\0';
files_name_in[num_files_in++] = strdup(line_buffer);
if (num_files_in % DEFAULT_NUM_INPUT_FILES)
files_name_in = (char**) realloc(files_name_in, (num_files_in*2)*sizeof(char*));
}
fclose(in);
input_file = TRUE;
}
else if ((num_files_in != 0) && num_files_out == 0)
{
file_name_out = argv[i];
num_files_out++;
}
else
{
fprintf(stderr, "ERROR: unknown argument '%s'\n",argv[i]);
usage();
exit(1);
}
} /* end looping through argc/argv */
num_of_entries = strlen(parse_string);
if (num_files_in == 0)
{
LASError_Print("No input filename was specified");
usage();
exit(1);
}
num_files = num_files_in;
/*Entries metadata*/
i = 0;
for (;;)
{
switch (parse_string[i])
{
/* // the morton code on xy */
case 'k':
entries[i] = ENTRY_k;
entriesType[i] = sizeof(int64_t);
/*Changes for Oscar's new Morton code function*/
//entriesFunc[i] = (void*)morton2D_encode;
entriesFunc[i] = (void*)morton2D_encodeOscar;
break;
/* // the x coordinate double*/
case 'x':
entries[i] = ENTRY_x;
entriesType[i] = sizeof(double);
entriesFunc[i] = (void*)LASPoint_GetX;
break;
/* // the y coordinate double*/
case 'y':
entries[i] = ENTRY_y;
entriesType[i] = sizeof(double);
entriesFunc[i] = (void*)LASPoint_GetY;
break;
/* // the z coordinate double*/
case 'z':
entries[i] = ENTRY_z;
entriesType[i] = sizeof(double);
entriesFunc[i] = (void*)LASPoint_GetZ;
break;
/* // the X coordinate decimal*/
case 'X':
entries[i] = ENTRY_X;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetX;
break;
/* // the y coordinate decimal*/
case 'Y':
entries[i] = ENTRY_Y;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetY;
break;
/* // the z coordinate decimal*/
case 'Z':
entries[i] = ENTRY_Z;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetZ;
break;
/* // the gps-time */
case 't':
entries[i] = ENTRY_t;
entriesType[i] = sizeof(double);
entriesFunc[i] = (void*)LASPoint_GetTime;
break;
/* // the intensity */
case 'i':
entries[i] = ENTRY_i;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetIntensity;
break;
/* the scan angle */
case 'a':
entries[i] = ENTRY_a;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetScanAngleRank;
break;
/* the number of the return */
case 'r':
entries[i] = ENTRY_r;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetReturnNumber;
break;
/* the classification */
case 'c':
entries[i] = ENTRY_c;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetClassification;
break;
/* the user data */
case 'u':
entries[i] = ENTRY_u;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetUserData;
break;
/* the number of returns of given pulse */
case 'n':
entries[i] = ENTRY_n;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetNumberOfReturns;
break;
/* the red channel color */
case 'R':
entries[i] = ENTRY_R;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASColor_GetRed;
break;
/* the green channel color */
case 'G':
entries[i] = ENTRY_G;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASColor_GetGreen;
break;
/* the blue channel color */
case 'B':
entries[i] = ENTRY_B;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASColor_GetBlue;
break;
case 'M':
entries[i] = ENTRY_M;
entriesType[i] = sizeof(unsigned int);
break;
case 'p':
entries[i] = ENTRY_p;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetPointSourceId;
break;
/* the edge of flight line flag */
case 'e':
entries[i] = ENTRY_e;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetFlightLineEdge;
break;
/* the direction of scan flag */
case 'd':
entries[i] = ENTRY_d;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetScanDirection;
break;
}
i++;
if (parse_string[i] == 0)
{
break;
}
}
/*Prepare the output files*/
if (file_name_out == NULL)
{
len = (int)strlen(file_name_in);
file_name_out = LASCopyString(file_name_in);
if (file_name_out[len-3] == '.' && file_name_out[len-2] == 'g' && file_name_out[len-1] == 'z')
{
len = len - 4;
}
while (len > 0 && file_name_out[len] != '.')
{
len--;
}
file_name_out[len] = '\0';
}
char *str = malloc(sizeof(char)*(strlen(file_name_out)+12));
files_out = (FILE**) malloc(sizeof(FILE*)*num_of_entries);
for (i = 0; i < num_of_entries; i++) {
sprintf(str, "%s_col_%c.dat", file_name_out, parse_string[i]);
if(doesFileExist(str)) {
remove(str);
}
files_out[i] = fopen(str, "wb");
if (files_out[i] == 0) {
LASError_Print("Could not open file for write");
usage();
exit(1);
}
}
free(str);
/*Initialize structures for the reading threads*/
//data = (struct writeT**) malloc(num_read_threads*sizeof(struct writeT*)); //Malloc is more efficient than calloc
data = (struct writeT**) calloc(num_read_threads, sizeof(struct writeT*));
dataRead = (struct readThreadArgs*) malloc(sizeof(struct readThreadArgs)*num_read_threads);
/* Launch read Threads */
stop = 0;
readThreads = (pthread_t*) malloc(sizeof(pthread_t)*num_read_threads);
for (i=0; i < num_read_threads; i++) {
dataRead[i].id = i;
dataRead[i].num_read_threads = num_read_threads;
dataRead[i].num_of_entries = num_of_entries;
dataRead[i].check = check;
dataRead[i].global_offset_x = global_offset_x;
dataRead[i].global_offset_y = global_offset_y;
dataRead[i].scale_x = scale_x;
dataRead[i].scale_y = scale_y;
pthread_create(&readThreads[i], NULL, readFile, (void*)dataRead);
}
int writeIndex = 0;
writeThreads = (pthread_t*) malloc(sizeof(pthread_t)*num_of_entries);
/* Launch Threads */
struct writeThreadArgs *dataWrite = (struct writeThreadArgs *) malloc(sizeof(struct writeThreadArgs) *num_of_entries);
for (i = 0; i < num_of_entries; i++) {
dataWrite[i].id = i;
dataWrite[i].out = files_out[i];
pthread_create(&writeThreads[i], NULL, writeFile, (void*)(&dataWrite[i]));
}
sleep(1);
//Do we need to comment this one out!?
int done = 0;
while (num_files) {
/*Obtain lock over data to get the pointer*/
MT_set_lock(&dataLock);
dataWriteT = data[writeIndex];
while (dataWriteT == NULL) {
/*Sleep and wait for data to be read*/
MT_cond_wait(&mainCond,&dataLock);
dataWriteT = data[writeIndex];
}
data[writeIndex] = NULL;
//Release the lock
/*Tell the write threads there is new data*/
pthread_cond_broadcast(&writeTCond);
/*Tell the read threads there is a new buf empty*/
pthread_cond_broadcast(&readCond);
MT_unset_lock(&dataLock);
/*Keep looping*/
writeIndex++;
writeIndex = (writeIndex % num_read_threads);
MT_set_lock(&dataLock);
while (done == 0) {
/*Sleep and wait for data to be read*/
MT_cond_wait(&mainCond,&dataLock);
done = 1;
for (i = 0; i < num_of_entries; i++) {
if (dataWriteT[i].values != NULL) {
done = 0;
break;
}
}
}
num_files--;
if (verbose)
printf("Files to go %d\n", num_files);
free(dataWriteT);
dataWriteT = NULL;
done = 0;
MT_unset_lock(&dataLock);
}
/*Tell the write threads to exit*/
MT_set_lock(&dataLock);
stop = 1;
pthread_cond_broadcast(&writeTCond);
MT_unset_lock(&dataLock);
/* Wait for Threads to Finish */
for (i=0; i<num_of_entries; i++) {
pthread_join(writeThreads[i], NULL);
}
free(dataWrite);
free(writeThreads);
MT_cond_destroy(&readCond);
MT_cond_destroy(&writeTCond);
MT_cond_destroy(&mainCond);
MT_lock_destroy(&dataLock);
for (i = 0; i < num_of_entries; i++) {
fflush(files_out[i]);
if (verbose)
printf("close file %d\n", i);
fsync(files_out[i]);
fclose(files_out[i]);
}
free(files_out);
if (input_file) {
for (i=0 ; i < num_files_in; i++)
free(files_name_in[i]);
free(files_name_in);
}
free(dataRead);
if (readThreads)
free(readThreads);
return 0;
}
/* We create a pool of GDKnr_threads-1 generic workers, that is,
* workers that will take on jobs from any clients. In addition, we
* create a single specific worker per client (i.e. each time we enter
* here). This specific worker will only do work for the client for
* which it was started. In this way we can guarantee that there will
* always be progress for the client, even if all other workers are
* doing something big.
*
* When all jobs for a client have been done (there are no more
* entries for the client in the queue), the specific worker turns
* itself into a generic worker. At the same time, we signal that one
* generic worker should exit and this function returns. In this way
* we make sure that there are once again GDKnr_threads-1 generic
* workers. */
str
runMALdataflow(Client cntxt, MalBlkPtr mb, int startpc, int stoppc, MalStkPtr stk)
{
DataFlow flow = NULL;
str msg = MAL_SUCCEED;
int size;
int *ret;
int i;
#ifdef DEBUG_FLOW
fprintf(stderr, "#runMALdataflow for block %d - %d\n", startpc, stoppc);
printFunction(GDKstdout, mb, 0, LIST_MAL_STMT | LIST_MAPI);
#endif
/* in debugging mode we should not start multiple threads */
if (stk == NULL)
throw(MAL, "dataflow", "runMALdataflow(): Called with stk == NULL");
ret = (int*) getArgReference(stk,getInstrPtr(mb,startpc),0);
*ret = FALSE;
if (stk->cmd) {
*ret = TRUE;
return MAL_SUCCEED;
}
assert(stoppc > startpc);
/* check existence of workers */
if (todo == NULL) {
/* create thread pool */
if (GDKnr_threads <= 1 || DFLOWinitialize() < 0) {
/* no threads created, run serially */
*ret = TRUE;
return MAL_SUCCEED;
}
i = THREADS; /* we didn't create an extra thread */
}
assert(todo);
/* in addition, create one more worker that will only execute
* tasks for the current client to compensate for our waiting
* until all work is done */
MT_lock_set(&dataflowLock, "runMALdataflow");
/* join with already exited threads */
{
int joined;
do {
joined = 0;
for (i = 0; i < THREADS; i++) {
if (workers[i].flag == EXITED) {
workers[i].flag = JOINING;
workers[i].cntxt = NULL;
joined = 1;
MT_lock_unset(&dataflowLock, "runMALdataflow");
MT_join_thread(workers[i].id);
MT_lock_set(&dataflowLock, "runMALdataflow");
workers[i].flag = IDLE;
}
}
} while (joined);
}
for (i = 0; i < THREADS; i++) {
if (workers[i].flag == IDLE) {
/* only create specific worker if we are not doing a
* recursive call */
if (stk->calldepth > 1) {
int j;
MT_Id pid = MT_getpid();
/* doing a recursive call: copy specificity from
* current worker to new worker */
workers[i].cntxt = NULL;
for (j = 0; j < THREADS; j++) {
if (workers[j].flag == RUNNING && workers[j].id == pid) {
workers[i].cntxt = workers[j].cntxt;
break;
}
}
} else {
/* not doing a recursive call: create specific worker */
workers[i].cntxt = cntxt;
}
workers[i].flag = RUNNING;
if (MT_create_thread(&workers[i].id, DFLOWworker, (void *) &workers[i], MT_THR_JOINABLE) < 0) {
/* cannot start new thread, run serially */
*ret = TRUE;
workers[i].flag = IDLE;
MT_lock_unset(&dataflowLock, "runMALdataflow");
return MAL_SUCCEED;
}
break;
}
}
MT_lock_unset(&dataflowLock, "runMALdataflow");
if (i == THREADS) {
/* no empty thread slots found, run serially */
*ret = TRUE;
return MAL_SUCCEED;
}
flow = (DataFlow)GDKzalloc(sizeof(DataFlowRec));
if (flow == NULL)
throw(MAL, "dataflow", "runMALdataflow(): Failed to allocate flow");
flow->cntxt = cntxt;
flow->mb = mb;
flow->stk = stk;
flow->error = 0;
/* keep real block count, exclude brackets */
flow->start = startpc + 1;
flow->stop = stoppc;
MT_lock_init(&flow->flowlock, "flow->flowlock");
flow->done = q_create(stoppc- startpc+1, "flow->done");
if (flow->done == NULL) {
MT_lock_destroy(&flow->flowlock);
GDKfree(flow);
throw(MAL, "dataflow", "runMALdataflow(): Failed to create flow->done queue");
}
flow->status = (FlowEvent)GDKzalloc((stoppc - startpc + 1) * sizeof(FlowEventRec));
if (flow->status == NULL) {
q_destroy(flow->done);
MT_lock_destroy(&flow->flowlock);
GDKfree(flow);
throw(MAL, "dataflow", "runMALdataflow(): Failed to allocate flow->status");
}
size = DFLOWgraphSize(mb, startpc, stoppc);
size += stoppc - startpc;
flow->nodes = (int*)GDKzalloc(sizeof(int) * size);
if (flow->nodes == NULL) {
GDKfree(flow->status);
q_destroy(flow->done);
MT_lock_destroy(&flow->flowlock);
GDKfree(flow);
throw(MAL, "dataflow", "runMALdataflow(): Failed to allocate flow->nodes");
}
flow->edges = (int*)GDKzalloc(sizeof(int) * size);
if (flow->edges == NULL) {
GDKfree(flow->nodes);
GDKfree(flow->status);
q_destroy(flow->done);
MT_lock_destroy(&flow->flowlock);
GDKfree(flow);
throw(MAL, "dataflow", "runMALdataflow(): Failed to allocate flow->edges");
}
msg = DFLOWinitBlk(flow, mb, size);
if (msg == MAL_SUCCEED)
msg = DFLOWscheduler(flow, &workers[i]);
GDKfree(flow->status);
GDKfree(flow->edges);
GDKfree(flow->nodes);
q_destroy(flow->done);
MT_lock_destroy(&flow->flowlock);
GDKfree(flow);
if (i != THREADS) {
/* we created one worker, now tell one worker to exit again */
MT_lock_set(&todo->l, "runMALdataflow");
todo->exitcount++;
MT_lock_unset(&todo->l, "runMALdataflow");
MT_sema_up(&todo->s, "runMALdataflow");
}
return msg;
}
