static void
q_destroy(Queue *q)
{
assert(q);
MT_lock_destroy(&q->l);
MT_sema_destroy(&q->s);
GDKfree(q->data);
GDKfree(q);
}
static void
destroy_reader_thread_data(reader_thread_data * d)
{
/* We can safely access d[0], since the bam_loader function
* made sure that nr_threads >= 1 and nr_threads is exactly
* the size of the array */
MT_lock_destroy(d[0].reader_lock);
GDKfree(d[0].reader_lock);
GDKfree(d[0].cur_file);
GDKfree(d[0].failure);
GDKfree(d);
}
void
list_destroy(list *l)
{
if (l) {
node *n = l->h;
MT_lock_destroy(&l->ht_lock);
l->h = NULL;
if (l->destroy || l->sa == NULL) {
while (n) {
node *t = n;
n = t->next;
node_destroy(l, t);
}
}
if (!l->sa)
_DELETE(l);
}
}
int main(int argc, char *argv[])
{
/*Initialize the catalog locks*/
MT_lock_init(&dataLock);
MT_cond_init(&mainCond);
MT_cond_init(&writeTCond);
MT_cond_init(&readCond);
char* file_name_in = 0;
char* file_name_out = 0;
char separator_sign = ' ';
char* parse_string = "xyz";
char* buffer;
char printstring[256];
LASReaderH reader = NULL;
LASHeaderH header = NULL;
LASPointH p = NULL;
FILE** files_out = NULL;
int len, j;
int64_t mortonkey = 0;
unsigned int index = 0;
int num_files_in = 0, num_files_out = 0, num_files, num_of_entries=0, check = 0, num_read_threads = DEFAULT_NUM_READ_THREADS;
int i;
pthread_t *writeThreads = NULL;
pthread_t *readThreads = NULL;
struct readThreadArgs *dataRead = NULL;
boolean input_file = FALSE;
int64_t global_offset_x = 0;
int64_t global_offset_y = 0;
double scale_x;
double scale_y;
if (argc == 1) {
usage();
exit(0);
}
/*Allocate space for input files*/
files_name_in = (char**) malloc(sizeof(char*)*DEFAULT_NUM_INPUT_FILES);
for (i = 1; i < argc; i++)
{
if ( strcmp(argv[i],"-h") == 0 ||
strcmp(argv[i],"--help") == 0
)
{
usage();
exit(0);
}
else if ( strcmp(argv[i],"-v") == 0 ||
strcmp(argv[i],"--verbose") == 0
)
{
verbose = TRUE;
}
else if ( strcmp(argv[i],"--num_read_threads") == 0)
{
num_read_threads = atoi(argv[++i]);
}
else if ( strcmp(argv[i],"-s") == 0 ||
strcmp(argv[i],"--skip_invalid") == 0
)
{
skip_invalid = TRUE;
}
else if ( strcmp(argv[i], "--parse") == 0 ||
strcmp(argv[i], "-parse") == 0
)
{
i++;
parse_string = argv[i];
}
else if ( strcmp(argv[i], "--moffset") == 0 ||
strcmp(argv[i], "-moffset") == 0
)
{
i++;
buffer = strtok (argv[i], ",");
j = 0;
while (buffer) {
if (j == 0) {
global_offset_x = S64(buffer);
}
else if (j == 1) {
global_offset_y = S64(buffer);
}
j++;
buffer = strtok (NULL, ",");
while (buffer && *buffer == '\040')
buffer++;
}
if (j != 2){
fprintf(stderr, "Only two int64_t are required in moffset option!\n");
exit(1);
}
}
else if ( strcmp(argv[i], "--check") == 0 ||
strcmp(argv[i], "-check") == 0
)
{
i++;
check = 1;
buffer = strtok (argv[i], ",");
j = 0;
while (buffer) {
if (j == 0) {
sscanf(buffer, "%lf", &scale_x);
}
else if (j == 1) {
sscanf(buffer, "%lf", &scale_y);
}
j++;
buffer = strtok (NULL, ",");
while (buffer && *buffer == '\040')
buffer++;
}
if (j != 2){
fprintf(stderr, "Only two doubles are required in moffset option!\n");
exit(1);
}
}
else if ( strcmp(argv[i],"--input") == 0 ||
strcmp(argv[i],"-input") == 0 ||
strcmp(argv[i],"-i") == 0 ||
strcmp(argv[i],"-in") == 0
)
{
i++;
files_name_in[num_files_in++] = argv[i];
if (num_files_in % DEFAULT_NUM_INPUT_FILES)
files_name_in = (char**) realloc(files_name_in, (num_files_in*2)*sizeof(char*));
}
else if (strcmp(argv[i],"--file") == 0 ||
strcmp(argv[i],"-file") == 0 ||
strcmp(argv[i],"-f") == 0
)
{
i++;
int read;
char line_buffer[BUFSIZ];
FILE* in = NULL;
in = fopen(argv[i], "r");
if (!in) {
fprintf(stderr, "ERROR: the path for file containing the input files is invalid %s\n", argv[i]);
exit(1);
}
while (fgets(line_buffer, sizeof(line_buffer), in)) {
line_buffer[strlen(line_buffer)-1]='\0';
files_name_in[num_files_in++] = strdup(line_buffer);
if (num_files_in % DEFAULT_NUM_INPUT_FILES)
files_name_in = (char**) realloc(files_name_in, (num_files_in*2)*sizeof(char*));
}
fclose(in);
input_file = TRUE;
}
else if ((num_files_in != 0) && num_files_out == 0)
{
file_name_out = argv[i];
num_files_out++;
}
else
{
fprintf(stderr, "ERROR: unknown argument '%s'\n",argv[i]);
usage();
exit(1);
}
} /* end looping through argc/argv */
num_of_entries = strlen(parse_string);
if (num_files_in == 0)
{
LASError_Print("No input filename was specified");
usage();
exit(1);
}
num_files = num_files_in;
/*Entries metadata*/
i = 0;
for (;;)
{
switch (parse_string[i])
{
/* // the morton code on xy */
case 'k':
entries[i] = ENTRY_k;
entriesType[i] = sizeof(int64_t);
/*Changes for Oscar's new Morton code function*/
//entriesFunc[i] = (void*)morton2D_encode;
entriesFunc[i] = (void*)morton2D_encodeOscar;
break;
/* // the x coordinate double*/
case 'x':
entries[i] = ENTRY_x;
entriesType[i] = sizeof(double);
entriesFunc[i] = (void*)LASPoint_GetX;
break;
/* // the y coordinate double*/
case 'y':
entries[i] = ENTRY_y;
entriesType[i] = sizeof(double);
entriesFunc[i] = (void*)LASPoint_GetY;
break;
/* // the z coordinate double*/
case 'z':
entries[i] = ENTRY_z;
entriesType[i] = sizeof(double);
entriesFunc[i] = (void*)LASPoint_GetZ;
break;
/* // the X coordinate decimal*/
case 'X':
entries[i] = ENTRY_X;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetX;
break;
/* // the y coordinate decimal*/
case 'Y':
entries[i] = ENTRY_Y;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetY;
break;
/* // the z coordinate decimal*/
case 'Z':
entries[i] = ENTRY_Z;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetZ;
break;
/* // the gps-time */
case 't':
entries[i] = ENTRY_t;
entriesType[i] = sizeof(double);
entriesFunc[i] = (void*)LASPoint_GetTime;
break;
/* // the intensity */
case 'i':
entries[i] = ENTRY_i;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetIntensity;
break;
/* the scan angle */
case 'a':
entries[i] = ENTRY_a;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetScanAngleRank;
break;
/* the number of the return */
case 'r':
entries[i] = ENTRY_r;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetReturnNumber;
break;
/* the classification */
case 'c':
entries[i] = ENTRY_c;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetClassification;
break;
/* the user data */
case 'u':
entries[i] = ENTRY_u;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetUserData;
break;
/* the number of returns of given pulse */
case 'n':
entries[i] = ENTRY_n;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetNumberOfReturns;
break;
/* the red channel color */
case 'R':
entries[i] = ENTRY_R;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASColor_GetRed;
break;
/* the green channel color */
case 'G':
entries[i] = ENTRY_G;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASColor_GetGreen;
break;
/* the blue channel color */
case 'B':
entries[i] = ENTRY_B;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASColor_GetBlue;
break;
case 'M':
entries[i] = ENTRY_M;
entriesType[i] = sizeof(unsigned int);
break;
case 'p':
entries[i] = ENTRY_p;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetPointSourceId;
break;
/* the edge of flight line flag */
case 'e':
entries[i] = ENTRY_e;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetFlightLineEdge;
break;
/* the direction of scan flag */
case 'd':
entries[i] = ENTRY_d;
entriesType[i] = sizeof(int);
entriesFunc[i] = (void*)LASPoint_GetScanDirection;
break;
}
i++;
if (parse_string[i] == 0)
{
break;
}
}
/*Prepare the output files*/
if (file_name_out == NULL)
{
len = (int)strlen(file_name_in);
file_name_out = LASCopyString(file_name_in);
if (file_name_out[len-3] == '.' && file_name_out[len-2] == 'g' && file_name_out[len-1] == 'z')
{
len = len - 4;
}
while (len > 0 && file_name_out[len] != '.')
{
len--;
}
file_name_out[len] = '\0';
}
char *str = malloc(sizeof(char)*(strlen(file_name_out)+12));
files_out = (FILE**) malloc(sizeof(FILE*)*num_of_entries);
for (i = 0; i < num_of_entries; i++) {
sprintf(str, "%s_col_%c.dat", file_name_out, parse_string[i]);
if(doesFileExist(str)) {
remove(str);
}
files_out[i] = fopen(str, "wb");
if (files_out[i] == 0) {
LASError_Print("Could not open file for write");
usage();
exit(1);
}
}
free(str);
/*Initialize structures for the reading threads*/
//data = (struct writeT**) malloc(num_read_threads*sizeof(struct writeT*)); //Malloc is more efficient than calloc
data = (struct writeT**) calloc(num_read_threads, sizeof(struct writeT*));
dataRead = (struct readThreadArgs*) malloc(sizeof(struct readThreadArgs)*num_read_threads);
/* Launch read Threads */
stop = 0;
readThreads = (pthread_t*) malloc(sizeof(pthread_t)*num_read_threads);
for (i=0; i < num_read_threads; i++) {
dataRead[i].id = i;
dataRead[i].num_read_threads = num_read_threads;
dataRead[i].num_of_entries = num_of_entries;
dataRead[i].check = check;
dataRead[i].global_offset_x = global_offset_x;
dataRead[i].global_offset_y = global_offset_y;
dataRead[i].scale_x = scale_x;
dataRead[i].scale_y = scale_y;
pthread_create(&readThreads[i], NULL, readFile, (void*)dataRead);
}
int writeIndex = 0;
writeThreads = (pthread_t*) malloc(sizeof(pthread_t)*num_of_entries);
/* Launch Threads */
struct writeThreadArgs *dataWrite = (struct writeThreadArgs *) malloc(sizeof(struct writeThreadArgs) *num_of_entries);
for (i = 0; i < num_of_entries; i++) {
dataWrite[i].id = i;
dataWrite[i].out = files_out[i];
pthread_create(&writeThreads[i], NULL, writeFile, (void*)(&dataWrite[i]));
}
sleep(1);
//Do we need to comment this one out!?
int done = 0;
while (num_files) {
/*Obtain lock over data to get the pointer*/
MT_set_lock(&dataLock);
dataWriteT = data[writeIndex];
while (dataWriteT == NULL) {
/*Sleep and wait for data to be read*/
MT_cond_wait(&mainCond,&dataLock);
dataWriteT = data[writeIndex];
}
data[writeIndex] = NULL;
//Release the lock
/*Tell the write threads there is new data*/
pthread_cond_broadcast(&writeTCond);
/*Tell the read threads there is a new buf empty*/
pthread_cond_broadcast(&readCond);
MT_unset_lock(&dataLock);
/*Keep looping*/
writeIndex++;
writeIndex = (writeIndex % num_read_threads);
MT_set_lock(&dataLock);
while (done == 0) {
/*Sleep and wait for data to be read*/
MT_cond_wait(&mainCond,&dataLock);
done = 1;
for (i = 0; i < num_of_entries; i++) {
if (dataWriteT[i].values != NULL) {
done = 0;
break;
}
}
}
num_files--;
if (verbose)
printf("Files to go %d\n", num_files);
free(dataWriteT);
dataWriteT = NULL;
done = 0;
MT_unset_lock(&dataLock);
}
/*Tell the write threads to exit*/
MT_set_lock(&dataLock);
stop = 1;
pthread_cond_broadcast(&writeTCond);
MT_unset_lock(&dataLock);
/* Wait for Threads to Finish */
for (i=0; i<num_of_entries; i++) {
pthread_join(writeThreads[i], NULL);
}
free(dataWrite);
free(writeThreads);
MT_cond_destroy(&readCond);
MT_cond_destroy(&writeTCond);
MT_cond_destroy(&mainCond);
MT_lock_destroy(&dataLock);
for (i = 0; i < num_of_entries; i++) {
fflush(files_out[i]);
if (verbose)
printf("close file %d\n", i);
fsync(files_out[i]);
fclose(files_out[i]);
}
free(files_out);
if (input_file) {
for (i=0 ; i < num_files_in; i++)
free(files_name_in[i]);
free(files_name_in);
}
free(dataRead);
if (readThreads)
free(readThreads);
return 0;
}
/* We create a pool of GDKnr_threads-1 generic workers, that is,
* workers that will take on jobs from any clients. In addition, we
* create a single specific worker per client (i.e. each time we enter
* here). This specific worker will only do work for the client for
* which it was started. In this way we can guarantee that there will
* always be progress for the client, even if all other workers are
* doing something big.
*
* When all jobs for a client have been done (there are no more
* entries for the client in the queue), the specific worker turns
* itself into a generic worker. At the same time, we signal that one
* generic worker should exit and this function returns. In this way
* we make sure that there are once again GDKnr_threads-1 generic
* workers. */
str
runMALdataflow(Client cntxt, MalBlkPtr mb, int startpc, int stoppc, MalStkPtr stk)
{
DataFlow flow = NULL;
str msg = MAL_SUCCEED;
int size;
int *ret;
int i;
#ifdef DEBUG_FLOW
fprintf(stderr, "#runMALdataflow for block %d - %d\n", startpc, stoppc);
printFunction(GDKstdout, mb, 0, LIST_MAL_STMT | LIST_MAPI);
#endif
/* in debugging mode we should not start multiple threads */
if (stk == NULL)
throw(MAL, "dataflow", "runMALdataflow(): Called with stk == NULL");
ret = (int*) getArgReference(stk,getInstrPtr(mb,startpc),0);
*ret = FALSE;
if (stk->cmd) {
*ret = TRUE;
return MAL_SUCCEED;
}
assert(stoppc > startpc);
/* check existence of workers */
if (todo == NULL) {
/* create thread pool */
if (GDKnr_threads <= 1 || DFLOWinitialize() < 0) {
/* no threads created, run serially */
*ret = TRUE;
return MAL_SUCCEED;
}
i = THREADS; /* we didn't create an extra thread */
}
assert(todo);
/* in addition, create one more worker that will only execute
* tasks for the current client to compensate for our waiting
* until all work is done */
MT_lock_set(&dataflowLock, "runMALdataflow");
/* join with already exited threads */
{
int joined;
do {
joined = 0;
for (i = 0; i < THREADS; i++) {
if (workers[i].flag == EXITED) {
workers[i].flag = JOINING;
workers[i].cntxt = NULL;
joined = 1;
MT_lock_unset(&dataflowLock, "runMALdataflow");
MT_join_thread(workers[i].id);
MT_lock_set(&dataflowLock, "runMALdataflow");
workers[i].flag = IDLE;
}
}
} while (joined);
}
for (i = 0; i < THREADS; i++) {
if (workers[i].flag == IDLE) {
/* only create specific worker if we are not doing a
* recursive call */
if (stk->calldepth > 1) {
int j;
MT_Id pid = MT_getpid();
/* doing a recursive call: copy specificity from
* current worker to new worker */
workers[i].cntxt = NULL;
for (j = 0; j < THREADS; j++) {
if (workers[j].flag == RUNNING && workers[j].id == pid) {
workers[i].cntxt = workers[j].cntxt;
break;
}
}
} else {
/* not doing a recursive call: create specific worker */
workers[i].cntxt = cntxt;
}
workers[i].flag = RUNNING;
if (MT_create_thread(&workers[i].id, DFLOWworker, (void *) &workers[i], MT_THR_JOINABLE) < 0) {
/* cannot start new thread, run serially */
*ret = TRUE;
workers[i].flag = IDLE;
MT_lock_unset(&dataflowLock, "runMALdataflow");
return MAL_SUCCEED;
}
break;
}
}
MT_lock_unset(&dataflowLock, "runMALdataflow");
if (i == THREADS) {
/* no empty thread slots found, run serially */
*ret = TRUE;
return MAL_SUCCEED;
}
flow = (DataFlow)GDKzalloc(sizeof(DataFlowRec));
if (flow == NULL)
throw(MAL, "dataflow", "runMALdataflow(): Failed to allocate flow");
flow->cntxt = cntxt;
flow->mb = mb;
flow->stk = stk;
flow->error = 0;
/* keep real block count, exclude brackets */
flow->start = startpc + 1;
flow->stop = stoppc;
MT_lock_init(&flow->flowlock, "flow->flowlock");
flow->done = q_create(stoppc- startpc+1, "flow->done");
if (flow->done == NULL) {
MT_lock_destroy(&flow->flowlock);
GDKfree(flow);
throw(MAL, "dataflow", "runMALdataflow(): Failed to create flow->done queue");
}
flow->status = (FlowEvent)GDKzalloc((stoppc - startpc + 1) * sizeof(FlowEventRec));
if (flow->status == NULL) {
q_destroy(flow->done);
MT_lock_destroy(&flow->flowlock);
GDKfree(flow);
throw(MAL, "dataflow", "runMALdataflow(): Failed to allocate flow->status");
}
size = DFLOWgraphSize(mb, startpc, stoppc);
size += stoppc - startpc;
flow->nodes = (int*)GDKzalloc(sizeof(int) * size);
if (flow->nodes == NULL) {
GDKfree(flow->status);
q_destroy(flow->done);
MT_lock_destroy(&flow->flowlock);
GDKfree(flow);
throw(MAL, "dataflow", "runMALdataflow(): Failed to allocate flow->nodes");
}
flow->edges = (int*)GDKzalloc(sizeof(int) * size);
if (flow->edges == NULL) {
GDKfree(flow->nodes);
GDKfree(flow->status);
q_destroy(flow->done);
MT_lock_destroy(&flow->flowlock);
GDKfree(flow);
throw(MAL, "dataflow", "runMALdataflow(): Failed to allocate flow->edges");
}
msg = DFLOWinitBlk(flow, mb, size);
if (msg == MAL_SUCCEED)
msg = DFLOWscheduler(flow, &workers[i]);
GDKfree(flow->status);
GDKfree(flow->edges);
GDKfree(flow->nodes);
q_destroy(flow->done);
MT_lock_destroy(&flow->flowlock);
GDKfree(flow);
if (i != THREADS) {
/* we created one worker, now tell one worker to exit again */
MT_lock_set(&todo->l, "runMALdataflow");
todo->exitcount++;
MT_lock_unset(&todo->l, "runMALdataflow");
MT_sema_up(&todo->s, "runMALdataflow");
}
return msg;
}
