static void q_destroy(Queue *q) { assert(q); MT_lock_destroy(&q->l); MT_sema_destroy(&q->s); GDKfree(q->data); GDKfree(q); }
static void destroy_reader_thread_data(reader_thread_data * d) { /* We can safely access d[0], since the bam_loader function * made sure that nr_threads >= 1 and nr_threads is exactly * the size of the array */ MT_lock_destroy(d[0].reader_lock); GDKfree(d[0].reader_lock); GDKfree(d[0].cur_file); GDKfree(d[0].failure); GDKfree(d); }
void list_destroy(list *l) { if (l) { node *n = l->h; MT_lock_destroy(&l->ht_lock); l->h = NULL; if (l->destroy || l->sa == NULL) { while (n) { node *t = n; n = t->next; node_destroy(l, t); } } if (!l->sa) _DELETE(l); } }
int main(int argc, char *argv[]) { /*Initialize the catalog locks*/ MT_lock_init(&dataLock); MT_cond_init(&mainCond); MT_cond_init(&writeTCond); MT_cond_init(&readCond); char* file_name_in = 0; char* file_name_out = 0; char separator_sign = ' '; char* parse_string = "xyz"; char* buffer; char printstring[256]; LASReaderH reader = NULL; LASHeaderH header = NULL; LASPointH p = NULL; FILE** files_out = NULL; int len, j; int64_t mortonkey = 0; unsigned int index = 0; int num_files_in = 0, num_files_out = 0, num_files, num_of_entries=0, check = 0, num_read_threads = DEFAULT_NUM_READ_THREADS; int i; pthread_t *writeThreads = NULL; pthread_t *readThreads = NULL; struct readThreadArgs *dataRead = NULL; boolean input_file = FALSE; int64_t global_offset_x = 0; int64_t global_offset_y = 0; double scale_x; double scale_y; if (argc == 1) { usage(); exit(0); } /*Allocate space for input files*/ files_name_in = (char**) malloc(sizeof(char*)*DEFAULT_NUM_INPUT_FILES); for (i = 1; i < argc; i++) { if ( strcmp(argv[i],"-h") == 0 || strcmp(argv[i],"--help") == 0 ) { usage(); exit(0); } else if ( strcmp(argv[i],"-v") == 0 || strcmp(argv[i],"--verbose") == 0 ) { verbose = TRUE; } else if ( strcmp(argv[i],"--num_read_threads") == 0) { num_read_threads = atoi(argv[++i]); } else if ( strcmp(argv[i],"-s") == 0 || strcmp(argv[i],"--skip_invalid") == 0 ) { skip_invalid = TRUE; } else if ( strcmp(argv[i], "--parse") == 0 || strcmp(argv[i], "-parse") == 0 ) { i++; parse_string = argv[i]; } else if ( strcmp(argv[i], "--moffset") == 0 || strcmp(argv[i], "-moffset") == 0 ) { i++; buffer = strtok (argv[i], ","); j = 0; while (buffer) { if (j == 0) { global_offset_x = S64(buffer); } else if (j == 1) { global_offset_y = S64(buffer); } j++; buffer = strtok (NULL, ","); while (buffer && *buffer == '\040') buffer++; } if (j != 2){ fprintf(stderr, "Only two int64_t are required in moffset option!\n"); exit(1); } } else if ( strcmp(argv[i], "--check") == 0 || strcmp(argv[i], "-check") == 0 ) { i++; check = 1; buffer = strtok (argv[i], ","); j = 0; while (buffer) { if (j == 0) { sscanf(buffer, "%lf", &scale_x); } else if (j == 1) { sscanf(buffer, "%lf", &scale_y); } j++; buffer = strtok (NULL, ","); while (buffer && *buffer == '\040') buffer++; } if (j != 2){ fprintf(stderr, "Only two doubles are required in moffset option!\n"); exit(1); } } else if ( strcmp(argv[i],"--input") == 0 || strcmp(argv[i],"-input") == 0 || strcmp(argv[i],"-i") == 0 || strcmp(argv[i],"-in") == 0 ) { i++; files_name_in[num_files_in++] = argv[i]; if (num_files_in % DEFAULT_NUM_INPUT_FILES) files_name_in = (char**) realloc(files_name_in, (num_files_in*2)*sizeof(char*)); } else if (strcmp(argv[i],"--file") == 0 || strcmp(argv[i],"-file") == 0 || strcmp(argv[i],"-f") == 0 ) { i++; int read; char line_buffer[BUFSIZ]; FILE* in = NULL; in = fopen(argv[i], "r"); if (!in) { fprintf(stderr, "ERROR: the path for file containing the input files is invalid %s\n", argv[i]); exit(1); } while (fgets(line_buffer, sizeof(line_buffer), in)) { line_buffer[strlen(line_buffer)-1]='\0'; files_name_in[num_files_in++] = strdup(line_buffer); if (num_files_in % DEFAULT_NUM_INPUT_FILES) files_name_in = (char**) realloc(files_name_in, (num_files_in*2)*sizeof(char*)); } fclose(in); input_file = TRUE; } else if ((num_files_in != 0) && num_files_out == 0) { file_name_out = argv[i]; num_files_out++; } else { fprintf(stderr, "ERROR: unknown argument '%s'\n",argv[i]); usage(); exit(1); } } /* end looping through argc/argv */ num_of_entries = strlen(parse_string); if (num_files_in == 0) { LASError_Print("No input filename was specified"); usage(); exit(1); } num_files = num_files_in; /*Entries metadata*/ i = 0; for (;;) { switch (parse_string[i]) { /* // the morton code on xy */ case 'k': entries[i] = ENTRY_k; entriesType[i] = sizeof(int64_t); /*Changes for Oscar's new Morton code function*/ //entriesFunc[i] = (void*)morton2D_encode; entriesFunc[i] = (void*)morton2D_encodeOscar; break; /* // the x coordinate double*/ case 'x': entries[i] = ENTRY_x; entriesType[i] = sizeof(double); entriesFunc[i] = (void*)LASPoint_GetX; break; /* // the y coordinate double*/ case 'y': entries[i] = ENTRY_y; entriesType[i] = sizeof(double); entriesFunc[i] = (void*)LASPoint_GetY; break; /* // the z coordinate double*/ case 'z': entries[i] = ENTRY_z; entriesType[i] = sizeof(double); entriesFunc[i] = (void*)LASPoint_GetZ; break; /* // the X coordinate decimal*/ case 'X': entries[i] = ENTRY_X; entriesType[i] = sizeof(int); entriesFunc[i] = (void*)LASPoint_GetX; break; /* // the y coordinate decimal*/ case 'Y': entries[i] = ENTRY_Y; entriesType[i] = sizeof(int); entriesFunc[i] = (void*)LASPoint_GetY; break; /* // the z coordinate decimal*/ case 'Z': entries[i] = ENTRY_Z; entriesType[i] = sizeof(int); entriesFunc[i] = (void*)LASPoint_GetZ; break; /* // the gps-time */ case 't': entries[i] = ENTRY_t; entriesType[i] = sizeof(double); entriesFunc[i] = (void*)LASPoint_GetTime; break; /* // the intensity */ case 'i': entries[i] = ENTRY_i; entriesType[i] = sizeof(int); entriesFunc[i] = (void*)LASPoint_GetIntensity; break; /* the scan angle */ case 'a': entries[i] = ENTRY_a; entriesType[i] = sizeof(int); entriesFunc[i] = (void*)LASPoint_GetScanAngleRank; break; /* the number of the return */ case 'r': entries[i] = ENTRY_r; entriesType[i] = sizeof(int); entriesFunc[i] = (void*)LASPoint_GetReturnNumber; break; /* the classification */ case 'c': entries[i] = ENTRY_c; entriesType[i] = sizeof(int); entriesFunc[i] = (void*)LASPoint_GetClassification; break; /* the user data */ case 'u': entries[i] = ENTRY_u; entriesType[i] = sizeof(int); entriesFunc[i] = (void*)LASPoint_GetUserData; break; /* the number of returns of given pulse */ case 'n': entries[i] = ENTRY_n; entriesType[i] = sizeof(int); entriesFunc[i] = (void*)LASPoint_GetNumberOfReturns; break; /* the red channel color */ case 'R': entries[i] = ENTRY_R; entriesType[i] = sizeof(int); entriesFunc[i] = (void*)LASColor_GetRed; break; /* the green channel color */ case 'G': entries[i] = ENTRY_G; entriesType[i] = sizeof(int); entriesFunc[i] = (void*)LASColor_GetGreen; break; /* the blue channel color */ case 'B': entries[i] = ENTRY_B; entriesType[i] = sizeof(int); entriesFunc[i] = (void*)LASColor_GetBlue; break; case 'M': entries[i] = ENTRY_M; entriesType[i] = sizeof(unsigned int); break; case 'p': entries[i] = ENTRY_p; entriesType[i] = sizeof(int); entriesFunc[i] = (void*)LASPoint_GetPointSourceId; break; /* the edge of flight line flag */ case 'e': entries[i] = ENTRY_e; entriesType[i] = sizeof(int); entriesFunc[i] = (void*)LASPoint_GetFlightLineEdge; break; /* the direction of scan flag */ case 'd': entries[i] = ENTRY_d; entriesType[i] = sizeof(int); entriesFunc[i] = (void*)LASPoint_GetScanDirection; break; } i++; if (parse_string[i] == 0) { break; } } /*Prepare the output files*/ if (file_name_out == NULL) { len = (int)strlen(file_name_in); file_name_out = LASCopyString(file_name_in); if (file_name_out[len-3] == '.' && file_name_out[len-2] == 'g' && file_name_out[len-1] == 'z') { len = len - 4; } while (len > 0 && file_name_out[len] != '.') { len--; } file_name_out[len] = '\0'; } char *str = malloc(sizeof(char)*(strlen(file_name_out)+12)); files_out = (FILE**) malloc(sizeof(FILE*)*num_of_entries); for (i = 0; i < num_of_entries; i++) { sprintf(str, "%s_col_%c.dat", file_name_out, parse_string[i]); if(doesFileExist(str)) { remove(str); } files_out[i] = fopen(str, "wb"); if (files_out[i] == 0) { LASError_Print("Could not open file for write"); usage(); exit(1); } } free(str); /*Initialize structures for the reading threads*/ //data = (struct writeT**) malloc(num_read_threads*sizeof(struct writeT*)); //Malloc is more efficient than calloc data = (struct writeT**) calloc(num_read_threads, sizeof(struct writeT*)); dataRead = (struct readThreadArgs*) malloc(sizeof(struct readThreadArgs)*num_read_threads); /* Launch read Threads */ stop = 0; readThreads = (pthread_t*) malloc(sizeof(pthread_t)*num_read_threads); for (i=0; i < num_read_threads; i++) { dataRead[i].id = i; dataRead[i].num_read_threads = num_read_threads; dataRead[i].num_of_entries = num_of_entries; dataRead[i].check = check; dataRead[i].global_offset_x = global_offset_x; dataRead[i].global_offset_y = global_offset_y; dataRead[i].scale_x = scale_x; dataRead[i].scale_y = scale_y; pthread_create(&readThreads[i], NULL, readFile, (void*)dataRead); } int writeIndex = 0; writeThreads = (pthread_t*) malloc(sizeof(pthread_t)*num_of_entries); /* Launch Threads */ struct writeThreadArgs *dataWrite = (struct writeThreadArgs *) malloc(sizeof(struct writeThreadArgs) *num_of_entries); for (i = 0; i < num_of_entries; i++) { dataWrite[i].id = i; dataWrite[i].out = files_out[i]; pthread_create(&writeThreads[i], NULL, writeFile, (void*)(&dataWrite[i])); } sleep(1); //Do we need to comment this one out!? int done = 0; while (num_files) { /*Obtain lock over data to get the pointer*/ MT_set_lock(&dataLock); dataWriteT = data[writeIndex]; while (dataWriteT == NULL) { /*Sleep and wait for data to be read*/ MT_cond_wait(&mainCond,&dataLock); dataWriteT = data[writeIndex]; } data[writeIndex] = NULL; //Release the lock /*Tell the write threads there is new data*/ pthread_cond_broadcast(&writeTCond); /*Tell the read threads there is a new buf empty*/ pthread_cond_broadcast(&readCond); MT_unset_lock(&dataLock); /*Keep looping*/ writeIndex++; writeIndex = (writeIndex % num_read_threads); MT_set_lock(&dataLock); while (done == 0) { /*Sleep and wait for data to be read*/ MT_cond_wait(&mainCond,&dataLock); done = 1; for (i = 0; i < num_of_entries; i++) { if (dataWriteT[i].values != NULL) { done = 0; break; } } } num_files--; if (verbose) printf("Files to go %d\n", num_files); free(dataWriteT); dataWriteT = NULL; done = 0; MT_unset_lock(&dataLock); } /*Tell the write threads to exit*/ MT_set_lock(&dataLock); stop = 1; pthread_cond_broadcast(&writeTCond); MT_unset_lock(&dataLock); /* Wait for Threads to Finish */ for (i=0; i<num_of_entries; i++) { pthread_join(writeThreads[i], NULL); } free(dataWrite); free(writeThreads); MT_cond_destroy(&readCond); MT_cond_destroy(&writeTCond); MT_cond_destroy(&mainCond); MT_lock_destroy(&dataLock); for (i = 0; i < num_of_entries; i++) { fflush(files_out[i]); if (verbose) printf("close file %d\n", i); fsync(files_out[i]); fclose(files_out[i]); } free(files_out); if (input_file) { for (i=0 ; i < num_files_in; i++) free(files_name_in[i]); free(files_name_in); } free(dataRead); if (readThreads) free(readThreads); return 0; }
/* We create a pool of GDKnr_threads-1 generic workers, that is, * workers that will take on jobs from any clients. In addition, we * create a single specific worker per client (i.e. each time we enter * here). This specific worker will only do work for the client for * which it was started. In this way we can guarantee that there will * always be progress for the client, even if all other workers are * doing something big. * * When all jobs for a client have been done (there are no more * entries for the client in the queue), the specific worker turns * itself into a generic worker. At the same time, we signal that one * generic worker should exit and this function returns. In this way * we make sure that there are once again GDKnr_threads-1 generic * workers. */ str runMALdataflow(Client cntxt, MalBlkPtr mb, int startpc, int stoppc, MalStkPtr stk) { DataFlow flow = NULL; str msg = MAL_SUCCEED; int size; int *ret; int i; #ifdef DEBUG_FLOW fprintf(stderr, "#runMALdataflow for block %d - %d\n", startpc, stoppc); printFunction(GDKstdout, mb, 0, LIST_MAL_STMT | LIST_MAPI); #endif /* in debugging mode we should not start multiple threads */ if (stk == NULL) throw(MAL, "dataflow", "runMALdataflow(): Called with stk == NULL"); ret = (int*) getArgReference(stk,getInstrPtr(mb,startpc),0); *ret = FALSE; if (stk->cmd) { *ret = TRUE; return MAL_SUCCEED; } assert(stoppc > startpc); /* check existence of workers */ if (todo == NULL) { /* create thread pool */ if (GDKnr_threads <= 1 || DFLOWinitialize() < 0) { /* no threads created, run serially */ *ret = TRUE; return MAL_SUCCEED; } i = THREADS; /* we didn't create an extra thread */ } assert(todo); /* in addition, create one more worker that will only execute * tasks for the current client to compensate for our waiting * until all work is done */ MT_lock_set(&dataflowLock, "runMALdataflow"); /* join with already exited threads */ { int joined; do { joined = 0; for (i = 0; i < THREADS; i++) { if (workers[i].flag == EXITED) { workers[i].flag = JOINING; workers[i].cntxt = NULL; joined = 1; MT_lock_unset(&dataflowLock, "runMALdataflow"); MT_join_thread(workers[i].id); MT_lock_set(&dataflowLock, "runMALdataflow"); workers[i].flag = IDLE; } } } while (joined); } for (i = 0; i < THREADS; i++) { if (workers[i].flag == IDLE) { /* only create specific worker if we are not doing a * recursive call */ if (stk->calldepth > 1) { int j; MT_Id pid = MT_getpid(); /* doing a recursive call: copy specificity from * current worker to new worker */ workers[i].cntxt = NULL; for (j = 0; j < THREADS; j++) { if (workers[j].flag == RUNNING && workers[j].id == pid) { workers[i].cntxt = workers[j].cntxt; break; } } } else { /* not doing a recursive call: create specific worker */ workers[i].cntxt = cntxt; } workers[i].flag = RUNNING; if (MT_create_thread(&workers[i].id, DFLOWworker, (void *) &workers[i], MT_THR_JOINABLE) < 0) { /* cannot start new thread, run serially */ *ret = TRUE; workers[i].flag = IDLE; MT_lock_unset(&dataflowLock, "runMALdataflow"); return MAL_SUCCEED; } break; } } MT_lock_unset(&dataflowLock, "runMALdataflow"); if (i == THREADS) { /* no empty thread slots found, run serially */ *ret = TRUE; return MAL_SUCCEED; } flow = (DataFlow)GDKzalloc(sizeof(DataFlowRec)); if (flow == NULL) throw(MAL, "dataflow", "runMALdataflow(): Failed to allocate flow"); flow->cntxt = cntxt; flow->mb = mb; flow->stk = stk; flow->error = 0; /* keep real block count, exclude brackets */ flow->start = startpc + 1; flow->stop = stoppc; MT_lock_init(&flow->flowlock, "flow->flowlock"); flow->done = q_create(stoppc- startpc+1, "flow->done"); if (flow->done == NULL) { MT_lock_destroy(&flow->flowlock); GDKfree(flow); throw(MAL, "dataflow", "runMALdataflow(): Failed to create flow->done queue"); } flow->status = (FlowEvent)GDKzalloc((stoppc - startpc + 1) * sizeof(FlowEventRec)); if (flow->status == NULL) { q_destroy(flow->done); MT_lock_destroy(&flow->flowlock); GDKfree(flow); throw(MAL, "dataflow", "runMALdataflow(): Failed to allocate flow->status"); } size = DFLOWgraphSize(mb, startpc, stoppc); size += stoppc - startpc; flow->nodes = (int*)GDKzalloc(sizeof(int) * size); if (flow->nodes == NULL) { GDKfree(flow->status); q_destroy(flow->done); MT_lock_destroy(&flow->flowlock); GDKfree(flow); throw(MAL, "dataflow", "runMALdataflow(): Failed to allocate flow->nodes"); } flow->edges = (int*)GDKzalloc(sizeof(int) * size); if (flow->edges == NULL) { GDKfree(flow->nodes); GDKfree(flow->status); q_destroy(flow->done); MT_lock_destroy(&flow->flowlock); GDKfree(flow); throw(MAL, "dataflow", "runMALdataflow(): Failed to allocate flow->edges"); } msg = DFLOWinitBlk(flow, mb, size); if (msg == MAL_SUCCEED) msg = DFLOWscheduler(flow, &workers[i]); GDKfree(flow->status); GDKfree(flow->edges); GDKfree(flow->nodes); q_destroy(flow->done); MT_lock_destroy(&flow->flowlock); GDKfree(flow); if (i != THREADS) { /* we created one worker, now tell one worker to exit again */ MT_lock_set(&todo->l, "runMALdataflow"); todo->exitcount++; MT_lock_unset(&todo->l, "runMALdataflow"); MT_sema_up(&todo->s, "runMALdataflow"); } return msg; }