static ssize_t release(FifoDescriptor* frd) { ssize_t wres = -1; int res; int fdadm = frd->fdp; FifoFilePointer* frp = frd->filePointer; off_t releasepos = frd->filePointer->releasePos; off_t readpos = frd->filePointer->readPos; int roll = frd->filePointer->roll; res = fifoReadFilePointer(frd); if ( res < 0 ) goto RETURN; if ( releasepos != frp->releasePos || readpos != frp->readPos ) { wres = -1; goto RETURN; } frp->current = frd->current; frp->releasePos = frp->readPos; if ( roll || frp->readPos > frd->parameters->switchSize ) { frp->current += 1; frp->releasePos = 0; frp->readPos = 0; } fifoWriteFilePointer(frd); RETURN: releaselock(fdadm); return wres; }
static ssize_t writelocked(FifoDescriptor* fwd, const char* buffer, size_t size) { ssize_t wres = -1; off_t sres; int fres; int res; const int fd = fwd->fd; const off_t max = fwd->parameters->switchSize; fres = takewritelock(fd); if ( fres < 0 ) { err("writelocked takewritelock:"); goto RETURN; } sres = lseek(fd, 0, SEEK_END); while ( sres > 0 && sres + (off_t) size > max ) { res = rolloverfile(fwd); if ( res < 0 ) { err("writelocked:"); goto RETURN; } sres = lseek(fd, 0, SEEK_END); } if ( sres < 0 ) { err("writelocked: lseek failed:"); goto RETURN; } wres = write(fd, buffer, size); if ( wres < 0 ) { err("writelocked write:"); } releaselock(fd); RETURN: return wres; }
int main (int argc, char **argv) { int i; int tmp; struct list_element_t *temp_element_p; int opt; process_name = argv[0]; pthread_mutex_init(&list_mtx, NULL); pthread_mutex_init(&data_ready_mtx, NULL); pthread_cond_init(&data_ready_cv, NULL); pthread_t thread_refs[MAX_NUM_PROCS]; //get command line options while ((opt = getopt(argc, argv, "hP:c:f:eQ:R:")) != -1) { switch (opt) { case 'c': command = optarg; break; case 'e': ensure_exit = 1; break; case 'f': if ( read_file_into_buffer(optarg,&command) == 0) { exit(2); } break; case 'P': num_procs = atoi(optarg); if (num_procs > MAX_NUM_PROCS) num_procs = MAX_NUM_PROCS; break; case 'R': consumer_bottleneck_poll_interval = atoi(optarg); break; case 'Q': list_elements_per_proc = atoi(optarg); break; default: print_usage(); fprintf(stderr,"Invalid option %c\n", opt); exit(2); break; } } //too many args or too few if (argc > optind || argc < 2) { print_usage(); fprintf(stderr,"Invalid command line options\n"); } //make sure there is a command to run if ( command == NULL ) { fprintf(stderr, "You must specify a command to run\n"); print_usage(); exit(2); } if (consumer_bottleneck_poll_interval < 0) { print_usage(); fprintf(stderr,"Invalid Wait Period\n"); exit(2); } if (list_elements_per_proc < 1) { print_usage(); fprintf(stderr,"Invalid line buffer size\n"); exit(2); } if (num_procs < 1) { print_usage(); fprintf(stderr,"Invalid number of processes\n"); exit(2); } num_list_elements = num_procs * list_elements_per_proc; //intialize the list elements and lists for (i = 0; i < num_list_elements; i++) { //malloc each element and set link temp_element_p = (struct list_element_t *) malloc(sizeof(struct list_element_t)); if (temp_element_p == NULL) { fprintf(stderr, "Malloc failed.\n"); exit(2); } list_add(&free_list, temp_element_p); } //now start up the threads. for (i=0; i < num_procs; i++) { tmp = pthread_create(&thread_refs[i], NULL, writer_thread, NULL); if (tmp != 0 ) { fprintf(stderr, "pthread create failed for proc %i with return value of: %i.\n", i, tmp); exit(2); } } grablock(); temp_element_p = list_remove(&free_list); releaselock(); //loop through each line of input while(fgets(temp_element_p->buffer, PATH_MAX, stdin) != NULL) { grablock(); list_add(&ready_list, temp_element_p); temp_element_p = list_remove(&free_list); releaselock(); signal_data_ready(); //if we are backed up, signal again if (list_length(&free_list) < (num_list_elements/2)) { broadcast_data_ready(); } //wait if we don't have any free list elements available, keep signalling all the while while (temp_element_p == NULL) { usleep(consumer_bottleneck_poll_interval); broadcast_data_ready(); grablock(); temp_element_p = list_remove(&free_list); releaselock(); } } //wait for ready_list to empty while(list_length(&ready_list) > 0) { signal_data_ready(); } input_done = 1; //make sure no one is stuck on condition variable broadcast_data_ready(); //fprintf(stderr, "Done processing input, finishing up\n"); //join all the writer threads for (i = 0; i < num_procs; i++) { pthread_join(thread_refs[i], NULL); } exit(0); }
//function for writer threads. void *writer_thread(void *arg) { FILE *output_fh; int proc_id; struct list_element_t *element; int tmp; char proc_id_str[MAX_NUM_PROCS_DIGITS+1]; proc_id_str[0] = '\0'; grablock(); proc_id = num_procs_initialized; num_procs_initialized++; //Set ENV if (snprintf(proc_id_str, MAX_NUM_PROCS_DIGITS, "%i", proc_id) > 0) { setenv( "XPIPES_INDEX", proc_id_str, 1); } //If you want to do cpu affinity locking, priorities, etc, do it here! output_fh = popen(command, popen_type); if (output_fh == NULL) { fprintf(stderr,"popen failed!\n"); exit(3); } //make out line buffered setvbuf(output_fh, NULL, _IOLBF, 0); //fprintf(stderr,"Initialzied proc %i \n",num_procs_initialized); releaselock(); while (input_done == 0) { wait_data_ready(); grablock(); if (list_length(&ready_list) > 0) { element = list_remove(&ready_list); releaselock(); if (element != NULL) { tmp = fprintf(output_fh, "%s", element->buffer); if (tmp < strlen(element->buffer)) { fprintf(stderr,"Error printing whole string to pipe\n"); } if (tmp < 0) { fprintf(stderr,"Pipe %i broken\n", proc_id); pclose(output_fh); pthread_exit(NULL); } element->buffer[0] = '\0'; grablock(); list_add(&free_list, element); releaselock(); } } else { releaselock(); } } //flush any data in pipe if (ensure_exit == 1) { //give consumers fair chance to consume data, but don't hang infinately usleep(1000); } else { //this can make us hang fflush(output_fh); } //do pclose pclose(output_fh); pthread_exit(NULL); }
static ssize_t readlocked(FifoDescriptor* frd, char* buffer, size_t size) { ssize_t wres = -1; ssize_t fres = -1; off_t sres; int res; int fd = frd->fd; int fdadm = frd->fdp; FifoParameters *fp = frd->parameters; FifoFilePointer *frp = frd->filePointer; res = takewritelock(fdadm); if ( res < 0 ) goto RETURN; res = takereadlock(fd); if ( res < 0 ) goto RETURN; res = fifoReadFilePointer(frd); if ( res < 0 ) goto RETURN; if ( frp->current != frd->current ) { /* re-open fd with other file */ res = fifoReOpenRead(frd); if ( res < 0 && errno == ENOENT ) { err(NULL); errno = EAGAIN; goto RETURN; } if ( res < 0 ) { err("fifoRead:"); goto RETURN; } } if ( frp->readPos > frp->releasePos ) { err("fifoRead: must first call release:"); wres = -1; goto RETURN; /* must first call release */ } sres = lseek(fd, frp->readPos, SEEK_SET); if ( sres < 0 ) { err("fifoRead lseek:"); wres = -1; goto RETURN; } fres = read(fd, buffer, size); if ( fres < 0 ) { err("fifoRead read:"); goto RETURN; } if ( fres == 0 ) { errno = EAGAIN; goto RETURN; } if (memcmp(buffer, fp->rollmark, strlen(fp->rollmark)) == 0) { frd->filePointer->roll = 1; } wres = fifoFormatReadBuffer(fp, buffer, &fres); if ( wres < 0 ) { goto RETURN; } frp->readPos += fres; RETURN: releaselock(fd); if ( wres >= 0 ) { fifoWriteFilePointer(frd); } else { releaselock(fdadm); } return wres; }
FifoDescriptor* fifoOpenW( const char* filename ) { char* name; int res = -1; long resl; FifoDescriptor* fwd; FifoDescriptor* fp = NULL; err(NULL); fwd = (FifoDescriptor*) malloc(sizeof(*fwd)); if ( fwd == NULL ) { err("fifoOpenW malloc descriptor:"); goto RETURN; } fwd->fdp = -1; fwd->fd = -1; fwd->parameters = (FifoParameters*) malloc(sizeof(*fwd->parameters)); if ( fwd->parameters == NULL ) { err("fifoOpenW malloc parameters:"); goto RETURN; } fwd->filePointer = (FifoFilePointer*) malloc(sizeof(*fwd->filePointer)); if ( fwd->filePointer == NULL ) { err("fifoOpenR malloc filePointer:"); goto RETURN; } fwd->parameters->pathName = fifoAbsfilename(filename); if ( fwd->parameters->pathName == NULL ) { err("fifoOpenW fifoAbsfilename:"); goto RETURN; } res = fifoReadParams(filename, fwd->parameters); if ( res < 0 ) { err("fifoOpenW read parameters:"); goto RETURN; } res = fifoOpenFilePointer(fwd, NULL); if ( res < 0 ) { err("fifoOpenR open read pointer:"); goto RETURN; } takewritelock(fwd->fdp); res = fifoReadFilePointer(fwd); if ( fwd->filePointer->current <= 0 ) { resl = fifoGetCurrent(fwd->parameters->pathName); fwd->filePointer->current = resl; } fwd->current = fwd->filePointer->current; if ( resl < 0 ) { err("fifoOpenW get current:"); goto RETURN; } name = fifoCurrentAbsfilename(filename, fwd->current); if ( name == NULL ) { err("fifoOpenW fifoCurrentAbsfilename:"); goto RETURN; } fwd->fd = open(name ,O_WRONLY | O_CREAT, 0666); free(name); if ( fwd->fd < 0 ) { err("fifoOpenW open "); err(name); err(":"); goto RETURN; } fifoWriteFilePointer(fwd); fp = fwd; RETURN: if ( fwd && fwd->fdp >= 0 ) releaselock(fwd->fdp); if ( fp == NULL ) { if ( fwd && fwd->parameters ) { if ( fwd->parameters->pathName ) { free(fwd->parameters->pathName); } free(fwd->parameters); free(fwd); } } return fp; }
static int rolloverfile(FifoDescriptor* fwd) { char* filename = NULL; int fd = fwd->fd; unsigned long newcurrent; int fd2 = -1; int res = -1; char* rollmark = fwd->parameters->rollmark; res = takewritelock(fwd->fdp); if (fd >= 0) write(fd, rollmark, strlen(rollmark)); if ( res < 0 ) { err("rolloverfile:"); goto RETURN; } res = fifoReadFilePointer(fwd); if ( res < 0 ) { err("rolloverfile:"); goto RETURN; } newcurrent = fwd->filePointer->current; if ( newcurrent == fwd->current ) { newcurrent = fwd->current + 1; } filename = fifoCurrentAbsfilename(fwd->parameters->pathName, newcurrent); if ( filename == NULL ) { err("rolloverfile new filename:"); goto RETURN; } fd2 = open(filename, O_WRONLY|O_CREAT, 0666); if ( fd2 < 0 ) { err("rolloverfile create and open new file "); err(filename); err(":"); goto RETURN; } fwd->current = newcurrent; fwd->filePointer->current = newcurrent; res = fifoWriteFilePointer(fwd); if ( res < 0 ) { err("rolloverfile:"); goto RETURN; } releaselock(fwd->fdp); res = takewritelock(fd2); if ( res < 0 ) { err("rolloverfile:"); goto RETURN; } close(fd); res = dup2(fd2, fd); if ( res < 0 ) { err("rolloverfile dup:"); } RETURN: if ( fd2 >= 0 ) close(fd2); if ( filename ) free(filename); return res; }