static ssize_t release(FifoDescriptor* frd) {
ssize_t wres = -1;
int res;
int fdadm = frd->fdp;
FifoFilePointer* frp = frd->filePointer;
off_t releasepos = frd->filePointer->releasePos;
off_t readpos = frd->filePointer->readPos;
int roll = frd->filePointer->roll;
res = fifoReadFilePointer(frd);
if ( res < 0 ) goto RETURN;
if ( releasepos != frp->releasePos || readpos != frp->readPos ) {
wres = -1;
goto RETURN;
}
frp->current = frd->current;
frp->releasePos = frp->readPos;
if ( roll || frp->readPos > frd->parameters->switchSize ) {
frp->current += 1;
frp->releasePos = 0;
frp->readPos = 0;
}
fifoWriteFilePointer(frd);
RETURN:
releaselock(fdadm);
return wres;
}
static ssize_t writelocked(FifoDescriptor* fwd, const char* buffer, size_t size) {
ssize_t wres = -1;
off_t sres;
int fres;
int res;
const int fd = fwd->fd;
const off_t max = fwd->parameters->switchSize;
fres = takewritelock(fd);
if ( fres < 0 ) {
err("writelocked takewritelock:");
goto RETURN;
}
sres = lseek(fd, 0, SEEK_END);
while ( sres > 0 && sres + (off_t) size > max ) {
res = rolloverfile(fwd);
if ( res < 0 ) {
err("writelocked:");
goto RETURN;
}
sres = lseek(fd, 0, SEEK_END);
}
if ( sres < 0 ) {
err("writelocked: lseek failed:");
goto RETURN;
}
wres = write(fd, buffer, size);
if ( wres < 0 ) {
err("writelocked write:");
}
releaselock(fd);
RETURN:
return wres;
}
int main (int argc, char **argv)
{
int i;
int tmp;
struct list_element_t *temp_element_p;
int opt;
process_name = argv[0];
pthread_mutex_init(&list_mtx, NULL);
pthread_mutex_init(&data_ready_mtx, NULL);
pthread_cond_init(&data_ready_cv, NULL);
pthread_t thread_refs[MAX_NUM_PROCS];
//get command line options
while ((opt = getopt(argc, argv, "hP:c:f:eQ:R:")) != -1)
{
switch (opt)
{
case 'c':
command = optarg;
break;
case 'e':
ensure_exit = 1;
break;
case 'f':
if ( read_file_into_buffer(optarg,&command) == 0)
{
exit(2);
}
break;
case 'P':
num_procs = atoi(optarg);
if (num_procs > MAX_NUM_PROCS) num_procs = MAX_NUM_PROCS;
break;
case 'R':
consumer_bottleneck_poll_interval = atoi(optarg);
break;
case 'Q':
list_elements_per_proc = atoi(optarg);
break;
default:
print_usage();
fprintf(stderr,"Invalid option %c\n", opt);
exit(2);
break;
}
}
//too many args or too few
if (argc > optind || argc < 2)
{
print_usage();
fprintf(stderr,"Invalid command line options\n");
}
//make sure there is a command to run
if ( command == NULL )
{
fprintf(stderr, "You must specify a command to run\n");
print_usage();
exit(2);
}
if (consumer_bottleneck_poll_interval < 0)
{
print_usage();
fprintf(stderr,"Invalid Wait Period\n");
exit(2);
}
if (list_elements_per_proc < 1)
{
print_usage();
fprintf(stderr,"Invalid line buffer size\n");
exit(2);
}
if (num_procs < 1)
{
print_usage();
fprintf(stderr,"Invalid number of processes\n");
exit(2);
}
num_list_elements = num_procs * list_elements_per_proc;
//intialize the list elements and lists
for (i = 0; i < num_list_elements; i++)
{
//malloc each element and set link
temp_element_p = (struct list_element_t *) malloc(sizeof(struct list_element_t));
if (temp_element_p == NULL)
{
fprintf(stderr, "Malloc failed.\n");
exit(2);
}
list_add(&free_list, temp_element_p);
}
//now start up the threads.
for (i=0; i < num_procs; i++)
{
tmp = pthread_create(&thread_refs[i], NULL, writer_thread, NULL);
if (tmp != 0 )
{
fprintf(stderr, "pthread create failed for proc %i with return value of: %i.\n", i, tmp);
exit(2);
}
}
grablock();
temp_element_p = list_remove(&free_list);
releaselock();
//loop through each line of input
while(fgets(temp_element_p->buffer, PATH_MAX, stdin) != NULL)
{
grablock();
list_add(&ready_list, temp_element_p);
temp_element_p = list_remove(&free_list);
releaselock();
signal_data_ready();
//if we are backed up, signal again
if (list_length(&free_list) < (num_list_elements/2))
{
broadcast_data_ready();
}
//wait if we don't have any free list elements available, keep signalling all the while
while (temp_element_p == NULL)
{
usleep(consumer_bottleneck_poll_interval);
broadcast_data_ready();
grablock();
temp_element_p = list_remove(&free_list);
releaselock();
}
}
//wait for ready_list to empty
while(list_length(&ready_list) > 0)
{
signal_data_ready();
}
input_done = 1;
//make sure no one is stuck on condition variable
broadcast_data_ready();
//fprintf(stderr, "Done processing input, finishing up\n");
//join all the writer threads
for (i = 0; i < num_procs; i++)
{
pthread_join(thread_refs[i], NULL);
}
exit(0);
}
//function for writer threads.
void *writer_thread(void *arg)
{
FILE *output_fh;
int proc_id;
struct list_element_t *element;
int tmp;
char proc_id_str[MAX_NUM_PROCS_DIGITS+1];
proc_id_str[0] = '\0';
grablock();
proc_id = num_procs_initialized;
num_procs_initialized++;
//Set ENV
if (snprintf(proc_id_str, MAX_NUM_PROCS_DIGITS, "%i", proc_id) > 0)
{
setenv( "XPIPES_INDEX", proc_id_str, 1);
}
//If you want to do cpu affinity locking, priorities, etc, do it here!
output_fh = popen(command, popen_type);
if (output_fh == NULL)
{
fprintf(stderr,"popen failed!\n");
exit(3);
}
//make out line buffered
setvbuf(output_fh, NULL, _IOLBF, 0);
//fprintf(stderr,"Initialzied proc %i \n",num_procs_initialized);
releaselock();
while (input_done == 0)
{
wait_data_ready();
grablock();
if (list_length(&ready_list) > 0)
{
element = list_remove(&ready_list);
releaselock();
if (element != NULL)
{
tmp = fprintf(output_fh, "%s", element->buffer);
if (tmp < strlen(element->buffer))
{
fprintf(stderr,"Error printing whole string to pipe\n");
}
if (tmp < 0)
{
fprintf(stderr,"Pipe %i broken\n", proc_id);
pclose(output_fh);
pthread_exit(NULL);
}
element->buffer[0] = '\0';
grablock();
list_add(&free_list, element);
releaselock();
}
} else
{
releaselock();
}
}
//flush any data in pipe
if (ensure_exit == 1)
{
//give consumers fair chance to consume data, but don't hang infinately
usleep(1000);
} else
{
//this can make us hang
fflush(output_fh);
}
//do pclose
pclose(output_fh);
pthread_exit(NULL);
}
static ssize_t readlocked(FifoDescriptor* frd, char* buffer, size_t size) {
ssize_t wres = -1;
ssize_t fres = -1;
off_t sres;
int res;
int fd = frd->fd;
int fdadm = frd->fdp;
FifoParameters *fp = frd->parameters;
FifoFilePointer *frp = frd->filePointer;
res = takewritelock(fdadm);
if ( res < 0 ) goto RETURN;
res = takereadlock(fd);
if ( res < 0 ) goto RETURN;
res = fifoReadFilePointer(frd);
if ( res < 0 ) goto RETURN;
if ( frp->current != frd->current ) {
/* re-open fd with other file */
res = fifoReOpenRead(frd);
if ( res < 0 && errno == ENOENT ) {
err(NULL);
errno = EAGAIN;
goto RETURN;
}
if ( res < 0 ) {
err("fifoRead:");
goto RETURN;
}
}
if ( frp->readPos > frp->releasePos ) {
err("fifoRead: must first call release:");
wres = -1;
goto RETURN; /* must first call release */
}
sres = lseek(fd, frp->readPos, SEEK_SET);
if ( sres < 0 ) {
err("fifoRead lseek:");
wres = -1;
goto RETURN;
}
fres = read(fd, buffer, size);
if ( fres < 0 ) {
err("fifoRead read:");
goto RETURN;
}
if ( fres == 0 ) {
errno = EAGAIN;
goto RETURN;
}
if (memcmp(buffer, fp->rollmark, strlen(fp->rollmark)) == 0) {
frd->filePointer->roll = 1;
}
wres = fifoFormatReadBuffer(fp, buffer, &fres);
if ( wres < 0 ) {
goto RETURN;
}
frp->readPos += fres;
RETURN:
releaselock(fd);
if ( wres >= 0 ) {
fifoWriteFilePointer(frd);
} else {
releaselock(fdadm);
}
return wres;
}
FifoDescriptor* fifoOpenW( const char* filename ) {
char* name;
int res = -1;
long resl;
FifoDescriptor* fwd;
FifoDescriptor* fp = NULL;
err(NULL);
fwd = (FifoDescriptor*) malloc(sizeof(*fwd));
if ( fwd == NULL ) {
err("fifoOpenW malloc descriptor:");
goto RETURN;
}
fwd->fdp = -1;
fwd->fd = -1;
fwd->parameters = (FifoParameters*) malloc(sizeof(*fwd->parameters));
if ( fwd->parameters == NULL ) {
err("fifoOpenW malloc parameters:");
goto RETURN;
}
fwd->filePointer = (FifoFilePointer*) malloc(sizeof(*fwd->filePointer));
if ( fwd->filePointer == NULL ) {
err("fifoOpenR malloc filePointer:");
goto RETURN;
}
fwd->parameters->pathName = fifoAbsfilename(filename);
if ( fwd->parameters->pathName == NULL ) {
err("fifoOpenW fifoAbsfilename:");
goto RETURN;
}
res = fifoReadParams(filename, fwd->parameters);
if ( res < 0 ) {
err("fifoOpenW read parameters:");
goto RETURN;
}
res = fifoOpenFilePointer(fwd, NULL);
if ( res < 0 ) {
err("fifoOpenR open read pointer:");
goto RETURN;
}
takewritelock(fwd->fdp);
res = fifoReadFilePointer(fwd);
if ( fwd->filePointer->current <= 0 ) {
resl = fifoGetCurrent(fwd->parameters->pathName);
fwd->filePointer->current = resl;
}
fwd->current = fwd->filePointer->current;
if ( resl < 0 ) {
err("fifoOpenW get current:");
goto RETURN;
}
name = fifoCurrentAbsfilename(filename, fwd->current);
if ( name == NULL ) {
err("fifoOpenW fifoCurrentAbsfilename:");
goto RETURN;
}
fwd->fd = open(name ,O_WRONLY | O_CREAT, 0666);
free(name);
if ( fwd->fd < 0 ) {
err("fifoOpenW open ");
err(name);
err(":");
goto RETURN;
}
fifoWriteFilePointer(fwd);
fp = fwd;
RETURN:
if ( fwd && fwd->fdp >= 0 ) releaselock(fwd->fdp);
if ( fp == NULL ) {
if ( fwd && fwd->parameters ) {
if ( fwd->parameters->pathName ) {
free(fwd->parameters->pathName);
}
free(fwd->parameters);
free(fwd);
}
}
return fp;
}
static int rolloverfile(FifoDescriptor* fwd) {
char* filename = NULL;
int fd = fwd->fd;
unsigned long newcurrent;
int fd2 = -1;
int res = -1;
char* rollmark = fwd->parameters->rollmark;
res = takewritelock(fwd->fdp);
if (fd >= 0) write(fd, rollmark, strlen(rollmark));
if ( res < 0 ) {
err("rolloverfile:");
goto RETURN;
}
res = fifoReadFilePointer(fwd);
if ( res < 0 ) {
err("rolloverfile:");
goto RETURN;
}
newcurrent = fwd->filePointer->current;
if ( newcurrent == fwd->current ) {
newcurrent = fwd->current + 1;
}
filename = fifoCurrentAbsfilename(fwd->parameters->pathName, newcurrent);
if ( filename == NULL ) {
err("rolloverfile new filename:");
goto RETURN;
}
fd2 = open(filename, O_WRONLY|O_CREAT, 0666);
if ( fd2 < 0 ) {
err("rolloverfile create and open new file ");
err(filename);
err(":");
goto RETURN;
}
fwd->current = newcurrent;
fwd->filePointer->current = newcurrent;
res = fifoWriteFilePointer(fwd);
if ( res < 0 ) {
err("rolloverfile:");
goto RETURN;
}
releaselock(fwd->fdp);
res = takewritelock(fd2);
if ( res < 0 ) {
err("rolloverfile:");
goto RETURN;
}
close(fd);
res = dup2(fd2, fd);
if ( res < 0 ) {
err("rolloverfile dup:");
}
RETURN:
if ( fd2 >= 0 ) close(fd2);
if ( filename ) free(filename);
return res;
}
