int queue_fast_open(struct queue *self, int queue_num, int af_family)
{
int ret;
if (self->_cb == NULL) {
throw_exception("Error: no callback set");
return -1;
}
ret = queue_open(self);
if (!ret)
return -1;
queue_unbind(self, af_family);
ret = queue_bind(self, af_family);
if (ret < 0) {
queue_close(self);
return -1;
}
ret = queue_create_queue(self,queue_num);
if (ret < 0) {
queue_unbind(self, af_family);
queue_close(self);
return -1;
}
return 0;
}
VCL_VOID vmod_mqueue__init(const struct vrt_ctx *ctx, struct vmod_msgsend_mqueue **qp, const char *vcl_name, VCL_STRING queue_name)
{
void *queue;
struct vmod_msgsend_mqueue *q;
CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
AN(qp);
AZ(*qp);
if (NULL == (queue = queue_init())) {
if (NULL != ctx->vsl) {
VSLb(ctx->vsl, SLT_Error, "Can't init queue");
}
}
if (QUEUE_ERR_OK != queue_open(queue, queue_name, QUEUE_FL_SENDER)) {
if (NULL != ctx->vsl) {
VSLb(ctx->vsl, SLT_Error, "Can't open queue '%s'", queue_name);
}
queue_close(&queue);
}
XXXAN(queue);
ALLOC_OBJ(q, VMOD_MSGSEND_OBJ_MAGIC);
AN(q);
*qp = q;
q->queue = queue;
q->queue_name = queue_name;
AN(*qp);
}
static void
queue_shutdown(void)
{
log_debug("debug: queue agent exiting");
queue_close();
_exit(0);
}
static void
queue_shutdown(void)
{
log_info("info: queue handler exiting");
queue_close();
_exit(0);
}
VCL_VOID vmod_mqueue__fini(struct vmod_msgsend_mqueue **qp)
{
AN(qp);
CHECK_OBJ_NOTNULL(*qp, VMOD_MSGSEND_OBJ_MAGIC);
queue_close(&(*qp)->queue);
FREE_OBJ(*qp);
*qp = NULL;
}
void job_pool_delete(jobpool* jp) {
queue_close(jp->jobqueue);
zassert(pthread_mutex_lock(&(jp->jobslock)));
while (jp->workers_total>0) {
jp->workers_term_waiting++;
zassert(pthread_cond_wait(&(jp->worker_term_cond),&(jp->jobslock)));
}
zassert(pthread_mutex_unlock(&(jp->jobslock)));
if (!queue_isempty(jp->statusqueue)) {
job_pool_check_jobs(0);
}
// syslog(LOG_NOTICE,"deleting jobqueue: %p",jp->jobqueue);
queue_delete(jp->jobqueue);
queue_delete(jp->statusqueue);
zassert(pthread_cond_destroy(&(jp->worker_term_cond)));
zassert(pthread_mutex_destroy(&(jp->pipelock)));
zassert(pthread_mutex_destroy(&(jp->jobslock)));
close(jp->rpipe);
close(jp->wpipe);
free(jp);
}
void *producer_thread(void *data) {
producer_data_t *pdata = (producer_data_t*)data;
int c;
ssize_t res = read(pdata->sockfd, &c, sizeof(c));
assert(res == sizeof(c));
c = ntohl((uint32_t)c);
if (c < 0) {
assert(c < -10);
queue_close(pdata->q);
close(pdata->closefd);
} else {
queue_produce(pdata->q, c);
}
close(pdata->sockfd);
free(pdata);
return NULL;
}
static void cleanup(void)
{
if (NULL != buffer) {
free(buffer);
buffer = NULL;
}
if (NULL != ctxt) {
if (NULL != engine->close) {
engine->close(ctxt);
}
free(ctxt);
ctxt = NULL;
}
queue_close(&queue);
if (NULL != pidfilename) {
if (0 != unlink(pidfilename)) {
warnc("unlink failed");
}
}
if (NULL != err_file && fileno(err_file) > 2) {
fclose(err_file);
err_file = NULL;
}
}
void master_process() {
MPI_Status status;
queue work_queue = generate_initial_work_queue();
work_item work, current_work, current_longest;
int * sequence_array, * work_array;
int array_buffer_size, n_work_items;
/* We don't want to terminate before all of the partitions of the
* space are calculated, so we keep track of how many we are
* waiting on as well.
*/
int n_outstanding = 0;
int i, j;
int index;
long long int n_evaluated = 0;
int current_longest_size = 0;
do {
/* Test for the existence of message coming from one of the
* worker processes.
*/
MPI_Probe(MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
if (!(n_evaluated % 100000)) {
printf("Evaluated: %lld\n", n_evaluated);
printf("Queue size: %d\n", work_queue->size);
if (current_longest_size)
display_work_item(current_longest);
}
if (status.MPI_TAG == REQUEST_WORK) {
/* If a request was received for more work, register that request. */
MPI_Recv(0, 0, MPI_INT, status.MPI_SOURCE, REQUEST_WORK, MPI_COMM_WORLD, &status);
if (work_queue->size > 0) {
queue_get(work_queue, (void **) &work);
MPI_Send(convert_work_to_array(work), BUFFER_SIZE, MPI_INT, status.MPI_SOURCE, DO_WORK, MPI_COMM_WORLD);
n_outstanding++;
n_evaluated++;
free(work);
} else {
MPI_Send(0, 0, MPI_INT, status.MPI_SOURCE, DELAY_WORK, MPI_COMM_WORLD);
}
} else if (status.MPI_TAG == DONE_WORK) {
/* Determine the size of the array that must be allocated. */
MPI_Get_count(&status, MPI_INT, &array_buffer_size);
sequence_array = (int *) malloc(sizeof(int) * array_buffer_size);
MPI_Recv(sequence_array, array_buffer_size, MPI_INT, status.MPI_SOURCE, DONE_WORK, MPI_COMM_WORLD, &status);
n_work_items = array_buffer_size / BUFFER_SIZE;
index = 0;
for (i = 0; i < n_work_items; i++) {
work_array = malloc(sizeof(int) * BUFFER_SIZE);
for (j = 0; j < BUFFER_SIZE; j++) {
work_array[j] = sequence_array[index];
index++;
}
current_work = convert_array_to_work(work_array);
if (current_work->size > current_longest_size) {
current_longest_size = current_work->size;
current_longest = current_work;
}
queue_put(work_queue, (void *) current_work);
free(work_array);
}
free(sequence_array);
/* A process submitted its results to the master process, so we have fewer
* processes outstanding at this time.
*/
n_outstanding--;
} else if (status.MPI_TAG == SEQUENCE_TERMINATED) {
MPI_Recv(0, 0, MPI_INT, status.MPI_SOURCE, SEQUENCE_TERMINATED, MPI_COMM_WORLD, &status);
n_outstanding--;
}
} while (work_queue->size > 0 || n_outstanding > 0);
queue_close(work_queue);
}
int main(int argc, char* argv[]){
// open output file.
FILE* outputfp;
outputfp = fopen(argv[argc-1], "w");
if(!outputfp){
perror("Error Opening Output File");
return EXIT_FAILURE;
}
/* Create two thread pools for five threads each for a total of 10 threads. */
pthread_t requesterThreads[NUM_THREADS];
pthread_t resolverThreads[NUM_THREADS];
/* Declare local vars */
int rc;
long t;
/* Initialize input file queue */
if (queue_init(&inputQueue, QSIZE) == QUEUE_FAILURE){
fprintf(stderr, "fileQ failed to initialize \n");
}
/* Create a requester thread pool for reading in the files */
/* creates 5 threads that start exicuting the requester function. */
/* one thread for each input file. */
for(t=0; t<NUM_THREADS; t++){
printf("In main: creating requester thread %ld\n", t);
rc = pthread_create(&(requesterThreads[t]), NULL, requester, argv[t+1]);
if (rc){
printf("ERROR: return code from pthread_create() is %d\n", rc);
exit(EXIT_FAILURE);
}
}
// Create a resolver pool for performing DNS calls from each line
/* Create 5 threads that exicute the resolver funciton. */
/* The threads constantly try to pop from the queue */
for(t=0;t<NUM_THREADS;t++){
printf("In main: creating resolver thread %ld\n", t);
rc = pthread_create(&(resolverThreads[t]), NULL, resolver, outputfp);
if (rc){
printf("ERROR: return code from pthread_create() is %d\n", rc);
exit(EXIT_FAILURE);
}
}
/* Wait for All Theads to Finish */
for(t=0;t<NUM_THREADS;t++){
// waits for the given requester thread to terminate.
// a requester thread terminates when all of the hostnames are pushed onto the queue.
pthread_join(requesterThreads[t],NULL);
}
printf("All of the requester threads were completed!\n");
requester_running = FALSE;
queue_close(&inputQueue); // this sets the queue finished flage = 1
for(t=0;t<NUM_THREADS;t++){
// waits for the given resolver thread to terminate.
// a resolver thread terminates when queue_pop returns NULL or when the queue is empty and requester_running is FALSE
// requester_running is set to FALSE before this code exicutes.
pthread_join(resolverThreads[t],NULL);
}
printf("All of the resolver threads were completed!\n");
queue_cleanup(&inputQueue);
fclose(outputfp);
return EXIT_SUCCESS;
}
void event_close(EventQueue* queue)
{
queue_close(queue);
}
