int main(int argc, char *argv[])
{
int en;
long i;
pthread_t id;
double starttime;
if ((en = pthread_create(&id, NULL, producer, NULL)) != 0) {
fprintf(stderr, "pthread_create: %s\n", strerror(en));
exit(EXIT_FAILURE);
}
if ((en = pthread_create(&id, NULL, consumer, NULL)) != 0) {
fprintf(stderr, "pthread_create: %s\n", strerror(en));
exit(EXIT_FAILURE);
}
for (i = 0; i < NWATCHERS; i++)
if ((en = pthread_create(&id, NULL, watcher, (void *)i)) != 0) {
fprintf(stderr, "pthread_create: %s\n", strerror(en));
exit(EXIT_FAILURE);
}
while (!producer_ready || !consumer_ready)
sched_yield();
goflag = 1;
starttime = dgettimeofday();
while ((dgettimeofday() - starttime < 3) && !consumer_done)
poll(NULL, 0, 1);
goflag = 0;
while (!consumer_done || !producer_done)
poll(NULL, 0, 1);
return EXIT_SUCCESS;
}
void *producer(void *ignored)
{
int en;
int i = 0;
producer_ready = 1;
while (!goflag)
sched_yield();
while (goflag) {
if ((en = pthread_mutex_lock(&snaplock)) != 0) {
fprintf(stderr,
"pthread_mutex_lock: %s\n", strerror(en));
exit(EXIT_FAILURE);
}
ss.t = dgettimeofday();
ss.a = ss.c + 1;
ss.b = ss.a + 1;
ss.c = ss.b + 1;
if ((en = pthread_mutex_unlock(&snaplock)) != 0) {
fprintf(stderr,
"pthread_mutex_unlock: %s\n", strerror(en));
exit(EXIT_FAILURE);
}
i++;
}
printf("producer exiting: %d samples\n", i);
producer_done = 1;
return (NULL);
}
void *consumer(void *ignored)
{
struct snapshot_consumer curssc;
int i = 0;
int j = 0;
consumer_ready = 1;
while (ss.t == 0.0) {
sched_yield();
}
while (goflag) {
curssc.tc = dgettimeofday();
curssc.t = ss.t;
curssc.a = ss.a;
curssc.b = ss.b;
curssc.c = ss.c;
curssc.sequence = curseq;
curssc.iserror = 0;
if ((curssc.t > curssc.tc) ||
modgreater(ssc[i].a, curssc.a) ||
modgreater(ssc[i].b, curssc.b) ||
modgreater(ssc[i].c, curssc.c) ||
modgreater(curssc.a, ssc[i].a + maxdelta) ||
modgreater(curssc.b, ssc[i].b + maxdelta) ||
modgreater(curssc.c, ssc[i].c + maxdelta)) {
i++;
curssc.iserror = 1;
} else if (ssc[i].iserror)
i++;
ssc[i] = curssc;
curseq++;
if (i + 1 >= NSNAPS)
break;
}
printf("consumer exited loop, collected %d items out of %d\n",
i, curseq);
if (ssc[0].iserror)
printf("0/%ld: %.6f %.6f (%.3f) %ld %ld %ld\n",
ssc[0].sequence,
ssc[j].t, ssc[j].tc, (ssc[j].tc - ssc[j].t) * 1000000,
ssc[j].a, ssc[j].b, ssc[j].c);
for (j = 0; j <= i; j++)
if (ssc[j].iserror)
printf("%d/%ld: %.6f (%.3f) %ld %ld %ld\n",
j, ssc[j].sequence,
ssc[j].t, (ssc[j].tc - ssc[j].t) * 1000000,
ssc[j].a - ssc[j - 1].a,
ssc[j].b - ssc[j - 1].b,
ssc[j].c - ssc[j - 1].c);
consumer_done = 1;
}
void *producer(void *ignored)
{
int i = 0;
producer_ready = 1;
while (!goflag)
sched_yield();
while (goflag) {
ss.t = dgettimeofday();
ss.a = ss.c + 1;
ss.b = ss.a + 1;
ss.c = ss.b + 1;
i++;
}
printf("producer exiting: %d samples\n", i);
producer_done = 1;
return (NULL);
}
int
adios_read_flexpath_advance_step(ADIOS_FILE *adiosfile, int last, float timeout_sec)
{
flexpath_reader_file *fp = (flexpath_reader_file*)adiosfile->fh;
MPI_Barrier(fp->comm);
int count = 0; // for perf measurements
send_flush_msg(fp, fp->writer_coordinator, STEP, 1);
//put this on a timer, so to speak, for timeout_sec
while(fp->mystep == fp->last_writer_step){
if(fp->writer_finalized){
adios_errno = err_end_of_stream;
return err_end_of_stream;
}
CMsleep(fp_read_data->fp_cm, 1);
send_flush_msg(fp, fp->writer_coordinator, STEP, 1);
}
double advclose_start = dgettimeofday();
int i=0;
for(i=0; i<fp->num_bridges; i++) {
if(fp->bridges[i].created && fp->bridges[i].opened) {
count++;
send_close_msg(fp, i);
}
}
MPI_Barrier(fp->comm);
double advclose_end = dgettimeofday();
count = 0;
adiosfile->current_step++;
fp->mystep = adiosfile->current_step;
double advopen_start = dgettimeofday();
for(i=0; i<fp->num_bridges; i++){
if(fp->bridges[i].created && !fp->bridges[i].opened){
send_open_msg(fp, i);
count++;
}
}
double advopen_end = dgettimeofday();
// need to remove selectors from each var now.
send_flush_msg(fp, fp->writer_coordinator, DATA, 1);
// should only happen if there are more steps available.
// writer should have advanced.
double offset_start = dgettimeofday();
send_flush_msg(fp, fp->writer_coordinator, EVGROUP, 1);
double offset_end = dgettimeofday();
return 0;
}
static int
raw_handler(CManager cm, void *vevent, int len, void *client_data, attr_list attrs)
{
ADIOS_FILE *adiosfile = client_data;
flexpath_reader_file *fp = (flexpath_reader_file*)adiosfile->fh;
double data_end = dgettimeofday();
if(fp->time_in == 0.00)
fp->time_in = data_end; // used for perf measurements only
int condition;
int writer_rank;
int flush_id;
double data_start;
get_double_attr(attrs, attr_atom_from_string("fp_starttime"), &data_start);
get_int_attr(attrs, attr_atom_from_string("fp_dst_condition"), &condition);
get_int_attr(attrs, attr_atom_from_string(FP_RANK_ATTR_NAME), &writer_rank);
get_int_attr(attrs, attr_atom_from_string("fp_flush_id"), &flush_id);
double format_start = dgettimeofday();
FMContext context = CMget_FMcontext(cm);
void *base_data = FMheader_skip(context, vevent);
FMFormat format = FMformat_from_ID(context, vevent);
// copy //FMfree_struct_desc_list call
FMStructDescList struct_list =
FMcopy_struct_list(format_list_of_FMFormat(format));
FMField *f = struct_list[0].field_list;
#if 0
uint64_t packet_size = calc_ffspacket_size(f, attrs, base_data);
fp->data_read += packet_size;
#endif
/* setting up initial vars from the format list that comes along with the
message. Message contains both an FFS description and the data. */
if(fp->num_vars == 0){
int var_count = 0;
fp->var_list = setup_flexpath_vars(f, &var_count);
adiosfile->var_namelist = malloc(var_count * sizeof(char *));
int i = 0;
while(f->field_name != NULL) {
adiosfile->var_namelist[i++] = strdup(f->field_name);
f++;
}
adiosfile->nvars = var_count;
fp->num_vars = var_count;
}
f = struct_list[0].field_list;
char *curr_offset = NULL;
while(f->field_name){
char atom_name[200] = "";
flexpath_var *var = find_fp_var(fp->var_list, f->field_name);
if(!var){
adios_error(err_file_open_error,
"file not opened correctly. var does not match format.\n");
return err_file_open_error;
}
int num_dims = get_ndims_attr(f->field_name, attrs);
var->num_dims = num_dims;
flexpath_var_chunk *curr_chunk = &var->chunks[0];
// has the var been scheduled?
if(var->sel){
if(var->sel->type == ADIOS_SELECTION_WRITEBLOCK){
if(num_dims == 0){ // writeblock selection for scalar
if(var->sel->u.block.index == writer_rank){
void *tmp_data = get_FMfieldAddr_by_name(f, f->field_name, base_data);
memcpy(var->chunks[0].user_buf, tmp_data, f->field_size);
}
}
else { // writeblock selection for arrays
/* if(var->num_dims == 0){ */
/* var->global_dims = malloc(sizeof(uint64_t)*num_dims); */
/* } */
if(var->sel->u.block.index == writer_rank){
var->array_size = var->type_size;
int i;
for(i=0; i<num_dims; i++){
char *dim;
atom_name[0] ='\0';
strcat(atom_name, FP_DIM_ATTR_NAME);
strcat(atom_name, "_");
strcat(atom_name, f->field_name);
strcat(atom_name, "_");
char dim_num[10] = "";
sprintf(dim_num, "%d", i+1);
strcat(atom_name, dim_num);
get_string_attr(attrs, attr_atom_from_string(atom_name), &dim);
FMField *temp_field = find_field_by_name(dim, f);
if(!temp_field){
adios_error(err_corrupted_variable,
"Could not find fieldname: %s\n",
dim);
}
else{
int *temp_data = get_FMfieldAddr_by_name(temp_field,
temp_field->field_name,
base_data);
uint64_t dim = (uint64_t)(*temp_data);
var->array_size = var->array_size * dim;
}
}
void *arrays_data = get_FMPtrField_by_name(f, f->field_name, base_data, 1);
memcpy(var->chunks[0].user_buf, arrays_data, var->array_size);
}
}
}
else if(var->sel->type == ADIOS_SELECTION_BOUNDINGBOX){
if(num_dims == 0){ // scalars; throw error
adios_error(err_offset_required,
"Only scalars can be scheduled with write_block selection.\n");
}
else{ // arrays
int i;
global_var *gv = find_gbl_var(fp->gp->vars,
var->varname,
fp->gp->num_vars);
array_displacements * disp = find_displacement(var->displ,
writer_rank,
var->num_displ);
if(disp){ // does this writer hold a chunk we've asked for, for this var?
uint64_t *temp = gv->offsets[0].local_dimensions;
int offsets_per_rank = gv->offsets[0].offsets_per_rank;
uint64_t *writer_sizes = &temp[offsets_per_rank * writer_rank];
uint64_t *sel_start = disp->start;
uint64_t *sel_count = disp->count;
char *writer_array = (char*)get_FMPtrField_by_name(f,
f->field_name,
base_data, 1);
char *reader_array = (char*)var->chunks[0].user_buf;
uint64_t reader_start_pos = disp->pos;
var->start_position += copyarray(writer_sizes,
sel_start,
sel_count,
disp->ndims,
f->field_size,
0,
writer_array,
reader_array+reader_start_pos);
}
}
}
}
else { //var has not been scheduled;
if(num_dims == 0){ // only worry about scalars
flexpath_var_chunk *chunk = &var->chunks[0];
if(!chunk->has_data){
void *tmp_data = get_FMfieldAddr_by_name(f, f->field_name, base_data);
chunk->data = malloc(f->field_size);
memcpy(chunk->data, tmp_data, f->field_size);
chunk->has_data = 1;
}
}
}
f++;
}
if(condition == -1){
fp->completed_requests++;
if(fp->completed_requests == fp->pending_requests){
pthread_mutex_lock(&fp->data_mutex);
pthread_cond_signal(&fp->data_condition);
pthread_mutex_unlock(&fp->data_mutex);
}
}
else{
CMCondition_signal(fp_read_data->fp_cm, condition);
}
free_fmstructdesclist(struct_list);
return 0;
}
/*
* Sets up local data structure for series of reads on an adios file
* - create evpath graph and structures
* -- create evpath control stone (outgoing)
* -- create evpath data stone (incoming)
* -- rank 0 dumps contact info to file
* -- create connections using contact info from file
*/
ADIOS_FILE*
adios_read_flexpath_open(const char * fname,
MPI_Comm comm,
enum ADIOS_LOCKMODE lock_mode,
float timeout_sec)
{
fp_log("FUNC", "entering flexpath_open\n");
ADIOS_FILE *adiosfile = malloc(sizeof(ADIOS_FILE));
if(!adiosfile){
adios_error (err_no_memory,
"Cannot allocate memory for file info.\n");
return NULL;
}
flexpath_reader_file *fp = new_flexpath_reader_file(fname);
adios_errno = 0;
fp->stone = EValloc_stone(fp_read_data->fp_cm);
fp->comm = comm;
MPI_Comm_size(fp->comm, &(fp->size));
MPI_Comm_rank(fp->comm, &(fp->rank));
EVassoc_terminal_action(fp_read_data->fp_cm,
fp->stone,
op_format_list,
op_msg_handler,
adiosfile);
EVassoc_terminal_action(fp_read_data->fp_cm,
fp->stone,
update_step_msg_format_list,
update_step_msg_handler,
adiosfile);
EVassoc_terminal_action(fp_read_data->fp_cm,
fp->stone,
evgroup_format_list,
group_msg_handler,
adiosfile);
EVassoc_raw_terminal_action(fp_read_data->fp_cm,
fp->stone,
raw_handler,
adiosfile);
/* Gather the contact info from the other readers
and write it to a file. Create a ready file so
that the writer knows it can parse this file. */
double setup_start = dgettimeofday();
char writer_ready_filename[200];
char writer_info_filename[200];
char reader_ready_filename[200];
char reader_info_filename[200];
sprintf(reader_ready_filename, "%s_%s", fname, READER_READY_FILE);
sprintf(reader_info_filename, "%s_%s", fname, READER_CONTACT_FILE);
sprintf(writer_ready_filename, "%s_%s", fname, WRITER_READY_FILE);
sprintf(writer_info_filename, "%s_%s", fname, WRITER_CONTACT_FILE);
char *string_list;
char data_contact_info[CONTACT_LENGTH];
string_list = attr_list_to_string(CMget_contact_list(fp_read_data->fp_cm));
sprintf(&data_contact_info[0], "%d:%s", fp->stone, string_list);
free(string_list);
char * recvbuf;
if(fp->rank == 0){
recvbuf = (char*)malloc(sizeof(char)*CONTACT_LENGTH*(fp->size));
}
MPI_Gather(data_contact_info, CONTACT_LENGTH, MPI_CHAR, recvbuf,
CONTACT_LENGTH, MPI_CHAR, 0, fp->comm);
if(fp->rank == 0){
// print our own contact information
FILE * fp_out = fopen(reader_info_filename, "w");
int i;
if(!fp_out){
adios_error(err_file_open_error,
"File for contact info could not be opened for writing.\n");
exit(1);
}
for(i=0; i<fp->size; i++) {
fprintf(fp_out,"%s\n", &recvbuf[i*CONTACT_LENGTH]);
}
fclose(fp_out);
free(recvbuf);
FILE * read_ready = fopen(reader_ready_filename, "w");
fprintf(read_ready, "ready");
fclose(read_ready);
}
MPI_Barrier(fp->comm);
FILE * fp_in = fopen(writer_ready_filename,"r");
while(!fp_in) {
//CMsleep(fp_read_data->fp_cm, 1);
fp_in = fopen(writer_ready_filename, "r");
}
fclose(fp_in);
fp_in = fopen(writer_info_filename, "r");
while(!fp_in){
//CMsleep(fp_read_data->fp_cm, 1);
fp_in = fopen(writer_info_filename, "r");
}
char in_contact[CONTACT_LENGTH] = "";
//fp->bridges = malloc(sizeof(bridge_info));
int num_bridges = 0;
int their_stone;
// change to read all numbers, dont create stones, turn bridge array into linked list
while(fscanf(fp_in, "%d:%s", &their_stone, in_contact) != EOF){
//fprintf(stderr, "writer contact: %d:%s\n", their_stone, in_contact);
fp->bridges = realloc(fp->bridges,
sizeof(bridge_info) * (num_bridges+1));
fp->bridges[num_bridges].their_num = their_stone;
fp->bridges[num_bridges].contact = strdup(in_contact);
fp->bridges[num_bridges].created = 0;
fp->bridges[num_bridges].step = 0;
fp->bridges[num_bridges].opened = 0;
fp->bridges[num_bridges].scheduled = 0;
num_bridges++;
}
fclose(fp_in);
fp->num_bridges = num_bridges;
// clean up of writer's files
MPI_Barrier(fp->comm);
if(fp->rank == 0){
unlink(writer_info_filename);
unlink(writer_ready_filename);
}
adiosfile->fh = (uint64_t)fp;
adiosfile->current_step = 0;
/* Init with a writer to get initial scalar
data so we can handle inq_var calls and
also populate the ADIOS_FILE struct. */
double bridge_start = MPI_Wtime();
if(fp->size < num_bridges){
int mystart = (num_bridges/fp->size) * fp->rank;
int myend = (num_bridges/fp->size) * (fp->rank+1);
fp->writer_coordinator = mystart;
int z;
for(z=mystart; z<myend; z++){
build_bridge(&fp->bridges[z]);
}
}
else{
int writer_rank = fp->rank % num_bridges;
build_bridge(&fp->bridges[writer_rank]);
fp->writer_coordinator = writer_rank;
}
// requesting initial data.
send_open_msg(fp, fp->writer_coordinator);
fp->data_read = 0;
send_flush_msg(fp, fp->writer_coordinator, DATA, 1);
send_flush_msg(fp, fp->writer_coordinator, EVGROUP, 1);
fp->data_read = 0;
// this has to change. Writer needs to have some way of
// taking the attributes out of the xml document
// and sending them over ffs encoded. Not yet implemented.
// the rest of this info for adiosfile gets filled in raw_handler.
adiosfile->nattrs = 0;
adiosfile->attr_namelist = NULL;
// first step is at least one, otherwise raw_handler will not execute.
// in reality, writer might be further along, so we might have to make
// the writer explitly send across messages each time it calls close, to
// indicate which timesteps are available.
adiosfile->last_step = 1;
adiosfile->path = strdup(fname);
// verifies these two fields. It's not BP, so no BP version.
// It's a stream, so how can the file size be known?
adiosfile->version = -1;
adiosfile->file_size = 0;
adios_errno = err_no_error;
fp_log("FUNC", "leaving flexpath_open\n");
return adiosfile;
}
int main(int argc, char *argv[])
{
printf("time = %.6f\n", dgettimeofday());
exit(0);
}
