int main(int argc, char *argv[])
{
char txtfile[255], bmpfile[255];
if (argc < 2) {
usage(argv[0]);
}
else if (argc == 2) {
char basename[255];
strcpy(bmpfile, argv[1]);
getbasename(basename, argv[1]);
sprintf(txtfile, "%s.txt", basename);
}
else if (argc == 3) {
strcpy(bmpfile, argv[1]);
strcpy(txtfile, argv[2]);
}
else {
usage(argv[0]);
}
dumpprocess(bmpfile, txtfile);
return 0;
}
static void
merge_list_set_val(struct obj_list_data *d, GtkTreeIter *iter, int row)
{
int cx;
unsigned int i = 0;
char *file, *bfile;
for (i = 0; i < MERG_WIN_COL_NUM; i++) {
switch (i) {
case MERG_WIN_COL_FILE:
getobj(d->obj, "file", row, 0, NULL, &file);
bfile = getbasename(file);
if (bfile) {
list_store_set_string(GTK_WIDGET(d->text), iter, i, CHK_STR(bfile));
g_free(bfile);
} else {
list_store_set_string(GTK_WIDGET(d->text), iter, i, "....................");
}
break;
case MERG_WIN_COL_HIDDEN:
getobj(d->obj, Mlist[i].name, row, 0, NULL, &cx);
cx = ! cx;
list_store_set_val(GTK_WIDGET(d->text), iter, i, Mlist[i].type, &cx);
break;
default:
if (Mlist[i].type == G_TYPE_DOUBLE) {
getobj(d->obj, Mlist[i].name, row, 0, NULL, &cx);
list_store_set_double(GTK_WIDGET(d->text), iter, i, cx / 100.0);
} else {
getobj(d->obj, Mlist[i].name, row, 0, NULL, &cx);
list_store_set_val(GTK_WIDGET(d->text), iter, i, Mlist[i].type, &cx);
}
}
}
}
int createdir_recursive( char* path)
{
int res;
char *dirs, *base;
/*printf(" called mkdir %s\n", path);*/
getbasename(path, &dirs, &base);
if ( dirs != NULL) {
if ( !is_dir(dirs) ) {
/*printf(" dirs=%s base=%s\n", dirs, base);*/
res = createdir_recursive(dirs);
} else res = 0;
free(dirs);
if (res) return res;
}
if (file_exists(path))
res = 0;
else
res = mkdir(path, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH);
if (res) {
if (errno == EEXIST) {
print("Concurrency error: createdir %s failed: %s\n", path, strerror(errno));
print("Some other process created this dir already during this function call\n");
res = 0;
} else {
print("createdir %s failed: %s\n", path, strerror(errno));
}
}
//else
// print("createdir %s succeeded\n", path);
free(base);
return (res);
}
char*ambixtest_getfname(char*inbuf, size_t length, const char*path_, const char*basename_, const char*ext_) {
static unsigned int count=0;
const char*ext=(ext_)?ext_:".caf";
const char*path=(path_)?path_:"";
const char*basename=getbasename(basename_);
snprintf(inbuf, length, "%s%s-%d.%d%s", path, basename, ambixtest_uniquenumber(), count, ext);
count++;
return inbuf;
}
bool readfields(void* object, const bsreq* fields) {
while(*p && !islinefeed()) {
const bsreq* req = 0;
if(readidentifier())
req = fields->find(buffer);
if(!req)
warning(ErrorNotFoundMember1pInBase2p, buffer, getbasename(fields));
readreq(object, req, 0);
}
return true;
}
bool eval(int mode, int n, int argc, char **argv)
{
if(n >= argc) disphelp(getbasename(argv[0]));
if(mode == -1) {
if(!strcmp(argv[n], "info-nwn")) return eval(0, 2, argc, argv);
else if(!strcmp(argv[n], "info-erf")) return eval(1, 2, argc, argv);
else if(!strcmp(argv[n], "info-2da")) return eval(2, 2, argc, argv);
else if(!strcmp(argv[n], "info-key")) return eval(3, 2, argc, argv);
else if(!strcmp(argv[n], "info-bif")) return eval(4, 2, argc, argv);
else if(!strcmp(argv[n], "info-gff")) return eval(5, 2, argc, argv);
else if(!strcmp(argv[n], "extract-erf")) return eval(6, 2, argc, argv);
else if(!strcmp(argv[n], "explode-gff")) return eval(7, 2, argc, argv);
else if(!strcmp(argv[n], "assemble-gff")) return eval(8, 2, argc, argv);
else if(!strcmp(argv[n], "explode-erf")) return eval(9, 2, argc, argv);
else if(!strcmp(argv[n], "assemble-erf")) return eval(10, 2, argc, argv);
else if(!strcmp(argv[n], "explode-bif")) return eval(11, 2, argc, argv);
else if(!strcmp(argv[n], "explode-key")) return eval(12, 2, argc, argv);
else if(!strcmp(argv[n], "assemble-key")) return eval(13, 2, argc, argv);
else if(!strcmp(argv[n], "assemble-bif")) return eval(14, 2, argc, argv);
else if(!strcmp(argv[n], "info-ssf")) return eval(15, 2, argc, argv);
else if(!strcmp(argv[n], "explode-ssf")) return eval(16, 2, argc, argv);
else if(!strcmp(argv[n], "assemble-ssf")) return eval(17, 2, argc, argv);
else if(!strcmp(argv[n], "explode-keybif")) return eval(18, 2, argc, argv);
else if(!strcmp(argv[n], "assemble-keybif")) return eval(19, 2, argc, argv);
else if(!strcmp(argv[n], "info-tlk")) return eval(20, 2, argc, argv);
else if(!strcmp(argv[n], "explode-tlk")) return eval(21, 2, argc, argv);
else if(!strcmp(argv[n], "assemble-tlk")) return eval(22, 2, argc, argv);
else disphelp(getbasename(argv[0]));
}
for(int i=n;i<argc;i++)
if(!strcmp(argv[i], "--basedir"))
{
if(i >= (argc-1)) disphelp(getbasename(argv[0]));
else basedir = argv[++i];
if(basedir[basedir.length()-1] != '/') basedir += "/";
}
else if(!modefuncs[mode](argv[i])) return false;
return true;
}
static void
file_dialog_set_current_neme(GtkWidget *dlg, const char *full_name)
{
char *name;
if (dlg == NULL || full_name == NULL)
return;
name = getbasename(full_name);
if (name) {
gtk_file_chooser_set_current_name(GTK_FILE_CHOOSER(dlg), name);
g_free(name);
}
}
int test_relative_file(void)
{
errorf("\n"); /* prints the function name */
File *file = newfile(".", "../tmp/foo");
assert(strcmp(getname(file), "../tmp/foo") == 0);
assert(strcmp(getpath(file), "./../tmp/foo") == 0);
char *dir = getdirname(file);
assert(strcmp(dir, "./../tmp") == 0);
free(dir);
char *base = getbasename(file);
assert(strcmp(base, "foo") == 0);
free(base);
free(file);
return 0;
}
int test_bare_file(void)
{
errorf("\n"); /* prints the function name */
File *file = newfile(".", "buf.c");
assert(strcmp(getname(file), "buf.c") == 0);
assert(strcmp(getpath(file), "./buf.c") == 0);
char *dir = getdirname(file);
assert(strcmp(dir, ".") == 0);
free(dir);
char *base = getbasename(file);
assert(strcmp(base, "buf.c") == 0);
free(base);
free(file);
return 0;
}
int main (int argc, char ** argv) {
//For varriable definitions:
//gbounds = global bounds string, lbounds = local bounds string, offs = offset string, tstring = temp string to hold temperary stuff
char gbounds[1007], lbounds[1007], offs[1007],tstring[100];
//size = number of cores, gidx = adios group index
int rank, size, gidx, i, j, k, ii;
//data = pointer to read-in data
void * data = NULL;
uint64_t s[] = {0,0,0,0,0,0,0,0,0,0}; //starting offset
uint64_t c[] = {1,1,1,1,1,1,1,1,1,1}; //chunk block array
uint64_t bytes_read = 0;
int element_size;
int64_t new_adios_group, m_adios_file;
uint64_t var_size; //portion_bound,
uint64_t adios_groupsize, adios_totalsize;
int read_buffer; //possible maximum size you the user would like for each chunk in MB
int write_buffer = 1536; //actual buffer size you use in MB
int itime;
int WRITEME=1;
uint64_t chunk_size; //chunk size in # of elements
char *var_path, *var_name; // full path cut into dir path and name
MPI_Init(&argc,&argv);
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&size);
// timing numbers
// we will time:
// 0: adios_open, adios_group_size
// 1: the total time to read in the data
// 2: times around each write (will only work if we do NOT buffer....
// 3: the time in the close
// 4: fopen, fclose
// 5: total time
// timers: the total I/O time
int timers = 6;
double start_time[timers], end_time[timers], total_time[timers];
if (TIMING==100) {
for (itime=0;itime<timers;itime++) {
start_time[itime] = 0;
end_time[itime] = 0;
total_time[itime]=0;
}
//MPI_Barrier(MPI_COMM_WORLD);
start_time[5] = MPI_Wtime();
}
if(rank==0)
printf("converting...\n");
if (argc < 5) {
if (rank==0) printf("Usage: %s <BP-file> <ADIOS-file> read_buffer(MB) write_buffer(MB) METHOD (LUSTRE_strip_count) (LUSTRE_strip_size) (LUSTRE_block_size)\n", argv[0]);
return 1;
}
if(TIMING==100)
start_time[4] = MPI_Wtime();
ADIOS_FILE * f = adios_fopen (argv[1], MPI_COMM_SELF);
if(TIMING==100){
end_time[4] = MPI_Wtime();
total_time[4] = end_time[4]-start_time[4];
}
adios_init_noxml(comm); // no xml will be used to write the new adios file
read_buffer = atoi(argv[3]);
write_buffer = atoi(argv[4]);
adios_allocate_buffer (ADIOS_BUFFER_ALLOC_NOW, write_buffer); // allocate MB buffer
if (f == NULL) {
printf("rank=%d, file cant be opened\n", rank);
if (DEBUG) printf ("%s\n", adios_errmsg());
return -1;
}
for (gidx = 0; gidx < f->groups_count; gidx++) { //group part
adios_groupsize = 0;
ADIOS_GROUP * g = adios_gopen (f, f->group_namelist[gidx]);
if (g == NULL) {
if (DEBUG) printf ("%s\n", adios_errmsg());
printf("rank %d: group cannot be opened.\n", rank);
return -1;
}
/* First create all of the groups */
// now I need to create this group in the file that will be written
adios_declare_group(&new_adios_group,f->group_namelist[gidx],"",adios_flag_yes);
if(strcmp(argv[5],"MPI_LUSTRE")!=0) //see whether or not the user uses MPI_LUSTRE method
adios_select_method (new_adios_group, argv[5], "", ""); //non-MPI_LUSTRE methods... like MPI, POSIX....
else{
char lustre_pars[1000];
strcpy(lustre_pars, "");
strcat(lustre_pars, "stripe_count=");
sprintf(tstring, "%d", atoi(argv[6]));
strcat(lustre_pars, tstring);
strcat(lustre_pars, ",stripe_size=");
sprintf(tstring, "%d", atoi(argv[7]));
strcat(lustre_pars, tstring);
strcat(lustre_pars, ",block_size=");
sprintf(tstring, "%d", atoi(argv[8]));
strcat(lustre_pars, tstring);
if(rank==0)
printf("lustre_pars=%s\n", lustre_pars);
adios_select_method (new_adios_group, argv[5], lustre_pars, ""); //Use MPI Lustre method
}
// variable definition part
for (i = 0; i < g->vars_count; i++) {
ADIOS_VARINFO * v = adios_inq_var_byid (g, i);
getbasename (g->var_namelist[i], &var_path, &var_name);
if (v->ndim == 0)
{
// scalars: every process does them the same.
adios_define_var(new_adios_group,var_name,var_path,v->type,0,0,0);
getTypeInfo( v->type, &element_size); //element_size is size per element based on its type
if (v->type == adios_string) { //special case when the scalar is string.
adios_groupsize += strlen(v->value);
} else {
adios_groupsize += element_size;
}
}
else
{
// vector variables
getTypeInfo( v->type, &element_size);
var_size=1;
for (ii=0;ii<v->ndim;ii++) {
var_size*=v->dims[ii];
}
uint64_t total_size = var_size; //total_size tells you the total number of elements in the current vector variable
var_size*=element_size; //var_size tells you the size of the current vector variable in bytess
//re-initialize the s and c variables
for(j=0; j<v->ndim; j++){
s[j] = 0;
c[j] = 1;
}
//find the approximate chunk_size you would like to use.
chunk_size = calcChunkSize(total_size, read_buffer*1024*1024/element_size, size);
//set the chunk block array with the total size as close to chunk_size as possible
calcC(chunk_size, v, c);
strcpy(lbounds,"");
for(j=0; j<v->ndim; j++){
sprintf(tstring, "%" PRId64 ",", c[j]);
strcat(lbounds, tstring);
}
printf("rank=%d, name=%s, chunk_size1=%" PRId64 " c[]=%s\n",rank,g->var_namelist[i],chunk_size,lbounds);
chunk_size = 1;
for(ii=0; ii<v->ndim; ii++) //reset chunk_size based on the created c. Now the chunk_size is exact.
chunk_size *= c[ii];
//current step points to where the process is in processing the vector. First sets with respect to rank.
uint64_t current_step = rank*chunk_size;
//First advance the starting point s by current_step. Of course, you don't do it if the current_step exceeds total_size.
if(current_step<total_size)
rS(v, s, current_step, rank);
uint64_t elements_defined = 0; //First, the number of elements you have defined is 0.
//You (the process) process your part of the vector when your current_step is smaller than the total_size
while(current_step < total_size)
{
//ts, temporary s, is introduced for the sake of the inner do while loop below. Copy s to ts.
uint64_t ts[] = {0,0,0,0,0,0,0,0,0,0};
arrCopy(s, ts);
//for every outer while iteration, you always have the whole chunk_size remained to process.
uint64_t remain_chunk = chunk_size;
if(current_step+chunk_size>total_size) //except when you are nearing the end of the vector....
remain_chunk = total_size-current_step;
//tc, temporary c, is introduced for the sake of the inner do while loop below. Copy s to tc.
uint64_t tc[] = {1,1,1,1,1,1,1,1,1,1};
arrCopy(c, tc);
do{
//how much of the remain chunk you wanna process? initially you think you can do all of it....
uint64_t used_chunk = remain_chunk;
//you feel like you should process the vector with tc block size, but given ts, you might go over bound.
uint64_t uc[] = {1,1,1,1,1,1,1,1,1,1};
//so you verify it by setting a new legit chunck block uc, and getting a new remain_chunk.
remain_chunk = checkBound(v, ts, tc, uc, remain_chunk);
//you check whether or not ts+uc goes over the bound. This is just checking to make sure there's no error.
//Thereotically, there should be no problem at all.
checkOverflow(0, v, ts, uc);
//the below code fragment simply calculates gbounds, and sets place holders for lbounds and offs.
strcpy(gbounds,"");
strcpy(lbounds,"");
strcpy(offs,"");
for(j=0; j<v->ndim-1; j++){
sprintf(tstring, "%d,", (int)v->dims[j]);
strcat(gbounds, tstring);
//sprintf(tstring, "ldim%d_%s,", j, var_name);
sprintf(tstring, "ldim%d,", j);
strcat(lbounds, tstring);
//sprintf(tstring, "offs%d_%s,", j, var_name);
sprintf(tstring, "offs%d,", j);
strcat(offs, tstring);
}
sprintf(tstring, "%d", (int)v->dims[v->ndim-1]);
strcat(gbounds, tstring);
//sprintf(tstring, "ldim%d_%s", v->ndim-1, var_name);
sprintf(tstring, "ldim%d", v->ndim-1);
strcat(lbounds, tstring);
//sprintf(tstring, "offs%d_%s", v->ndim-1, var_name);
sprintf(tstring, "offs%d", v->ndim-1);
strcat(offs, tstring);
//sprintf(tstring, "%d", v->ndim);
for(j=0; j<v->ndim; j++){
//sprintf(tstring, "ldim%d_%s", j, var_name);
sprintf(tstring, "ldim%d", j);
adios_define_var(new_adios_group, tstring, "bp2bp", adios_unsigned_long, 0, 0, 0);
//sprintf(tstring, "offs%d_%s", j, var_name);
sprintf(tstring, "offs%d", j);
adios_define_var(new_adios_group, tstring, "bp2bp", adios_unsigned_long, 0, 0, 0);
}
adios_define_var(new_adios_group,var_name,var_path,v->type,lbounds,gbounds,offs);
if (DEBUG){
strcpy(lbounds,"");
strcpy(offs,"");
for(j=0; j<v->ndim; j++){
sprintf(tstring, "%" PRId64 ",", ts[j]);
strcat(offs, tstring);
sprintf(tstring, "%" PRId64 ",", uc[j]);
strcat(lbounds, tstring);
}
printf("rank=%d, name=%s, gbounds=%s: lbounds=%s: offs=%s \n",rank,g->var_namelist[i],gbounds, lbounds, offs);
}
used_chunk -= remain_chunk; //you get the actual used_chunk here.
elements_defined += used_chunk;
if(remain_chunk!=0){
rS(v, ts, used_chunk, rank); //advance ts by used_chunk.
for(k=0; k<10; k++)
tc[k] = 1;
calcC(remain_chunk, v, tc); //based on the remain_chunk, calculate new tc chunk block remained to process.
}
adios_groupsize+= used_chunk*element_size+2*v->ndim*8;
}while(remain_chunk!=0);
current_step += size*chunk_size; //once a whole chunk_size is processed, advance the current_step in roll-robin manner.
if(current_step<total_size){ //advance s in the same way.
rS(v, s, size*chunk_size, rank);
}
}
//beside checkOverflow above, here you check whether or not the total number of elements processed across processes matches
//the total number of elements in the original vector.
if(DEBUG){
uint64_t* sb = (uint64_t *) malloc(sizeof(uint64_t));
uint64_t* rb = (uint64_t *) malloc(sizeof(uint64_t));
sb[0] = elements_defined;
MPI_Reduce(sb,rb,1,MPI_UNSIGNED_LONG_LONG,MPI_SUM,0, comm);
if(rank==0 && rb[0]!=total_size)
printf("some array define mismatch. please use debug mode\n");
free(sb); free(rb);
}
}
free (var_name);
free (var_path);
} // finished declaring all of the variables
// Now we can define the attributes....
for (i = 0; i < g->attrs_count; i++) {
enum ADIOS_DATATYPES atype;
int asize;
void *adata;
adios_get_attr_byid (g, i, &atype, &asize, &adata);
// if (DEBUG) printf("attribute name=%s\n",g->attr_namelist[i]);
adios_define_attribute(new_adios_group,g->attr_namelist[i],"",atype,adata,0);
}
/*------------------------------ NOW WE WRITE -------------------------------------------- */
// Now we have everything declared... now we need to write them out!!!!!!
if (WRITEME==1) {
// open up the file for writing....
if (DEBUG) printf("rank=%d, opening file = %s, with group %s, size=%" PRId64 "\n",rank,argv[2],f->group_namelist[gidx],adios_groupsize);
if(TIMING==100)
start_time[0] = MPI_Wtime();
adios_open(&m_adios_file, f->group_namelist[gidx],argv[2],"w",comm);
adios_group_size( m_adios_file, adios_groupsize, &adios_totalsize);
//get both the total adios_totalsize and total adios_groupsize summed across processes.
uint64_t* sb = (uint64_t *) malloc(sizeof(uint64_t));;
uint64_t* rb = (uint64_t *) malloc(sizeof(uint64_t));
sb[0] = adios_groupsize;
MPI_Reduce(sb,rb,1,MPI_UNSIGNED_LONG_LONG,MPI_SUM,0, comm);
uint64_t* sb2 = (uint64_t *) malloc(sizeof(uint64_t));;
uint64_t* rb2 = (uint64_t *) malloc(sizeof(uint64_t));
sb2[0] = adios_totalsize;
MPI_Reduce(sb2,rb2,1,MPI_UNSIGNED_LONG_LONG,MPI_SUM,0, comm);
if(rank==0){
printf("total adios_totalsize = %" PRId64 "\n", *rb2);
printf("total adios_groupsize = %" PRId64 "\n", *rb);
}
free(sb); free(rb); free(sb2); free(rb2);
if (TIMING==100) {
end_time[0] = MPI_Wtime();
total_time[0]+=end_time[0] - start_time[0]; //variable definition time taken
}
// now we have to write out the variables.... since they are all declared now
// This will be the place we actually write out the data!!!!!!!!
for (i = 0; i < g->vars_count; i++) {
ADIOS_VARINFO * v = adios_inq_var_byid (g, i);
getbasename (g->var_namelist[i], &var_path, &var_name);
if (v->ndim == 0)
{
if (DEBUG) {
printf ("ADIOS WRITE SCALAR: rank=%d, name=%s value=",
rank,g->var_namelist[i]);
print_data (v->value, 0, v->type);
printf ("\n");
}
if (TIMING==100) {
start_time[2] = MPI_Wtime();
}
adios_write(m_adios_file,g->var_namelist[i],v->value);
if (TIMING==100) {
end_time[2] = MPI_Wtime();
total_time[2]+=end_time[2] - start_time[2]; //IO write time...
}
}
else
{
for(j=0; j<v->ndim; j++){
s[j] = 0;
c[j] = 1;
}
getTypeInfo( v->type, &element_size);
uint64_t total_size = 1;
for (ii=0;ii<v->ndim;ii++)
total_size*=v->dims[ii];
chunk_size = calcChunkSize(total_size, read_buffer*1024*1024/element_size, size);
calcC(chunk_size, v, c);
chunk_size = 1;
for(ii=0; ii<v->ndim; ii++)
chunk_size *= c[ii];
uint64_t current_step = rank*chunk_size;
if(current_step<total_size)
rS(v, s, current_step, rank);
uint64_t elements_written = 0;
while(current_step < total_size)
{
uint64_t ts[] = {0,0,0,0,0,0,0,0,0,0};
arrCopy(s, ts);
uint64_t remain_chunk = chunk_size;
if(current_step+chunk_size>total_size)
remain_chunk = total_size-current_step;
uint64_t tc[] = {1,1,1,1,1,1,1,1,1,1};
arrCopy(c, tc);
do{
uint64_t uc[] = {1,1,1,1,1,1,1,1,1,1};
uint64_t used_chunk = remain_chunk;
remain_chunk = checkBound(v, ts, tc, uc, remain_chunk);
checkOverflow(1, v, ts, uc);
used_chunk -= remain_chunk;
elements_written += used_chunk;
//allocated space for data read-in
data = (void *) malloc(used_chunk*element_size);
if (TIMING==100) {
start_time[1] = MPI_Wtime();
}
if(PERFORMANCE_CHECK) printf("rank=%d, read start\n",rank);
bytes_read = adios_read_var_byid(g,v->varid,ts,uc,data);
if(PERFORMANCE_CHECK) printf("rank=%d, read end\n",rank);
if (TIMING==100) {
end_time[1] = MPI_Wtime();
total_time[1]+=end_time[1] -start_time[1]; //IO read time
}
if (DEBUG)
printf ("ADIOS WRITE: rank=%d, name=%s datasize=%" PRId64 "\n",rank,g->var_namelist[i],bytes_read);
if (TIMING==100) {
start_time[2] = MPI_Wtime();
}
if (DEBUG){
printf("rank=%d, write ts=",rank);
int k;
for(k=0; k<v->ndim; k++)
printf("%" PRId64 ",", ts[k]);
printf(" uc=");
for(k=0; k<v->ndim; k++)
printf("%" PRId64 ",", uc[k]);
printf("\n");
}
//local bounds and offets placeholders are not written out with actual values.
if(PERFORMANCE_CHECK) printf("rank=%d, adios write start\n", rank);
for(k=0; k<v->ndim; k++){
//sprintf(tstring, "ldim%d_%s", k, var_name);
sprintf(tstring, "ldim%d", k);
if (DEBUG) {
printf ("ADIOS WRITE DIMENSION: rank=%d, name=%s value=",
rank,tstring);
print_data (&uc[k], 0, adios_unsigned_long);
printf ("\n");
}
adios_write(m_adios_file, tstring, &uc[k]);
//sprintf(tstring, "offs%d_%s", k, var_name);
sprintf(tstring, "offs%d", k);
if (DEBUG) {
printf ("ADIOS WRITE OFFSET: rank=%d, name=%s value=",
rank,tstring);
print_data (&ts[k], 0, adios_unsigned_long);
printf ("\n");
}
adios_write(m_adios_file, tstring, &ts[k]);
}
adios_write(m_adios_file,g->var_namelist[i],data);
if(PERFORMANCE_CHECK) printf("rank=%d, adios write end\n", rank);
if (TIMING==100) {
end_time[2] = MPI_Wtime();
total_time[2]+=end_time[2] - start_time[2]; //IO write time
}
free(data);
if(remain_chunk!=0){
rS(v, ts, used_chunk, rank);
for(k=0; k<10; k++)
tc[k] = 1;
calcC(remain_chunk, v, tc);
}
}while(remain_chunk!=0);
current_step += size*chunk_size;
if(current_step<total_size)
rS(v, s, size*chunk_size,rank);
}
if(DEBUG){
uint64_t* sb = (uint64_t *) malloc(sizeof(uint64_t));;
uint64_t* rb = (uint64_t *) malloc(sizeof(uint64_t));
sb[0] = elements_written;
MPI_Reduce(sb,rb,1,MPI_UNSIGNED_LONG_LONG,MPI_SUM,0, comm);
if(rank==0 && rb[0]!=total_size)
printf("some array read mismatch. please use debug mode\n");
free(sb); free(rb);
}
}
free (var_name);
free (var_path);
}// end of the writing of the variable..
if (TIMING==100) {
start_time[3] = MPI_Wtime();
}
if(PERFORMANCE_CHECK) printf("rank=%d, adios_close start\n", rank);
adios_close(m_adios_file);
if(PERFORMANCE_CHECK) printf("rank=%d, adios_close end\n", rank);
if (TIMING==100) {
end_time[3] = MPI_Wtime();
total_time[3]+=end_time[3] - start_time[3];
}
adios_gclose(g);
} //end of WRITEME
} // end of all of the groups
if(rank==0)
printf("conversion done!\n");
if(TIMING==100)
start_time[4] = MPI_Wtime();
adios_fclose(f);
if(TIMING==100){
end_time[4] = MPI_Wtime();
total_time[4] = total_time[4]+end_time[4]-start_time[4];
}
adios_finalize(rank);
// now, we write out the timing data, for each category, we give max, min, avg, std, all in seconds, across all processes.
if(TIMING==100){
// 0: adios_open, adios_group_size
// 1: the total time to read in the data
// 2: times around each write (will only work if we do NOT buffer....
// 3: the time in the close
// 4: fopen, fclose
// 5: total time
end_time[5] = MPI_Wtime();
total_time[5] = end_time[5] - start_time[5];
double sb[7];
sb[0] = total_time[1]; sb[1] = total_time[4]; //read_var, fopen+fclose
sb[2] = sb[0]+sb[1];
sb[3] = total_time[0]; sb[4] = total_time[2]+total_time[3]; //adios_open+adios_group_size, write+close
sb[5] = sb[3]+sb[4];
sb[6] = total_time[5]; //total
double * rb = NULL;
if(rank==0)
rb = (double *)malloc(size*7*sizeof(double));
//MPI_Barrier(comm);
MPI_Gather(sb, 7, MPI_DOUBLE, rb, 7, MPI_DOUBLE, 0, comm);
if(rank==0){
double read_avg1 = 0;
double read_avg2 = 0;
double tread_avg = 0;
double write_avg1 = 0;
double write_avg2 = 0;
double twrite_avg = 0;
double total_avg = 0;
for(j=0; j<size; j++){
read_avg1 += rb[7*j];
read_avg2 += rb[7*j+1];
tread_avg += rb[7*j+2];
write_avg1 += rb[7*j+3];
write_avg2 += rb[7*j+4];
twrite_avg += rb[7*j+5];
total_avg += rb[7*j+6];
}
read_avg1 /= size;
read_avg2 /= size;
tread_avg /= size;
write_avg1 /= size;
write_avg2 /= size;
twrite_avg /= size;
total_avg /= size;
double read1_max = rb[0];
double read1_min = rb[0];
double read1_std = rb[0]-read_avg1; read1_std *= read1_std;
double read2_max = rb[1];
double read2_min = rb[1];
double read2_std = rb[1]-read_avg2; read2_std *= read2_std;
double tread_max = rb[2];
double tread_min = rb[2];
double tread_std = rb[2]-tread_avg; tread_std *= tread_std;
double write1_max = rb[3];
double write1_min = rb[3];
double write1_std = rb[3]-write_avg1; write1_std *= write1_std;
double write2_max = rb[4];
double write2_min = rb[4];
double write2_std = rb[4]-write_avg2; write2_std *= write2_std;
double twrite_max = rb[5];
double twrite_min = rb[5];
double twrite_std = rb[5]-twrite_avg; twrite_std *= twrite_std;
double total_max = rb[6];
double total_min = rb[6];
double total_std = rb[6]-total_avg; total_std *= total_std;
for(j=1; j<size; j++){
if(rb[7*j]>read1_max)
read1_max = rb[7*j];
else if(rb[7*j]<read1_min)
read1_min = rb[7*j];
double std = rb[7*j]-read_avg1; std *= std;
read1_std += std;
if(rb[7*j+1]>read2_max)
read2_max = rb[7*j+1];
else if(rb[7*j+1]<read2_min)
read2_min = rb[7*j+1];
std = rb[7*j+1]-read_avg2; std *= std;
read2_std += std;
if(rb[7*j+2]>tread_max)
tread_max = rb[7*j+2];
else if(rb[7*j+2]<tread_min)
tread_min = rb[7*j+2];
std = rb[7*j+2]-tread_avg; std *= std;
tread_std += std;
if(rb[7*j+3]>write1_max)
write1_max = rb[7*j+3];
else if(rb[7*j+3]<write1_min)
write1_min = rb[7*j+3];
std = rb[7*j+3]-write_avg1; std *= std;
write1_std += std;
if(rb[7*j+4]>write2_max)
write2_max = rb[7*j+4];
else if(rb[7*j+4]<write2_min)
write2_min = rb[7*j+4];
std = rb[7*j+4]-write_avg2; std *= std;
write2_std += std;
if(rb[7*j+5]>twrite_max)
twrite_max = rb[7*j+5];
else if(rb[7*j+5]<twrite_min)
twrite_min = rb[7*j+5];
std = rb[7*j+5]-twrite_avg; std *= std;
twrite_std += std;
if(rb[7*j+6]>total_max)
total_max = rb[7*j+6];
else if(rb[7*j+6]<total_min)
total_min = rb[7*j+6];
std = rb[7*j+6]-total_avg; std *= std;
total_std += std;
}
read1_std /= size; read1_std = sqrt(read1_std);
read2_std /= size; read2_std = sqrt(read2_std);
tread_std /= size; tread_std = sqrt(tread_std);
write1_std /= size; write1_std = sqrt(write1_std);
write2_std /= size; write2_std = sqrt(write2_std);
twrite_std /= size; twrite_std = sqrt(twrite_std);
total_std /= size; total_std = sqrt(total_std);
printf("---type--- max\tmin\tavg\tstd\n");
printf("---read_var--- %lf\t%lf\t%lf\t%lf\n", read1_max, read1_min, read_avg1, read1_std);
printf("---fopen+fclose--- %lf\t%lf\t%lf\t%lf\n", read2_max, read2_min, read_avg2, read2_std);
printf("---total_read--- %lf\t%lf\t%lf\t%lf\n", tread_max, tread_min, tread_avg, tread_std);
printf("---adios_open+adios_groupsize--- %lf\t%lf\t%lf\t%lf\n", write1_max, write1_min, write_avg1, write1_std);
printf("---write+close--- %lf\t%lf\t%lf\t%lf\n", write2_max, write2_min, write_avg2, write2_std);
printf("---total_write--- %lf\t%lf\t%lf\t%lf\n", twrite_max, twrite_min, twrite_avg, twrite_std);
printf("---total--- %lf\t%lf\t%lf\t%lf\n", total_max, total_min, total_avg, total_std);
free(rb);
}
}
// if (TIMING==100 && rank==0) {
// printf("------------------------------------------------------------------\n");
// printf("Define variables = %lf\n",total_time[0]);
// printf("Read variables = %lf\n",total_time[1]);
// printf("Write variables = %lf\n",total_time[2]);
// printf("Close File for write = %lf\n",total_time[3]);
// printf("Total write time = %lf\n",total_time[2] + total_time[3]);
// for (itime=0;itime<timers-1;itime++)
// total_time[timers-1]+=total_time[itime];
// printf("Total I/O time = %lf\n",total_time[timers-1]);
// }
MPI_Finalize();
return(0);
}
static int run_amz_file(clamz_downloader *dl,
const clamz_config *cfg,
FILE *amzfile,
const char *fname)
{
char *inbuf;
unsigned char *xml;
size_t sz;
clamz_playlist *pl;
int i;
int status, rv = 0;
char *logname;
FILE *logfile;
sz = 0;
inbuf = NULL;
while (!feof(amzfile) && !ferror(amzfile)) {
if (inbuf)
inbuf = realloc(inbuf, (sz + 1024) * sizeof(char));
else
inbuf = malloc((sz + 1024) * sizeof(char));
sz += fread(&inbuf[sz], 1, 1024, amzfile);
}
if (amzfile != stdin)
fclose(amzfile);
if (cfg->printonly && cfg->printasxml) {
xml = decrypt_amz_file(inbuf, sz, fname);
if (!xml) {
free(inbuf);
return 2;
}
printf("%s", xml);
free(xml);
free(inbuf);
return 0;
}
else {
if (!cfg->printonly) {
if (write_backup_file(inbuf, sz, getbasename(fname))) {
free(inbuf);
return 3;
}
}
pl = new_playlist();
if (read_amz_file(pl, inbuf, sz, fname)) {
free(inbuf);
free_playlist(pl);
return 2;
}
if (!cfg->printonly) {
logname = get_config_file_name("logs", getbasename(fname), ".log");
if (!logname) {
free(inbuf);
return 1;
}
logfile = fopen(logname, "w");
if (!logfile) {
perror(logname);
free(logname);
free(inbuf);
return 3;
}
free(logname);
set_download_log_file(dl, logfile);
}
else {
logfile = NULL;
}
if (cfg->printonly || cfg->verbose)
print_pl_info(pl, fname);
for (i = 0; i < pl->num_tracks; i++) {
if (cfg->printonly || cfg->verbose)
print_tr_info(pl->tracks[i], i + 1);
status = download_track(dl, pl->tracks[i]);
if (!rv)
rv = status;
fputc('\n', stderr);
}
set_download_log_file(dl, NULL);
free(inbuf);
free_playlist(pl);
if (logfile)
fclose(logfile);
return rv;
}
}
int main(void)
{
int ch;
char comm[256];
struct stat st;
printf("RECV -- 99/4A ROM dump receiver by Edward Swartz.\n\n"
"Companion to TRANS on the 99/4A.\n\n");
if (stat("MODULES",&st) || stat("ROMS",&st) || stat("MODULES.INF",&st))
{
printf("This program needs to be run from the directory where V9t9.EXE,\n"
"MODULES.INF, etc., are located. (Use RECV.BAT from that directory.)\n");
exit(1);
}
printf("In order to use this program, TRANS must be running on a\n"
"99/4A system connected to this PC by a serial cable.\n"
"It's best to start the 99/4A TRANS before running this program.\n\n"
"Press <Enter> to continue, or <Esc> to exit RECV:");
do
ch=getch();
while (ch!=13 && ch!=27);
if (ch==27)
exit(1);
do
{
printf("\n\n\nEnter the DOS serial port you are using to connect\n"
"to the 99/4A. Valid values are 1-4.\n\n: ");
comm[0]=2;
port=atoi(cgets(comm));
} while (port<1 || port>4);
do
{
printf("\n\n\nEnter the IRQ of COM%d. Typical value is %d.\n\n: ",
port, ((port==1 || port==3) ? 4 : 3));
comm[0]=2;
irq=atoi(cgets(comm));
} while (irq<0 || irq>7);
port--; // fix port to 0-3
do
{
printf("\n\n\nEnter the baud rate you used to initialize the RS232\n"
"in TRANS. Valid baud rates are\n"
"\t110 300 600 1200 2400 4800 9600.\n\n: ");
comm[0]=5;
baudrate=atoi(cgets(comm));
} while (baudrate<110 || baudrate>9600);
com_init(port,baudrate,irq);
printf("\n\nRECV is ready to go.\n\n"
"Press a key at any time to abort.\n\n");
while (!kbhit())
{
// ch=bioscom(_COM_RECEIVE,0,port);
buf_init();
if (!gs(comm))
switch(comm[0])
{
case 'S': ps("G");
if (dump())
Err();
break;
case 'M': getbasename();
ps("N");
break;
case 'N': closemodule();
ps("M");
break;
default: Err();
break;
}
else
Err();
}
com_off();
return 0;
}
int process_metadata(int step)
{
int retval = 0;
int i, j;
char gdims[256], ldims[256], offs[256];
uint64_t sum_count;
ADIOS_VARINFO *v; // shortcut pointer
if (step > 1)
{
// right now, nothing to prepare in later steps
print("Step %d. return immediately\n",step);
return 0;
}
/* First step processing */
// get groupname of stream, then declare for output
adios_get_grouplist(f, &group_namelist);
print0("Group name is %s\n", group_namelist[0]);
adios_declare_group(&gh,group_namelist[0],"",adios_flag_yes);
varinfo = (VarInfo *) malloc (sizeof(VarInfo) * f->nvars);
if (!varinfo) {
print("ERROR: rank %d cannot allocate %lu bytes\n", rank, sizeof(VarInfo)*f->nvars);
return 1;
}
write_total = 0;
largest_block = 0;
// Decompose each variable and calculate output buffer size
for (i=0; i<f->nvars; i++)
{
print0 ("Get info on variable %d: %s\n", i, f->var_namelist[i]);
varinfo[i].v = adios_inq_var_byid (f, i);
v = varinfo[i].v; // just a shortcut
if (v == NULL) {
print ("rank %d: ERROR: Variable %s inquiry failed: %s\n",
rank, f->var_namelist[i], adios_errmsg());
return 1;
}
// print variable type and dimensions
print0(" %-9s %s", adios_type_to_string(v->type), f->var_namelist[i]);
if (v->ndim > 0) {
print0("[%llu", v->dims[0]);
for (j = 1; j < v->ndim; j++)
print0(", %llu", v->dims[j]);
print0("] :\n");
} else {
print0("\tscalar\n");
}
// determine subset we will write
decompose (numproc, rank, v->ndim, v->dims, decomp_values,
varinfo[i].count, varinfo[i].start, &sum_count);
varinfo[i].writesize = sum_count * adios_type_size(v->type, v->value);
if (varinfo[i].writesize != 0) {
write_total += varinfo[i].writesize;
if (largest_block < varinfo[i].writesize)
largest_block = varinfo[i].writesize;
}
}
// determine output buffer size and allocate it
uint64_t bufsize = write_total + f->nvars*128 + f->nattrs*32 + 1024;
if (bufsize > max_write_buffer_size) {
print ("ERROR: rank %d: write buffer size needs to hold about %llu bytes, "
"but max is set to %d\n", rank, bufsize, max_write_buffer_size);
return 1;
}
print0 ("Rank %d: allocate %llu MB for output buffer\n", rank, bufsize/1048576+1);
adios_allocate_buffer (ADIOS_BUFFER_ALLOC_NOW, bufsize/1048576+1);
// allocate read buffer
bufsize = largest_block + 128;
if (bufsize > max_read_buffer_size) {
print ("ERROR: rank %d: read buffer size needs to hold at least %llu bytes, "
"but max is set to %d\n", rank, bufsize, max_read_buffer_size);
return 1;
}
print0 ("Rank %d: allocate %g MB for input buffer\n", rank, (double)bufsize/1048576.0);
readbuf = (char *) malloc ((size_t)bufsize);
if (!readbuf) {
print ("ERROR: rank %d: cannot allocate %llu bytes for read buffer\n",
rank, bufsize);
return 1;
}
// Select output method
adios_select_method (gh, wmethodname, wmethodparams, "");
// Define variables for output based on decomposition
char *vpath, *vname;
for (i=0; i<f->nvars; i++)
{
v = varinfo[i].v;
if (varinfo[i].writesize != 0) {
// define variable for ADIOS writes
getbasename (f->var_namelist[i], &vpath, &vname);
if (v->ndim > 0)
{
int64s_to_str (v->ndim, v->dims, gdims);
int64s_to_str (v->ndim, varinfo[i].count, ldims);
int64s_to_str (v->ndim, varinfo[i].start, offs);
print ("rank %d: Define variable path=\"%s\" name=\"%s\" "
"gdims=%s ldims=%s offs=%s\n",
rank, vpath, vname, gdims, ldims, offs);
adios_define_var (gh, vname, vpath, v->type, ldims, gdims, offs);
}
else
{
print ("rank %d: Define scalar path=\"%s\" name=\"%s\"\n",
rank, vpath, vname);
adios_define_var (gh, vname, vpath, v->type, "", "", "");
}
free(vpath);
free(vname);
}
}
if (rank == 0)
{
// get and define attributes
enum ADIOS_DATATYPES attr_type;
void * attr_value;
char * attr_value_str;
int attr_size;
for (i=0; i<f->nattrs; i++)
{
adios_get_attr_byid (f, i, &attr_type, &attr_size, &attr_value);
attr_value_str = (char *)value_to_string (attr_type, attr_value, 0);
getbasename (f->attr_namelist[i], &vpath, &vname);
if (vpath && !strcmp(vpath,"/__adios__")) {
// skip on /__adios/... attributes
print ("rank %d: Ignore this attribute path=\"%s\" name=\"%s\" value=\"%s\"\n",
rank, vpath, vname, attr_value_str);
} else {
adios_define_attribute (gh, vname, vpath,
attr_type, attr_value_str, "");
print ("rank %d: Define attribute path=\"%s\" name=\"%s\" value=\"%s\"\n",
rank, vpath, vname, attr_value_str);
free (attr_value);
}
}
}
return retval;
}
int main(int argc, char **argv)
{
if(argc < 3) disphelp(getbasename(argv[0]));
return eval(-1, 1, argc, argv)-1;
}
bool assemblekeybif(char *fname, uint8 **mem, uint32 *size)
{
int pr;
std::string bfname, rfname, resref;
std::vector<uint8 *> datas;
uint16 drives=0;
uint32 asize;
uint64 tmp;
NWNFileType restype=NWN_FILE_UNDEFINED;
bool newbif=true, withCd=false, exploded=false;
XMLFile xml;
KEYFile key;
BIFFile bif;
if(xml.open(fname))
{
std::cerr << "Error opening file \"" << fname << "\": ";
std::cerr << xml.getStrError() << "\n";
return false;
}
while((pr = xml.parse()) >= 0)
{
switch(pr)
{
case 0:
if(!mem && (xml.section == ".key.filename"))
key.filename = xml.value;
else if(xml.section == ".key.filetype")
{
if((key.type = key.getFileResTypeByExt(xml.value)) == -1)
return printxmlerr(xml, "Unknown filetype");
if(!key.typeIsCorrect()) return printxmlerr(xml, "No KEY type");
}
else if(xml.section == ".key.buildyear")
{
sscanf(xml.value.c_str(), S_UINT64, &tmp);
key.buildyear = (uint32) (tmp - 1900);
}
else if(xml.section == ".key.buildday")
{
sscanf(xml.value.c_str(), S_UINT64, &tmp);
key.buildday = (uint32) tmp;
}
else if(xml.section == ".key.bif.filename")
bfname = xml.value;
else if(xml.section == ".key.bif.drives")
{
sscanf(xml.value.c_str(), S_UINT64, &tmp);
drives = (uint16) tmp;
}
else if(xml.section == ".key.bif.variable.resource.resref")
resref = xml.value;
else if(xml.section == ".key.bif.variable.resource.filename")
rfname = xml.value;
else if(xml.section == ".key.bif.variable.resource.restype")
{
if((restype = key.getFileResTypeByExt(xml.value)) == -1)
return printxmlerr(xml, "Unknown filetype");
}
else if(xml.section == ".key.bif.variable.resource.withCd")
{
sscanf(xml.value.c_str(), S_UINT64, &tmp);
withCd = (bool) tmp;
}
else if(xml.section == ".key.bif.variable.resource.exploded")
{
sscanf(xml.value.c_str(), S_UINT64, &tmp);
exploded = (bool) tmp;
}
break;
case 1:
if(xml.section == ".bif.fixed")
return printxmlerr(xml, "Fixed resources aren't supported");
else if((xml.section != ".?xml") && (xml.section != ".key") &&
(getpos(validkeybiftags, KEYBIFTAGS, xml.section) == -1))
return printxmlerr(xml, "Invalid tag");
break;
case 2:
if(xml.value == ".key.bif")
{
bfname = bif.filename;
if(bif.endwrite()) return printxmlerr(xml, bif.getStrError());
if(bif.open(bfname)) return printxmlerr(xml, bif.getStrError());
key.fsizes[key.bifcount-1] = bif.getFileSize();
bif.deInit();
newbif = true;
bfname.clear();
drives = 0;
}
else if(xml.value == ".key.bif.variable.resource")
{
if(newbif)
{
if(bfname.empty()) return printxmlerr(xml, "Missing filename");
if(key.addBif(bfname, drives, 0))
return printxmlerr(xml, key.getStrError());
if(bif.beginWrite(getbasename((char*)bfname.c_str())))
return printxmlerr(xml, bif.getStrError());
bif.type = NWN_FILE_BIF;
newbif = false;
}
if(resref.empty()) return printxmlerr(xml, "Missing resref");
if(rfname.empty()) return printxmlerr(xml, "Missing filename");
if(restype == NWN_FILE_UNDEFINED)
return printxmlerr(xml, "Missing restype");
if(exploded)
{
datas.resize((tmp=datas.size())+1);
if(!(datas[tmp] = expd(xml, bif.getBaseType(restype), rfname, asize)))
return false;
if(bif.writeData(restype, withCd, true, datas[tmp], asize, false))
return printxmlerr(xml, bif.getStrError());
}
else
if(bif.writeData(restype, withCd, true, rfname))
return printxmlerr(xml, bif.getStrError());
if(key.addResource(key.bifcount-1, resref, restype, bif.vrescount-1))
return printxmlerr(xml, key.getStrError());
resref.clear();
rfname.clear();
restype = NWN_FILE_UNDEFINED;
withCd = exploded = false;
}
break;
}
}
if(pr != -6) return printxmlerr(xml, xml.getStrError());
xml.close();
if(!mem)
{
if(key.filename.empty()) return printxmlerr(xml, "Missing filename");
else if(key.write(key.filename)) return printnwnerr(key);
}
else if(key.write(mem, size)) return printnwnerr(key);
for(uint32 i=0;i<datas.size();i++) free(datas[i]);
return true;
}
