bool
ADIOSFileObject::ReadVariable(const std::string &nm,
                              int ts,
                              vtkDataArray **array)
{
    debug5<<"ADIOSFileObject::ReadVariable("<<nm<<" time= "<<ts<<")"<<endl;
    Open();

    varIter vi = variables.find(nm);
    if (vi == variables.end())
    {
        debug5<<"Variable "<<nm<<" not found."<<endl;
        return false;
    }
    ADIOSVar v = vi->second;

    int tupleSz = adios_type_size(v.type, NULL);
    *array = ADIOSFileObject::AllocateArray(v.type);

    int ntuples = 1;
    uint64_t start[4] = {0,0,0,0}, count[4] = {0,0,0,0};
    v.GetReadArrays(ts, start, count, &ntuples);
    
    (*array)->SetNumberOfTuples(ntuples);
    void *data = (*array)->GetVoidPointer(0);

    debug5<<"ARR: adios_read_var:"<<endl<<v<<endl;
    OpenGroup(v.groupIdx);

    uint64_t retval = adios_read_var_byid(gps[v.groupIdx], v.varid, start, count, data);
        
    CloseGroup(v.groupIdx);

    return (retval > 0);
}
void
ADIOSFileObject::GetTimes(std::string &varNm, std::vector<double> &times)
{
    Open();
    times.resize(0);
    
    varIter vi = variables.find(varNm);
    if (vi == variables.end())
    {
        debug5<<"Variable "<<varNm<<" not found."<<endl;
        return;
    }
    ADIOSVar v = vi->second;

    int tupleSz = adios_type_size(v.type, NULL);
    uint64_t start[4] = {0,0,0,0}, count[4] = {0,0,0,0};

    count[0] = v.count[0];
    int ntuples = v.count[0];
    
    void *readData = malloc(ntuples*tupleSz);
    OpenGroup(v.groupIdx);
    uint64_t retval = adios_read_var_byid(gps[v.groupIdx], v.varid, start, count, readData);
    CloseGroup(v.groupIdx);

    if (retval > 0)
    {
        times.resize(ntuples);
        ConvertTo(&times[0], ntuples, v.type, readData);
    }
    free(readData);
}
bool
ADIOSFileObject::ReadVariable(const std::string &nm,
                              int ts,
                              vtkFloatArray **array)
{
    debug5<<"ADIOSFileObject::ReadVariable("<<nm<<" time= "<<ts<<")"<<endl;
    Open();

    varIter vi = variables.find(nm);
    if (vi == variables.end())
    {
        debug5<<"Variable "<<nm<<" not found."<<endl;
        return false;
    }
    ADIOSVar v = vi->second;

    int tupleSz = adios_type_size(v.type, NULL);
    *array = vtkFloatArray::New();

    int ntuples = 1;
    uint64_t start[4] = {0,0,0,0}, count[4] = {0,0,0,0};
    v.GetReadArrays(ts, start, count, &ntuples);
    
    (*array)->SetNumberOfTuples(ntuples);
    float *data = (float *)(*array)->GetVoidPointer(0);
    void *readData = (void *)data;

    bool convertData = (v.type != adios_real);
    if (convertData)
        readData = malloc(ntuples*tupleSz);

    debug5<<"ARR: adios_read_var:"<<endl<<v<<endl;
    OpenGroup(v.groupIdx);

    uint64_t retval = adios_read_var_byid(gps[v.groupIdx], v.varid, start, count, readData);
    CloseGroup(v.groupIdx);

    if (convertData)
    {
        if (retval > 0)
            ConvertTo(data, ntuples, v.type, readData);
        free(readData);
    }

    return (retval > 0);
}
bool
ADIOSFileObject::ReadCoordinates(const std::string &nm,
                                 int ts,
                                 vtkPoints **pts)
{
    Open();
    
    varIter vi = variables.find(nm);
    if (vi == variables.end())
    {
        debug5<<"Variable "<<nm<<" not found."<<endl;
        return false;
    }
    ADIOSVar v = vi->second;
    
    *pts = ADIOSFileObject::AllocatePoints(v.type);

    int ntuples = 1;
    uint64_t start[4] = {0,0,0,0}, count[4] = {0,0,0,0};
    
    v.GetReadArrays(ts, start, count, &ntuples);
    ntuples /= v.dim;
    (*pts)->SetNumberOfPoints(ntuples);

    void *data = (*pts)->GetVoidPointer(0), *data2 = NULL;
    void *readData = data;
    
    if (v.dim < 3)
    {
        if (v.type == adios_real)
            data2 = malloc(ntuples*3*sizeof(float));
        else if (v.type == adios_double)
            data2 = malloc(ntuples*3*sizeof(double));
        else if (v.type == adios_integer)
            data2 = malloc(ntuples*3*sizeof(int));
        readData = data2;
    }
    
    OpenGroup(v.groupIdx);
    
    debug5<<"adios_read_var:"<<endl<<v<<endl;
    uint64_t retval = adios_read_var_byid(gps[v.groupIdx], v.varid, start, count, readData);
    if (retval > 0)
    {
        if (readData != data)
        {
            int i, j, n = ntuples*3;
            for (i=0, j=0; i < n; i += 3, j += v.dim)
            {
                int k;
                for (k = 0; k < v.dim; k++)
                    ((double *)data)[i+k] = ((double *)data2)[j+k];
                for ( ; k < 3; k++)
                    ((double *)data)[i+k] = 0.0;
            }
            free(data2);
        }
    }

    CloseGroup(v.groupIdx);    
    return (retval > 0);
}
Ejemplo n.º 5
0
int main (int argc, char ** argv) {
    //For varriable definitions:
    //gbounds = global bounds string, lbounds = local bounds string, offs = offset string, tstring = temp string to hold temperary stuff
    char       gbounds[1007], lbounds[1007], offs[1007],tstring[100];
    //size = number of cores,  gidx = adios group index
    int        rank, size, gidx, i, j, k, ii;
    //data = pointer to read-in data
    void       * data = NULL;
    uint64_t   s[] = {0,0,0,0,0,0,0,0,0,0};  //starting offset
    uint64_t   c[] = {1,1,1,1,1,1,1,1,1,1};  //chunk block array
    uint64_t   bytes_read = 0;
    int        element_size;
    int64_t    new_adios_group, m_adios_file;
    uint64_t   var_size;  //portion_bound,
    uint64_t   adios_groupsize, adios_totalsize;
    int        read_buffer;        //possible maximum size you the user would like for each chunk in MB
    int           write_buffer = 1536;  //actual buffer size you use in MB
    int        itime;
    int        WRITEME=1;
    uint64_t   chunk_size;   //chunk size in # of elements
    char      *var_path, *var_name; // full path cut into dir path and name
    MPI_Init(&argc,&argv);
    MPI_Comm_rank(comm,&rank);
    MPI_Comm_size(comm,&size);

    // timing numbers
    // we will time:
    // 0: adios_open, adios_group_size
    // 1: the total time to read in the data
    // 2: times around each write (will only work if we do NOT buffer....
    // 3: the time in the close
    // 4: fopen, fclose
    // 5: total time
    // timers: the total I/O time
    int        timers = 6;
    double     start_time[timers], end_time[timers], total_time[timers];

    if (TIMING==100) {
        for (itime=0;itime<timers;itime++) {
            start_time[itime] = 0;
            end_time[itime] = 0;
            total_time[itime]=0;
        }
        //MPI_Barrier(MPI_COMM_WORLD);
        start_time[5] = MPI_Wtime();
    }

    if(rank==0)
        printf("converting...\n");

    if (argc < 5) {
        if (rank==0) printf("Usage: %s <BP-file> <ADIOS-file> read_buffer(MB) write_buffer(MB) METHOD (LUSTRE_strip_count) (LUSTRE_strip_size) (LUSTRE_block_size)\n", argv[0]);
        return 1;
    }



    if(TIMING==100)
        start_time[4] = MPI_Wtime();
    ADIOS_FILE * f = adios_fopen (argv[1], MPI_COMM_SELF);
    if(TIMING==100){
        end_time[4] = MPI_Wtime();
        total_time[4] = end_time[4]-start_time[4];
    }
    adios_init_noxml(comm); // no xml will be used to write the new adios file
    read_buffer = atoi(argv[3]);
    write_buffer = atoi(argv[4]);
    adios_allocate_buffer (ADIOS_BUFFER_ALLOC_NOW, write_buffer); // allocate MB buffer



    if (f == NULL) {
        printf("rank=%d, file cant be opened\n", rank);
        if (DEBUG) printf ("%s\n", adios_errmsg());
        return -1;
    }


    for (gidx = 0; gidx < f->groups_count; gidx++) {    //group part
        adios_groupsize = 0;
        ADIOS_GROUP * g = adios_gopen (f, f->group_namelist[gidx]);


        if (g == NULL) {
            if (DEBUG) printf ("%s\n", adios_errmsg());
            printf("rank %d: group cannot be opened.\n", rank);
            return -1;
        }
        /* First create all of the groups */
        // now I need to create this group in the file that will be written

        adios_declare_group(&new_adios_group,f->group_namelist[gidx],"",adios_flag_yes);


        if(strcmp(argv[5],"MPI_LUSTRE")!=0)   //see whether or not the user uses MPI_LUSTRE method
            adios_select_method (new_adios_group, argv[5], "", "");  //non-MPI_LUSTRE methods... like MPI, POSIX....
        else{
            char lustre_pars[1000];
            strcpy(lustre_pars, "");
            strcat(lustre_pars, "stripe_count=");
            sprintf(tstring, "%d", atoi(argv[6]));
            strcat(lustre_pars, tstring);
            strcat(lustre_pars, ",stripe_size=");
            sprintf(tstring, "%d", atoi(argv[7]));
            strcat(lustre_pars, tstring);
            strcat(lustre_pars, ",block_size=");
            sprintf(tstring, "%d", atoi(argv[8]));
            strcat(lustre_pars, tstring);

            if(rank==0)
                printf("lustre_pars=%s\n", lustre_pars);

            adios_select_method (new_adios_group, argv[5], lustre_pars, "");  //Use MPI Lustre method

        }



        // variable definition part
        for (i = 0; i < g->vars_count; i++) {
            ADIOS_VARINFO * v = adios_inq_var_byid (g, i);
            getbasename (g->var_namelist[i], &var_path, &var_name);

            if (v->ndim == 0) 
            {   
                // scalars: every process does them the same.
                adios_define_var(new_adios_group,var_name,var_path,v->type,0,0,0);
                getTypeInfo( v->type, &element_size);    //element_size is size per element based on its type
                if (v->type == adios_string) {  //special case when the scalar is string.
                    adios_groupsize += strlen(v->value);
                } else {
                    adios_groupsize += element_size;
                }
            } 
            else 
            { 
                // vector variables
                getTypeInfo( v->type, &element_size);
                var_size=1;
                for (ii=0;ii<v->ndim;ii++) {
                    var_size*=v->dims[ii];
                }
                uint64_t total_size = var_size;  //total_size tells you the total number of elements in the current vector variable
                var_size*=element_size; //var_size tells you the size of the current vector variable in bytess

                //re-initialize the s and c variables
                for(j=0; j<v->ndim; j++){
                    s[j] = 0;
                    c[j] = 1;
                }

                //find the approximate chunk_size you would like to use.
                chunk_size = calcChunkSize(total_size, read_buffer*1024*1024/element_size, size);

                //set the chunk block array with the total size as close to chunk_size as possible
                calcC(chunk_size, v, c);
                strcpy(lbounds,"");
                for(j=0; j<v->ndim; j++){
                    sprintf(tstring, "%" PRId64 ",", c[j]);
                    strcat(lbounds, tstring);
                }
                printf("rank=%d, name=%s, chunk_size1=%" PRId64 " c[]=%s\n",rank,g->var_namelist[i],chunk_size,lbounds);


                chunk_size = 1;
                for(ii=0; ii<v->ndim; ii++)            //reset chunk_size based on the created c. Now the chunk_size is exact.
                    chunk_size *= c[ii];

                //current step points to where the process is in processing the vector. First sets with respect to rank.
                uint64_t current_step = rank*chunk_size;

                //First advance the starting point s by current_step. Of course, you don't do it if the current_step exceeds total_size.
                if(current_step<total_size)
                    rS(v, s, current_step, rank);

                uint64_t elements_defined = 0;  //First, the number of elements you have defined is 0.

                //You (the process) process your part of the vector when your current_step is smaller than the total_size
                while(current_step < total_size)
                {
                    //ts, temporary s, is introduced for the sake of the inner do while loop below. Copy s to ts.
                    uint64_t ts[] = {0,0,0,0,0,0,0,0,0,0};
                    arrCopy(s, ts);

                    //for every outer while iteration, you always have the whole chunk_size remained to process.
                    uint64_t remain_chunk = chunk_size;
                    if(current_step+chunk_size>total_size) //except when you are nearing the end of the vector....
                        remain_chunk = total_size-current_step;

                    //tc, temporary c, is introduced for the sake of the inner do while loop below. Copy s to tc.
                    uint64_t tc[] = {1,1,1,1,1,1,1,1,1,1};
                    arrCopy(c, tc);

                    do{
                        //how much of the remain chunk you wanna process? initially you think you can do all of it....
                        uint64_t used_chunk = remain_chunk;

                        //you feel like you should process the vector with tc block size, but given ts, you might go over bound.
                        uint64_t uc[] = {1,1,1,1,1,1,1,1,1,1};
                        //so you verify it by setting a new legit chunck block uc, and getting a new remain_chunk.
                        remain_chunk = checkBound(v, ts, tc, uc, remain_chunk);

                        //you check whether or not ts+uc goes over the bound. This is just checking to make sure there's no error.
                        //Thereotically, there should be no problem at all.
                        checkOverflow(0, v, ts, uc);


                        //the below code fragment simply calculates gbounds, and sets place holders for lbounds and offs.
                        strcpy(gbounds,"");
                        strcpy(lbounds,"");
                        strcpy(offs,"");

                        for(j=0; j<v->ndim-1; j++){
                            sprintf(tstring, "%d,", (int)v->dims[j]);
                            strcat(gbounds, tstring);
                            //sprintf(tstring, "ldim%d_%s,", j, var_name);
                            sprintf(tstring, "ldim%d,", j);
                            strcat(lbounds, tstring);
                            //sprintf(tstring, "offs%d_%s,", j, var_name);
                            sprintf(tstring, "offs%d,", j);
                            strcat(offs, tstring);
                        }

                        sprintf(tstring, "%d", (int)v->dims[v->ndim-1]);
                        strcat(gbounds, tstring);
                        //sprintf(tstring, "ldim%d_%s", v->ndim-1, var_name);
                        sprintf(tstring, "ldim%d", v->ndim-1);
                        strcat(lbounds, tstring);
                        //sprintf(tstring, "offs%d_%s", v->ndim-1, var_name);
                        sprintf(tstring, "offs%d", v->ndim-1);
                        strcat(offs, tstring);

                        //sprintf(tstring, "%d", v->ndim);
                        for(j=0; j<v->ndim; j++){
                            //sprintf(tstring, "ldim%d_%s", j, var_name);
                            sprintf(tstring, "ldim%d", j);
                            adios_define_var(new_adios_group, tstring, "bp2bp", adios_unsigned_long, 0, 0, 0);
                            //sprintf(tstring, "offs%d_%s", j, var_name);
                            sprintf(tstring, "offs%d", j);
                            adios_define_var(new_adios_group, tstring, "bp2bp", adios_unsigned_long, 0, 0, 0);
                        }

                        adios_define_var(new_adios_group,var_name,var_path,v->type,lbounds,gbounds,offs);


                        if (DEBUG){
                            strcpy(lbounds,"");
                            strcpy(offs,"");
                            for(j=0; j<v->ndim; j++){
                                sprintf(tstring, "%" PRId64 ",", ts[j]);
                                strcat(offs, tstring);
                                sprintf(tstring, "%" PRId64 ",", uc[j]);
                                strcat(lbounds, tstring);
                            }

                            printf("rank=%d, name=%s, gbounds=%s: lbounds=%s: offs=%s \n",rank,g->var_namelist[i],gbounds, lbounds, offs);
                        }

                        used_chunk -= remain_chunk; //you get the actual used_chunk here.
                        elements_defined += used_chunk;
                        if(remain_chunk!=0){
                            rS(v, ts, used_chunk, rank);  //advance ts by used_chunk.
                            for(k=0; k<10; k++)
                                tc[k] = 1;
                            calcC(remain_chunk, v, tc);   //based on the remain_chunk, calculate new tc chunk block remained to process.
                        }

                        adios_groupsize+= used_chunk*element_size+2*v->ndim*8;

                    }while(remain_chunk!=0);

                    current_step += size*chunk_size;  //once a whole chunk_size is processed, advance the current_step in roll-robin manner.

                    if(current_step<total_size){   //advance s in the same way.
                        rS(v, s, size*chunk_size, rank);
                    }
                }

                //beside checkOverflow above, here you check whether or not the total number of elements processed across processes matches
                //the total number of elements in the original vector.
                if(DEBUG){
                    uint64_t* sb = (uint64_t *) malloc(sizeof(uint64_t));
                    uint64_t* rb = (uint64_t *) malloc(sizeof(uint64_t));
                    sb[0] = elements_defined;
                    MPI_Reduce(sb,rb,1,MPI_UNSIGNED_LONG_LONG,MPI_SUM,0, comm);

                    if(rank==0 && rb[0]!=total_size)
                        printf("some array define mismatch. please use debug mode\n");
                    free(sb); free(rb);
                }
            }
            free (var_name);
            free (var_path);
        } // finished declaring all of the variables


        // Now we can define the attributes....
        for (i = 0; i < g->attrs_count; i++) {
            enum ADIOS_DATATYPES atype;
            int  asize;
            void *adata;
            adios_get_attr_byid (g, i, &atype, &asize, &adata);
            // if (DEBUG) printf("attribute name=%s\n",g->attr_namelist[i]);
            adios_define_attribute(new_adios_group,g->attr_namelist[i],"",atype,adata,0);
        }



        /*------------------------------ NOW WE WRITE -------------------------------------------- */
        // Now we have everything declared... now we need to write them out!!!!!!
        if (WRITEME==1) {
            // open up the file for writing....
            if (DEBUG) printf("rank=%d, opening file = %s, with group %s, size=%" PRId64 "\n",rank,argv[2],f->group_namelist[gidx],adios_groupsize);

            if(TIMING==100)
                start_time[0] = MPI_Wtime();

            adios_open(&m_adios_file, f->group_namelist[gidx],argv[2],"w",comm);
            adios_group_size( m_adios_file, adios_groupsize, &adios_totalsize);

            //get both the total adios_totalsize and total adios_groupsize summed across processes.
            uint64_t* sb = (uint64_t *) malloc(sizeof(uint64_t));;
            uint64_t* rb = (uint64_t *) malloc(sizeof(uint64_t));
            sb[0] = adios_groupsize;
            MPI_Reduce(sb,rb,1,MPI_UNSIGNED_LONG_LONG,MPI_SUM,0, comm);

            uint64_t* sb2 = (uint64_t *) malloc(sizeof(uint64_t));;
            uint64_t* rb2 = (uint64_t *) malloc(sizeof(uint64_t));
            sb2[0] = adios_totalsize;
            MPI_Reduce(sb2,rb2,1,MPI_UNSIGNED_LONG_LONG,MPI_SUM,0, comm);
            if(rank==0){
                printf("total adios_totalsize = %" PRId64 "\n", *rb2);
                printf("total adios_groupsize = %" PRId64 "\n", *rb);
            }
            free(sb); free(rb); free(sb2); free(rb2);

            if (TIMING==100) {
                end_time[0] = MPI_Wtime();
                total_time[0]+=end_time[0] - start_time[0];    //variable definition time taken
            }

            // now we have to write out the variables.... since they are all declared now
            // This will be the place we actually write out the data!!!!!!!!
            for (i = 0; i < g->vars_count; i++) {
                ADIOS_VARINFO * v = adios_inq_var_byid (g, i);
                getbasename (g->var_namelist[i], &var_path, &var_name);
                if (v->ndim == 0) 
                {
                    if (DEBUG) {
                        printf ("ADIOS WRITE SCALAR: rank=%d, name=%s value=",
                                rank,g->var_namelist[i]);
                        print_data (v->value, 0, v->type);
                        printf ("\n");
                    }
                    if (TIMING==100) {
                        start_time[2] = MPI_Wtime();
                    }
                    adios_write(m_adios_file,g->var_namelist[i],v->value);
                    if (TIMING==100) {
                        end_time[2] = MPI_Wtime();
                        total_time[2]+=end_time[2] - start_time[2];     //IO write time...
                    }
                } 
                else 
                {
                    for(j=0; j<v->ndim; j++){
                        s[j] = 0;
                        c[j] = 1;
                    }
                    getTypeInfo( v->type, &element_size);

                    uint64_t total_size = 1;
                    for (ii=0;ii<v->ndim;ii++)
                        total_size*=v->dims[ii];

                    chunk_size = calcChunkSize(total_size, read_buffer*1024*1024/element_size, size);
                    calcC(chunk_size, v, c);
                    chunk_size = 1;
                    for(ii=0; ii<v->ndim; ii++)
                        chunk_size *= c[ii];


                    uint64_t current_step = rank*chunk_size;
                    if(current_step<total_size)
                        rS(v, s, current_step, rank);

                    uint64_t elements_written = 0;

                    while(current_step < total_size)
                    {
                        uint64_t ts[] = {0,0,0,0,0,0,0,0,0,0};
                        arrCopy(s, ts);
                        uint64_t remain_chunk = chunk_size;
                        if(current_step+chunk_size>total_size)
                            remain_chunk = total_size-current_step;
                        uint64_t tc[] = {1,1,1,1,1,1,1,1,1,1};
                        arrCopy(c, tc);

                        do{
                            uint64_t uc[] = {1,1,1,1,1,1,1,1,1,1};
                            uint64_t used_chunk = remain_chunk;
                            remain_chunk = checkBound(v, ts, tc, uc, remain_chunk);

                            checkOverflow(1, v, ts, uc);

                            used_chunk -= remain_chunk;
                            elements_written += used_chunk;

                            //allocated space for data read-in
                            data = (void *) malloc(used_chunk*element_size);

                            if (TIMING==100) {
                                start_time[1] = MPI_Wtime();
                            }
                            if(PERFORMANCE_CHECK) printf("rank=%d, read start\n",rank);
                            bytes_read = adios_read_var_byid(g,v->varid,ts,uc,data);
                            if(PERFORMANCE_CHECK) printf("rank=%d, read end\n",rank);
                            if (TIMING==100) {
                                end_time[1] = MPI_Wtime();
                                total_time[1]+=end_time[1] -start_time[1];      //IO read time
                            }

                            if (DEBUG)
                                printf ("ADIOS WRITE: rank=%d, name=%s datasize=%" PRId64 "\n",rank,g->var_namelist[i],bytes_read);


                            if (TIMING==100) {
                                start_time[2] = MPI_Wtime();
                            }
                            if (DEBUG){
                                printf("rank=%d, write ts=",rank);
                                int k;
                                for(k=0; k<v->ndim; k++)
                                    printf("%" PRId64 ",", ts[k]);
                                printf("  uc=");
                                for(k=0; k<v->ndim; k++)
                                    printf("%" PRId64 ",", uc[k]);
                                printf("\n");
                            }

                            //local bounds and offets placeholders are not written out with actual values.
                            if(PERFORMANCE_CHECK) printf("rank=%d, adios write start\n", rank);
                            for(k=0; k<v->ndim; k++){
                                //sprintf(tstring, "ldim%d_%s", k, var_name);
                                sprintf(tstring, "ldim%d", k);
                                if (DEBUG) {
                                    printf ("ADIOS WRITE DIMENSION: rank=%d, name=%s value=",
                                            rank,tstring);
                                    print_data (&uc[k], 0, adios_unsigned_long);
                                    printf ("\n");
                                }
                                adios_write(m_adios_file, tstring, &uc[k]);

                                //sprintf(tstring, "offs%d_%s", k, var_name);
                                sprintf(tstring, "offs%d", k);
                                if (DEBUG) {
                                    printf ("ADIOS WRITE OFFSET: rank=%d, name=%s value=",
                                            rank,tstring);
                                    print_data (&ts[k], 0, adios_unsigned_long);
                                    printf ("\n");
                                }
                                adios_write(m_adios_file, tstring, &ts[k]);
                            }
                            adios_write(m_adios_file,g->var_namelist[i],data);
                            if(PERFORMANCE_CHECK) printf("rank=%d, adios write end\n", rank);


                            if (TIMING==100) {
                                end_time[2] = MPI_Wtime();
                                total_time[2]+=end_time[2] - start_time[2];   //IO write time
                            }

                            free(data);


                            if(remain_chunk!=0){
                                rS(v, ts, used_chunk, rank);
                                for(k=0; k<10; k++)
                                    tc[k] = 1;
                                calcC(remain_chunk, v, tc);
                            }

                        }while(remain_chunk!=0);

                        current_step += size*chunk_size;

                        if(current_step<total_size)
                            rS(v, s, size*chunk_size,rank);
                    }

                    if(DEBUG){
                        uint64_t* sb = (uint64_t *) malloc(sizeof(uint64_t));;
                        uint64_t* rb = (uint64_t *) malloc(sizeof(uint64_t));
                        sb[0] = elements_written;
                        MPI_Reduce(sb,rb,1,MPI_UNSIGNED_LONG_LONG,MPI_SUM,0, comm);
                        if(rank==0 && rb[0]!=total_size)
                            printf("some array read mismatch. please use debug mode\n");
                        free(sb); free(rb);
                    }
                }
                free (var_name);
                free (var_path);
            }// end of the writing of the variable..
            if (TIMING==100) {
                start_time[3] = MPI_Wtime();
            }
            if(PERFORMANCE_CHECK) printf("rank=%d, adios_close start\n", rank);
            adios_close(m_adios_file);
            if(PERFORMANCE_CHECK) printf("rank=%d, adios_close end\n", rank);
            if (TIMING==100) {
                end_time[3] = MPI_Wtime();
                total_time[3]+=end_time[3] - start_time[3];
            }
            adios_gclose(g);
        } //end of WRITEME
    } // end of all of the groups

    if(rank==0)
        printf("conversion done!\n");

    if(TIMING==100)
        start_time[4] = MPI_Wtime();
    adios_fclose(f);
    if(TIMING==100){
        end_time[4] = MPI_Wtime();
        total_time[4] = total_time[4]+end_time[4]-start_time[4];
    }
    adios_finalize(rank);


    // now, we write out the timing data, for each category, we give max, min, avg, std, all in seconds, across all processes.
    if(TIMING==100){

        // 0: adios_open, adios_group_size
        // 1: the total time to read in the data
        // 2: times around each write (will only work if we do NOT buffer....
        // 3: the time in the close
        // 4: fopen, fclose
        // 5: total time
        end_time[5] = MPI_Wtime();
        total_time[5] = end_time[5] - start_time[5];

        double sb[7];
        sb[0] = total_time[1]; sb[1] = total_time[4];   //read_var, fopen+fclose
        sb[2] = sb[0]+sb[1];
        sb[3] = total_time[0]; sb[4] = total_time[2]+total_time[3]; //adios_open+adios_group_size, write+close
        sb[5] = sb[3]+sb[4];
        sb[6] = total_time[5]; //total

        double * rb = NULL;

        if(rank==0)
            rb = (double *)malloc(size*7*sizeof(double));
        //MPI_Barrier(comm);
        MPI_Gather(sb, 7, MPI_DOUBLE, rb, 7, MPI_DOUBLE, 0, comm);

        if(rank==0){
            double read_avg1 = 0;
            double read_avg2 = 0;
            double tread_avg = 0;
            double write_avg1 = 0;
            double write_avg2 = 0;
            double twrite_avg = 0;
            double total_avg = 0;
            for(j=0; j<size; j++){
                read_avg1 += rb[7*j];
                read_avg2 += rb[7*j+1];
                tread_avg += rb[7*j+2];
                write_avg1 += rb[7*j+3];
                write_avg2 += rb[7*j+4];
                twrite_avg += rb[7*j+5];
                total_avg += rb[7*j+6];
            }
            read_avg1 /= size;
            read_avg2 /= size;
            tread_avg /= size;
            write_avg1 /= size;
            write_avg2 /= size;
            twrite_avg /= size;
            total_avg /= size;

            double read1_max = rb[0];
            double read1_min = rb[0];
            double read1_std = rb[0]-read_avg1; read1_std *= read1_std;

            double read2_max = rb[1];
            double read2_min = rb[1];
            double read2_std = rb[1]-read_avg2; read2_std *= read2_std;

            double tread_max = rb[2];
            double tread_min = rb[2];
            double tread_std = rb[2]-tread_avg; tread_std *= tread_std;

            double write1_max = rb[3];
            double write1_min = rb[3];
            double write1_std = rb[3]-write_avg1; write1_std *= write1_std;

            double write2_max = rb[4];
            double write2_min = rb[4];
            double write2_std = rb[4]-write_avg2; write2_std *= write2_std;

            double twrite_max = rb[5];
            double twrite_min = rb[5];
            double twrite_std = rb[5]-twrite_avg; twrite_std *= twrite_std;

            double total_max = rb[6];
            double total_min = rb[6];
            double total_std = rb[6]-total_avg; total_std *= total_std;

            for(j=1; j<size; j++){
                if(rb[7*j]>read1_max)
                    read1_max = rb[7*j];
                else if(rb[7*j]<read1_min)
                    read1_min = rb[7*j];
                double std = rb[7*j]-read_avg1; std *= std;
                read1_std += std;

                if(rb[7*j+1]>read2_max)
                    read2_max = rb[7*j+1];
                else if(rb[7*j+1]<read2_min)
                    read2_min = rb[7*j+1];
                std = rb[7*j+1]-read_avg2; std *= std;
                read2_std += std;

                if(rb[7*j+2]>tread_max)
                    tread_max = rb[7*j+2];
                else if(rb[7*j+2]<tread_min)
                    tread_min = rb[7*j+2];
                std = rb[7*j+2]-tread_avg; std *= std;
                tread_std += std;

                if(rb[7*j+3]>write1_max)
                    write1_max = rb[7*j+3];
                else if(rb[7*j+3]<write1_min)
                    write1_min = rb[7*j+3];
                std = rb[7*j+3]-write_avg1; std *= std;
                write1_std += std;

                if(rb[7*j+4]>write2_max)
                    write2_max = rb[7*j+4];
                else if(rb[7*j+4]<write2_min)
                    write2_min = rb[7*j+4];
                std = rb[7*j+4]-write_avg2; std *= std;
                write2_std += std;

                if(rb[7*j+5]>twrite_max)
                    twrite_max = rb[7*j+5];
                else if(rb[7*j+5]<twrite_min)
                    twrite_min = rb[7*j+5];
                std = rb[7*j+5]-twrite_avg; std *= std;
                twrite_std += std;

                if(rb[7*j+6]>total_max)
                    total_max = rb[7*j+6];
                else if(rb[7*j+6]<total_min)
                    total_min = rb[7*j+6];
                std = rb[7*j+6]-total_avg; std *= std;
                total_std += std;
            }
            read1_std /= size;  read1_std = sqrt(read1_std);
            read2_std /= size;  read2_std = sqrt(read2_std);
            tread_std /= size;    tread_std = sqrt(tread_std);
            write1_std /= size; write1_std = sqrt(write1_std);
            write2_std /= size; write2_std = sqrt(write2_std);
            twrite_std /= size;    twrite_std = sqrt(twrite_std);
            total_std /= size; total_std = sqrt(total_std);

            printf("---type---                       max\tmin\tavg\tstd\n");
            printf("---read_var---                   %lf\t%lf\t%lf\t%lf\n", read1_max, read1_min, read_avg1, read1_std);
            printf("---fopen+fclose---               %lf\t%lf\t%lf\t%lf\n", read2_max, read2_min, read_avg2, read2_std);
            printf("---total_read---                 %lf\t%lf\t%lf\t%lf\n", tread_max, tread_min, tread_avg, tread_std);
            printf("---adios_open+adios_groupsize--- %lf\t%lf\t%lf\t%lf\n", write1_max, write1_min, write_avg1, write1_std);
            printf("---write+close---                %lf\t%lf\t%lf\t%lf\n", write2_max, write2_min, write_avg2, write2_std);
            printf("---total_write---                %lf\t%lf\t%lf\t%lf\n", twrite_max, twrite_min, twrite_avg, twrite_std);
            printf("---total---                      %lf\t%lf\t%lf\t%lf\n", total_max, total_min, total_avg, total_std);
            free(rb);

        }

    }

    //    if (TIMING==100 && rank==0) {
    //        printf("------------------------------------------------------------------\n");
    //        printf("Define variables     = %lf\n",total_time[0]);
    //        printf("Read   variables     = %lf\n",total_time[1]);
    //        printf("Write  variables     = %lf\n",total_time[2]);
    //        printf("Close File for write = %lf\n",total_time[3]);
    //        printf("Total write time     = %lf\n",total_time[2] + total_time[3]);
    //        for (itime=0;itime<timers-1;itime++)
    //            total_time[timers-1]+=total_time[itime];
    //        printf("Total I/O time       = %lf\n",total_time[timers-1]);
    //    }
    MPI_Finalize();


    return(0);
}
Ejemplo n.º 6
0
int main (int argc, char ** argv) 
{
    int         gidx, i, j, k,l;
    MPI_Comm    comm_dummy = 0;  /* MPI_Comm is defined through adios_read.h */
    void      * data = NULL;
    uint64_t    start[] = {0,0,0,0,0,0,0,0,0,0};
    uint64_t    count[10];
    int64_t     bytes_read = 0;

    if (argc < 2) {
        printf("Usage: %s <BP-file>\n", argv[0]);
        return 1;
    }

    ADIOS_FILE * f;
    //int step;
    //for (step=0; step < 2; step++) {
        f = adios_fopen (argv[1], comm_dummy);
        if (f == NULL) {
            printf ("%s\n", adios_errmsg());
            return -1;
        }

        /* For all groups */
        for (gidx = 0; gidx < f->groups_count; gidx++) {
            printf("Group %s:\n", f->group_namelist[gidx]);
            ADIOS_GROUP * g = adios_gopen (f, f->group_namelist[gidx]);
            if (g == NULL) {
                printf ("%s\n", adios_errmsg());
                return -1;
            }

            /* For all variables */
            printf("  Variables=%d:\n", g->vars_count);
            for (i = 0; i < g->vars_count; i++) {
                ADIOS_VARINFO * v = adios_inq_var_byid (g, i);

                uint64_t total_size = adios_type_size (v->type, v->value);
                for (j = 0; j < v->ndim; j++)
                    total_size *= v->dims[j];

                printf("    %-9s  %s", adios_type_to_string(v->type), g->var_namelist[i]);
                if (v->ndim == 0) {
                    /* Scalars do not need to be read in, we get it from the metadata
                       when using adios_inq_var */
                    printf(" = %s\n", value_to_string(v->type, v->value, 0));
                } else {
                    /* Arrays have to be read in from the file */
                    printf("[%" PRIu64,v->dims[0]);
                    for (j = 1; j < v->ndim; j++)
                        printf(", %" PRIu64,v->dims[j]);
                    //printf("] = \n");
                    if (v->type == adios_integer)
                        printf("] = min=%d  max=%d  timedim=%d\n", (*(int*)v->gmin), (*(int*)v->gmax), v->timedim);
                    else if (v->type == adios_double)
                        printf("] = min=%lg  max=%lg  timedim=%d\n", (*(double*)v->gmin), (*(double*)v->gmax), v->timedim);
                    if (total_size > 1024*1024*1024) {
                        printf("        // too big, do not read in\n");
                    } else {
                        data = malloc (total_size);
                        if (data == NULL) {
                            fprintf (stderr, "malloc failed.\n");
                            return -1;
                        }

                        for (j = 0; j < v->ndim; j++) 
                            count[j] = v->dims[j];   

                        bytes_read = adios_read_var_byid (g, i, start, count, data);

                        if (bytes_read < 0) {
                            printf ("%s\n", adios_errmsg());
                        } else if (bytes_read > 1024*1024) {
                            printf ("Too big to print\n");
                        } else if (v->ndim == 1) {
                            printf ("        [");
                            for (j = 0; j < v->dims[0]; j++) 
                                printf("%s ", value_to_string(v->type, data, j));
                            printf ("]\n");
                        } else if (v->ndim == 2) {
                            for (j = 0; j < v->dims[0]; j++) {
                                printf ("        row %d: [", j);
                                for (k = 0; k < v->dims[1]; k++) 
                                    printf("%s ", value_to_string(v->type, data, j*v->dims[1] + k));
                                printf ("]\n");
                            }
                        } else if (v->ndim == 3) {
                            for (j = 0; j < v->dims[0]; j++) {
                                printf ("      block %d: \n", j);
                                for (k = 0; k < v->dims[1]; k++) {
                                    printf ("        row %d: [", k);
                                    for (l = 0; l < v->dims[2]; l++) {
                                        printf("%s ", value_to_string(v->type, data, j*v->dims[1]*v->dims[2] + k*v->dims[1] + l));
                                    }
                                    printf ("]\n");
                                }
                                printf ("\n");
                            }
                        } else {
                            printf ("    cannot print arrays with >3 dimensions\n");
                        }
                        free (data);
                    }
                }

                adios_free_varinfo (v);
            } /* variables */

            /* For all attributes */
            printf("  Attributes=%d:\n", g->attrs_count);
            for (i = 0; i < g->attrs_count; i++) {
                enum ADIOS_DATATYPES atype;
                int  asize;
                void *adata;
                adios_get_attr_byid (g, i, &atype, &asize, &adata);
                printf("    %-9s  %s = %s\n", adios_type_to_string(atype), 
                        g->attr_namelist[i], value_to_string(atype, adata, 0));
                free(adata);
            } /* attributes */

            adios_gclose (g);
        } /* groups */

        adios_fclose (f);

    //} /* loop 'step' */
    return 0;
}
Ejemplo n.º 7
0
int readVar(ADIOS_GROUP *gp, ADIOS_VARINFO *vi, const char * name)
{
  int i,j;
  uint64_t start_t[MAX_DIMS], count_t[MAX_DIMS]; // processed <0 values in start/count
  uint64_t s[MAX_DIMS], c[MAX_DIMS]; // for block reading of smaller chunks
  hsize_t  h5_start[MAX_DIMS], h5_count[MAX_DIMS], h5_stride[MAX_DIMS];
  hsize_t  h5_dims[MAX_DIMS];
  uint64_t nelems;         // number of elements to read
  int      elemsize;            // size in bytes of one element
  uint64_t st, ct;
  void     *data;
  uint64_t sum;           // working var to sum up things
  int      maxreadn;     // max number of elements to read once up to a limit (10MB of data)
  int      actualreadn;       // our decision how much to read at once
  int      readn[MAX_DIMS];   // how big chunk to read in in each dimension?
  int64_t  bytes_read;     // retval from adios_get_var()
  int      incdim;            // used in incremental reading in
  hid_t    grp_id, space_id, dataset, global_memspace, dataspace;
  hid_t    local_memspace, h5_ndim ;
  hid_t    h5_err;
  hid_t    h5_type_id;


  if (getTypeInfo(vi->type, &elemsize)) {
    fprintf(stderr, "Adios type %d (%s) not supported in bpls. var=%s\n", 
	    vi->type, adios_type_to_string(vi->type), name);
    return 10;
  }

   h5_err = bp_getH5TypeId (vi->type, &h5_type_id);
   h5_ndim = (hsize_t) vi->ndim;
   for (j=0;j<h5_ndim;j++)
       h5_dims[j] = vi->dims[j];
// create the hdf5 dataspace.
 
  // create the counter arrays with the appropriate lengths
  // transfer start and count arrays to format dependent arrays
  for (j=0; j<vi->ndim; j++)  {
      icount[j]=-1;
      h5_stride[j]= (hsize_t) 1;
  }
  nelems = 1;
  for (j=0; j<vi->ndim; j++) {
    if (istart[j] < 0)  // negative index means last-|index|
      st = vi->dims[j]+istart[j];
    else
      st = istart[j];
    if (icount[j] < 0)  // negative index means last-|index|+1-start
      ct = vi->dims[j]+icount[j]+1-st;
    else
      ct = icount[j];
    if (verbose>2) 
      printf("    j=%d, st=%llu ct=%llu\n", j, st, ct);
    start_t[j] = st;
    count_t[j] = ct;
    nelems *= ct;
    if (verbose>1) 
      printf("    s[%d]=%llu, c[%d]=%llu, n=%llu\n", j, start_t[j], j, count_t[j], nelems);
  }
  if (verbose>1) {
    printf(" total size of data to read = %llu\n", nelems*elemsize);
  }
  maxreadn = MAX_BUFFERSIZE/elemsize;
  if (nelems < maxreadn)
	  maxreadn = nelems;
  // special case: string. Need to use different elemsize
  if (vi->type == adios_string) {
    if (vi->value)
      elemsize = strlen(vi->value)+1;
    maxreadn = elemsize;
  }
  // allocate data array
  data = (void *) malloc (maxreadn*elemsize); // SAK: don't want the +8.... 
  //+8 for just to be sure

  // determine strategy how to read in:
  //  - at once
  //  - loop over 1st dimension
  //  - loop over 1st & 2nd dimension
  //  - etc
  if (verbose>1) printf("Read size strategy:\n");
  sum = (uint64_t) 1;
  actualreadn = (uint64_t) 1;
  for (i=vi->ndim-1; i>=0; i--) {
    if (sum >= (uint64_t) maxreadn) {
      readn[i] = 1;
    } else {
      readn[i] = maxreadn / (int)sum; // sum is small for 4 bytes here
      // this may be over the max count for this dimension
      if (readn[i] > count_t[i]) 
	readn[i] = count_t[i];
    }
    if (verbose>1) printf("    dim %d: read %d elements\n", i, readn[i]);
    sum = sum * (uint64_t) count_t[i];
    actualreadn = actualreadn * readn[i];
  }
  if (verbose>1) printf("    read %d elements at once, %lld in total (nelems=%lld)\n", actualreadn, sum, nelems);


  // init s and c
  for (j=0; j<vi->ndim; j++) {
    s[j]=start_t[j];
    c[j]=readn[j];
    h5_count[j] = (hsize_t) c[j];
    h5_start[j] = (hsize_t) s[j];
  }

  // read until read all 'nelems' elements
  sum = 0;
  while (sum < nelems) {

    // how many elements do we read in next?
    actualreadn = 1;
    for (j=0; j<vi->ndim; j++) 
      actualreadn *= c[j];

    if (verbose>2) {
      printf("adios_read_var name=%s ", name);
      //PRINT_DIMS("  start", s, vi->ndim, j); 
      //PRINT_DIMS("  count", c, vi->ndim, j); 
      printf("  read %d elems\n", actualreadn);
    }

    // read a slice finally
    bytes_read = adios_read_var_byid (gp, vi->varid, s, c, data); 

    if (bytes_read < 0) {
      fprintf(stderr, "Error when reading variable %s. errno=%d : %s \n", name, adios_errno, adios_errmsg());
      free(data);
      return 11;
    }

// now we must place this inside the hdf5 file... we know the offset (s)
// we know the count, c
// we know the global rank v->ndim
// we know the global dimensions v->dims

// get the hdf5 calls for writing the hyperslab.....

    dataset = H5Dopen(HDF5_FILE,name);
    if (dataset> 0) {
       global_memspace = H5Dget_space(dataset);
       hid_t rank_old = H5Sget_simple_extent_ndims(global_memspace);
       hsize_t *maxdims = (hsize_t *) malloc (h5_ndim * sizeof (hsize_t));
       h5_err = H5Sget_simple_extent_dims(global_memspace,maxdims,NULL);
       free(maxdims);
    } else {
       global_memspace = H5Screate_simple (h5_ndim, h5_dims, NULL);
       hid_t cparms = H5Pcreate(H5P_DATASET_CREATE);
       h5_err = H5Pset_chunk(cparms,h5_ndim,h5_count);
   
       dataset = H5Dcreate(HDF5_FILE,name,h5_type_id, global_memspace,cparms);
       H5Pclose(cparms);
    }

    local_memspace = H5Screate_simple (h5_ndim, h5_count, NULL);
     for (j=0;j<vi->ndim;j++) 

    h5_err =  H5Sselect_hyperslab (global_memspace, H5S_SELECT_SET
                                  ,h5_start, h5_stride, h5_count, NULL);

    h5_err = H5Dwrite(dataset,h5_type_id,local_memspace,global_memspace,H5P_DEFAULT, data);
    H5Sclose(local_memspace);
    H5Dclose(dataset);
    //H5Tclose(h5_type_id);

    if (verbose>2) printf("  read %lld bytes\n", bytes_read);

    // print slice

    // prepare for next read
    sum += actualreadn;
    incdim=1; // largest dim should be increased 
    for (j=vi->ndim-1; j>=0; j--) {
      if (incdim==1) {
	if (s[j]+c[j] == start_t[j]+count_t[j]) {
	  // reached the end of this dimension
	  s[j] = start_t[j];
	  c[j] = readn[j];
          h5_count[j] = (hsize_t) c[j];
          h5_start[j] = (hsize_t) s[j];
           
	  incdim = 1; // next smaller dim can increase too
	} else {
	  // move up in this dimension up to total count
	  s[j] += readn[j];
          h5_start[j] = (hsize_t) s[j];
	  if (s[j]+c[j] > start_t[j]+count_t[j]) {
	    // do not reach over the limit
	    c[j] = start_t[j]+count_t[j]-s[j];
            h5_count[j] = (hsize_t) c[j];
	  }
	  incdim = 0;
	}
      }
    }
  } // end while sum < nelems
  H5Sclose(global_memspace);

  free(data);
  return 0;
}