bool ADIOSFileObject::ReadVariable(const std::string &nm, int ts, vtkDataArray **array) { debug5<<"ADIOSFileObject::ReadVariable("<<nm<<" time= "<<ts<<")"<<endl; Open(); varIter vi = variables.find(nm); if (vi == variables.end()) { debug5<<"Variable "<<nm<<" not found."<<endl; return false; } ADIOSVar v = vi->second; int tupleSz = adios_type_size(v.type, NULL); *array = ADIOSFileObject::AllocateArray(v.type); int ntuples = 1; uint64_t start[4] = {0,0,0,0}, count[4] = {0,0,0,0}; v.GetReadArrays(ts, start, count, &ntuples); (*array)->SetNumberOfTuples(ntuples); void *data = (*array)->GetVoidPointer(0); debug5<<"ARR: adios_read_var:"<<endl<<v<<endl; OpenGroup(v.groupIdx); uint64_t retval = adios_read_var_byid(gps[v.groupIdx], v.varid, start, count, data); CloseGroup(v.groupIdx); return (retval > 0); }
void adios_checkpoint_verify_variables(ADIOS_FILE* fp, const char* name, int* origin) { ADIOS_VARINFO *vi; int count = 1; int size; vi = adios_inq_var(fp, name); if (vi->ndim > 0) { cout<<name<<" verification not passed, not a scalar"<<endl; return; } size = count*adios_type_size(vi->type, vi->value); int * mem= (int * )malloc(size); ADIOS_SELECTION *sel = adios_selection_writeblock(OHMMS::Controller->rank()); adios_schedule_read(fp, sel, name, 0, 1, mem); //adios_schedule_read(fp, NULL, name, 0, 1, mem); adios_perform_reads(fp, 1); if(mem[0] == *origin) { cout<<name<<" verification passed "<<mem[0]<<endl; } else { cout<<name<<" verification not passed, readin: "<<mem[0]<<" writeout: "<<*origin<<endl; } adios_free_varinfo (vi); adios_selection_delete(sel); free(mem); }
// k is numbered from 1 to sum_nblocks void verifyData(ADIOS_FILE* f, ADIOS_VARINFO* v, int k, int timestep) { uint64_t blockBytes = adios_type_size (v->type, v->value); int j=0; if (v->ndim <= 0) { return; } //printf("verify block[%d]: ", k); for (j=0; j<v->ndim; j++) { blockBytes *= v->blockinfo[k].count[j]; //printf("%" PRIu64 ":%" PRIu64 " ", v->blockinfo[k].start[j], v->blockinfo[k].count[j]); } void* data = NULL; data = malloc(blockBytes); ADIOS_SELECTION* sel = adios_selection_boundingbox (v->ndim, v->blockinfo[k].start, v->blockinfo[k].count); int err = adios_schedule_read_byid(f, sel, v->varid, timestep, 1, data); if (!err) { err = adios_perform_reads(f, 1); } //fastbit_adios_util_printData(data, v->type, blockBytes/adios_type_size(v->type, v->value)); adios_selection_delete(sel); free(data); data = NULL; }
void adios_checkpoint_verify_variables(ADIOS_FILE* fp, const char* name, unsigned long origin) { ADIOS_VARINFO *vi; int count = 1; int size; vi = adios_inq_var(fp, name); if (vi->ndim > 0) { cout<<name<<" verification not passed, not a scalar"<<endl; return; } size = count*adios_type_size(vi->type, vi->value); unsigned long* mem= (unsigned long* )malloc(size); adios_schedule_read(fp, NULL, name, 0, 1, mem); adios_perform_reads(fp, 1); if(mem[0] == origin) { cout<<name<<" verification passed"<<mem[0]<<endl; } else { cout<<name<<" verification not passed, readin: "<<mem[0]<<" writeout: "<<origin<<endl; } adios_free_varinfo (vi); free(mem); }
uint64_t estimateBytesOnVar(ADIOS_FILE* f, ADIOS_VARINFO* v) { if (v->ndim == 0) { return 0; } adios_inq_var_blockinfo(f, v); uint64_t result = 0; int i = 0; int j = 0; for (i=0; i<v->sum_nblocks; i++) { ADIOS_VARBLOCK curr = v->blockinfo[i]; uint64_t blockSize = fastbit_adios_util_getBlockSize(v, -1, i); result += blockSize; //uint64_t blockDataByteSize = adios_type_size (v->type, v->value) * blockSize0; /*uint64_t blockSize = 1; for (j=0; j<v->ndim; j++) { blockSize = blockSize* curr.count[j]; } result += blockSize; */ } return adios_type_size (v->type, v->value) * result; //int typeSize = adios_type_size(v->type, NULL); //return result * typeSize; }
void adios_checkpoint_verify_variables(ADIOS_FILE* fp, const char* name, RealType* origin) { ADIOS_VARINFO *vi; int count = 1; int size; vi = adios_inq_var(fp, name); adios_inq_var_blockinfo (fp, vi); if (vi->ndim > 0) { count*=vi->dims[0]; for (int j = 1; j < vi->ndim; j++) { count *= vi->dims[j]; } } size = count*adios_type_size(vi->type, vi->value); RealType *mem= (RealType *)malloc(size); adios_schedule_read(fp, NULL, name, 0, 1, mem); adios_perform_reads(fp, 1); for(int i=0; i<count; i++) { if(mem[i] == origin[i]) { cout<<name<<"["<<i<<"]verification passed "<<mem[i]<<endl; } else { cout<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl; } } adios_free_varinfo (vi); free(mem); }
void ADIOSFileObject::GetTimes(std::string &varNm, std::vector<double> ×) { Open(); times.resize(0); varIter vi = variables.find(varNm); if (vi == variables.end()) { debug5<<"Variable "<<varNm<<" not found."<<endl; return; } ADIOSVar v = vi->second; int tupleSz = adios_type_size(v.type, NULL); uint64_t start[4] = {0,0,0,0}, count[4] = {0,0,0,0}; count[0] = v.count[0]; int ntuples = v.count[0]; void *readData = malloc(ntuples*tupleSz); OpenGroup(v.groupIdx); uint64_t retval = adios_read_var_byid(gps[v.groupIdx], v.varid, start, count, readData); CloseGroup(v.groupIdx); if (retval > 0) { times.resize(ntuples); ConvertTo(×[0], ntuples, v.type, readData); } free(readData); }
std::string AdiosCheckpointInput::getScalar(const std::string& var_name) { //Scalars are all stored in the metadata so we can read them without disk access ADIOS_VARINFO* adios_inq = adios_inq_var(adios_file_handle, var_name.c_str()); int string_size = adios_type_size(adios_inq->type, adios_inq->value); std::string string_val = static_cast<char*>(*adios_inq->value); adios_free_varinfo(adios_inq); return string_val; }
int64_t ADIOS_Var::getDataLen() { uint64_t total_bytes = adios_type_size(getType(), readValue()); for (int i=0; i<getNumDimension(); i++) { total_bytes *= getDimensionSize(i); } return total_bytes; }
/// Read a sub-array. /// /// @Note The length of the buffer is assumed to be sufficiently large /// if the argument bufLen is less or equal to 0. The caller is /// responsible for allocating the buffer (@c buf) of the currect size. int64_t ADIOS_Var::readData(void* buf, int64_t bufLen, const uint64_t* start, const uint64_t* count) { uint64_t total_elm = 1; for (int i=0; i<getNumDimension(); i++) { total_elm *= count[i]; #ifdef DEBUG LOGGER(ibis::gVerbose > 5) << "ADIOS_Var::readData: [" << i << "] start=" << start[i] << ", count=" << count[i]; #endif } { uint64_t total_bytes = total_elm * adios_type_size(getType(), readValue()); if (bufLen > 0 && (uint64_t)bufLen < total_bytes) { LOGGER(ibis::gVerbose > 0) << "Warning -- ADIOS_Var::readData: bufLen (" << bufLen << ") < total_bytes (" << total_bytes << ")"; return -1; } } ADIOS_SELECTION *sel = adios_selection_boundingbox (m_handle->ndim, start, count); if (sel == 0) { LOGGER(ibis::gVerbose > 1) << "Warning -- ADIOS_Var::readData failed to create a selection"; return -2; } IBIS_BLOCK_GUARD(adios_selection_delete, sel); int ierr = adios_schedule_read_byid (getFile(), sel, index(), getFile()->current_step, 1, buf); if (ierr != 0) { LOGGER(ibis::gVerbose > 0) << "Warning -- ADIOS_Var::readData call to " "adios_schedule_read_byid failed due to " << adios_errmsg(); return -3; } ierr = adios_perform_reads(getFile(), 1); // 1 == blocking if (ierr != 0) { LOGGER(ibis::gVerbose > 0) << "Warning -- ADIOS_Var::readData call to adios_perform_reads on " << getFileName() << " failed due to " << adios_errmsg(); return -4; } LOGGER(ibis::gVerbose > 5) << "ADIOS_Var::readData: competeled reading " << total_elm << " element" << (total_elm>1?"s":"") << " for " << getName() << " from " << getFileName(); return total_elm; } // ADIOS_Var::readData
void adios_checkpoint_verify_random_variables(ADIOS_FILE* fp, const char* name, uint_type* origin) { ADIOS_VARINFO *vi; int count_int = 1; int size; uint64_t *start; uint64_t *count; vi = adios_inq_var(fp, name); adios_inq_var_blockinfo (fp, vi); if (vi->ndim > 0) { start = (uint64_t *)malloc(vi->ndim * sizeof(uint64_t)); count = (uint64_t *)malloc(vi->ndim * sizeof(uint64_t)); } for (int j=0; j<vi->nblocks[0]; j++) { if(j == OHMMS::Controller->rank()) { for (int k=0; k<vi->ndim; k++) { start[k] = vi->blockinfo[j].start[k]; count[k] = vi->blockinfo[j].count[k]; count_int *= count[k]; //cout<<OHMMS::Controller->rank()<<" count "<<start[k]<<" "<<count[k]<<endl; } } } size = count_int*adios_type_size(vi->type, vi->value); uint_type *mem= (uint_type*)malloc(size); ADIOS_SELECTION *sel = adios_selection_boundingbox(vi->ndim, start, count); adios_schedule_read(fp, sel, name, 0, 1, mem); adios_perform_reads(fp, 1); int flag = 0; for(int i=0; i<count_int; i++) { if(mem[i] == origin[i]) { //cout<<name<<"["<<i<<"]verification passed "<<mem[i]<<endl; } else { flag = 1; cout<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl; } } if (flag == 0) cout<<name<<" verification passed "<<endl; else cout<<name<<" verification not passed "<<endl; adios_free_varinfo (vi); adios_selection_delete(sel); free(start); free(count); free(mem); }
void onBlock(int rank, ADIOS_FILE* f, ADIOS_VARINFO* v, int i, int j, int blockCounter, FastBitDataType ft) { char bmsVarName[100]; char keyVarName[100]; char offsetName[100]; int64_t var_ids_bms[v->nblocks[i]]; int64_t var_ids_key[v->nblocks[i]]; int64_t var_ids_offset[v->nblocks[i]]; sprintf(bmsVarName, "bms-%d-%d-%d", v->varid, i, j); sprintf(keyVarName, "key-%d-%d-%d", v->varid, i, j); sprintf(offsetName, "offset-%d-%d-%d", v->varid, i, j); uint64_t blockSize = fastbit_adios_util_getBlockSize(v, i, j); uint64_t blockDataByteSize = adios_type_size (v->type, v->value) * blockSize; char notes[100]; logTime(NULL); logTimeMillis(NULL); sprintf(notes, " reading data from adios on varid=%d, time=%d, block: %d, size=%ld bytes=%ld", v->varid, i, j, blockSize, blockDataByteSize); logTime(notes); logTimeMillis(notes); localtime(&indexRefresh); //printf(" %d th block / (%d), size= %" PRIu64 " bytes=%" PRIu64, j, blockSize, blockCounter, blockDataByteSize); void* data = malloc (blockDataByteSize); ADIOS_SELECTION* blockSel = adios_selection_writeblock(j); //adios_selcton_writeblock(num), 0 <= num < nblocks[timestep] //ADIOS_SELECTION* blockSel = adios_selection_writeblock(blockCounter); int err = adios_schedule_read_byid(f, blockSel, v->varid, i, 1, data); if (!err) { err = adios_perform_reads(f, 1); } else { printf("Unable to read block %d at timestep: %d \n", j, i); return; //break; } //fastbit_adios_util_printData(data, v->type, blockSize); char selName[20]; sprintf(selName, "block-%d", j); processData(data, blockSize, rank, i, selName, ft, v); //processData(void* data, uint64_t dataCount, int rank, int timestep, char* selName, FastBitDataType ft, ADIOS_VARINFO* v) adios_selection_delete(blockSel); verifyData(f, v, blockCounter, i); } // onblock
void ADIOS1CommonRead::ScheduleReadCommon(const std::string &name, const Dims &offs, const Dims &ldims, const int fromStep, const int nSteps, const bool readAsLocalValue, const bool readAsJoinedArray, void *data) { if (readAsLocalValue) { /* Get all the requested values from metadata now */ ADIOS_VARINFO *vi = adios_inq_var(m_fh, name.c_str()); if (vi) { adios_inq_var_stat(m_fh, vi, 0, 1); int elemsize = adios_type_size(vi->type, nullptr); long long blockidx = 0; for (int i = 0; i < fromStep; i++) { blockidx += vi->nblocks[i]; } char *dest = (char *)data; for (int i = fromStep; i < fromStep + nSteps; i++) { for (int j = 0; j < vi->nblocks[i]; j++) { memcpy(dest, vi->statistics->blocks->mins[blockidx], elemsize); ++blockidx; dest += elemsize; } } adios_free_varinfo(vi); } } else { uint64_t start[32], count[32]; for (int i = 0; i < ldims.size(); i++) { start[i] = (uint64_t)offs[i]; count[i] = (uint64_t)ldims[i]; } ADIOS_SELECTION *sel = nullptr; if (ldims.size() > 0) { sel = adios_selection_boundingbox(ldims.size(), start, count); } adios_schedule_read(m_fh, sel, name.c_str(), (int)fromStep, (int)nSteps, data); adios_selection_delete(sel); } }
T AdiosCheckpointInput::getScalar(const std::string& var_name) { //Scalars are all stored in the metadata so we can read them without disk access ADIOS_VARINFO* adios_inq = adios_inq_var(adios_file_handle, var_name.c_str()); if (adios_type_size(adios_inq->type, adios_inq->value) != sizeof(T)) { qmcplusplus::app_error() << "Data type does not match data type found in file: " << std::endl; return ; } T scalar_value = static_cast<T>(*adios_inq->value); adios_free_varinfo(adios_inq); return scalar_value; }
void ToolsAdiosParallel::convertToText() { if(m_options.data.size() == 0) throw std::runtime_error("No datasets requested"); for (size_t i = 0; i < m_options.data.size(); ++i) { ADIOS_VARINFO *pVarInfo; //get name of dataset to print std::string nodeName = m_options.data[i]; uint8_t *P; int varElement = 1; int varTypeSize = 0; adios_read_init_method(ADIOS_READ_METHOD_BP, comm, nodeName.c_str()); pVarInfo = adios_inq_var(pFile, nodeName.c_str()); varTypeSize = adios_type_size(pVarInfo->type, NULL); // get number of elements combined in a dataset for(int j = 0; j < pVarInfo->ndim; j++) { varElement = varElement * pVarInfo->dims[j]; } // allocate memory P = (uint8_t*) malloc (sizeof(uint8_t) * varTypeSize * varElement); adios_schedule_read(pFile, NULL, nodeName.c_str(), 0, 1, P); adios_perform_reads(pFile, 1); if(pVarInfo->ndim > 0) { for(int k = 0; k < varElement; k++) { printValue(pVarInfo->type, &P[k*varTypeSize]); } } else { printValue(pVarInfo->type, pVarInfo->value); } adios_free_varinfo(pVarInfo); } }
bool ADIOSFileObject::ReadVariable(const std::string &nm, int ts, vtkFloatArray **array) { debug5<<"ADIOSFileObject::ReadVariable("<<nm<<" time= "<<ts<<")"<<endl; Open(); varIter vi = variables.find(nm); if (vi == variables.end()) { debug5<<"Variable "<<nm<<" not found."<<endl; return false; } ADIOSVar v = vi->second; int tupleSz = adios_type_size(v.type, NULL); *array = vtkFloatArray::New(); int ntuples = 1; uint64_t start[4] = {0,0,0,0}, count[4] = {0,0,0,0}; v.GetReadArrays(ts, start, count, &ntuples); (*array)->SetNumberOfTuples(ntuples); float *data = (float *)(*array)->GetVoidPointer(0); void *readData = (void *)data; bool convertData = (v.type != adios_real); if (convertData) readData = malloc(ntuples*tupleSz); debug5<<"ARR: adios_read_var:"<<endl<<v<<endl; OpenGroup(v.groupIdx); uint64_t retval = adios_read_var_byid(gps[v.groupIdx], v.varid, start, count, readData); CloseGroup(v.groupIdx); if (convertData) { if (retval > 0) ConvertTo(data, ntuples, v.type, readData); free(readData); } return (retval > 0); }
void AdiosCheckpointInput::getVector(const std::string& var_name, vector<T>& buffer) { ADIOS_VARINFO* adios_inq = adios_inq_var(adios_file_handle, var_name.c_str()); //Check to make sure the T is the same size as the data type on the disk if (adios_type_size(adios_inq->type, NULL) != sizeof(T)) { qmcplusplus::app_error() << "Data type does not match data type found in file: " << std::endl; APP_ABORT("qmcapp"); return ; } uint64_t total_size = 1; for (int i = 0; i < adios_inq->ndim; i++) total_size *= adios_inq->dims[i]; //make sure we have enough space to copy all the data from the file buffer.reserve(total_size); //schedule the read adios_schedule_read(adios_file_handle, sel, var_name.c_str(), 0, 1, &(buffer[0])); //perform the read adios_perform_reads(adios_file_handle, 1); //Don't need the information about the variable anymore adios_free_varinfo(adios_inq); }
void adios_trace_verify_local_variables(ADIOS_FILE* fp, const char* name, double* origin) { ADIOS_VARINFO *vi; int count = 1; unsigned long size = 1; vi = adios_inq_var(fp, name); adios_inq_var_blockinfo(fp, vi); for (int j=0; j<vi->nblocks[0]; j++) { if(OHMMS::Controller->rank() == j) { for (int k=0; k<vi->ndim; k++) { count *= vi->blockinfo[j].count[k]; } } } size = count * adios_type_size(vi->type, vi->value); double* mem = (double*)malloc(size); ADIOS_SELECTION *sel = adios_selection_writeblock(OHMMS::Controller->rank()); adios_schedule_read(fp, sel, name, 0, 1, mem); adios_perform_reads(fp, 1); int flag = 0; for(int i=0; i<count; i++) { if(mem[i] == origin[i]) { } else { flag = 1; cout<<OHMMS::Controller->rank()<<" "<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl; } } if(flag == 0) cout<<OHMMS::Controller->rank()<<" "<<name<<" verification passed"<<endl; adios_selection_delete(sel); adios_free_varinfo (vi); free(mem); }
void AdiosCheckpointInput::performReads() { for (int i = 0; i < read_queue.size(); i++) { const char* var_name = read_queue[i].c_str(); //grab the information from the metadata about this variable ADIOS_VARINFO* adios_inq = adios_inq_var(adios_file_handle, var_name); //Calculate the total size of the vector based on the metadata uint64_t total_size = adios_type_size(adios_inq->type, NULL); for (int i = 0; i < adios_inq->ndim; i++) total_size *= adios_inq->dims[i]; //allocate and then store for when retrieve void* buff = malloc(total_size); buffers.push_back(buff); //schedule the read adios_schedule_read(adios_file_handle, sel, var_name, 0, 1, buff); //Don't need the information about the variable anymore adios_free_varinfo(adios_inq); } //perform the read adios_perform_reads(adios_file_handle, 1); }
int process_metadata(int step) { int retval = 0; int i, j; char gdims[256], ldims[256], offs[256]; uint64_t sum_count; ADIOS_VARINFO *v; // shortcut pointer if (step > 1) { // right now, nothing to prepare in later steps print("Step %d. return immediately\n",step); return 0; } /* First step processing */ // get groupname of stream, then declare for output adios_get_grouplist(f, &group_namelist); print0("Group name is %s\n", group_namelist[0]); adios_declare_group(&gh,group_namelist[0],"",adios_flag_yes); varinfo = (VarInfo *) malloc (sizeof(VarInfo) * f->nvars); if (!varinfo) { print("ERROR: rank %d cannot allocate %lu bytes\n", rank, sizeof(VarInfo)*f->nvars); return 1; } write_total = 0; largest_block = 0; // Decompose each variable and calculate output buffer size for (i=0; i<f->nvars; i++) { print0 ("Get info on variable %d: %s\n", i, f->var_namelist[i]); varinfo[i].v = adios_inq_var_byid (f, i); v = varinfo[i].v; // just a shortcut if (v == NULL) { print ("rank %d: ERROR: Variable %s inquiry failed: %s\n", rank, f->var_namelist[i], adios_errmsg()); return 1; } // print variable type and dimensions print0(" %-9s %s", adios_type_to_string(v->type), f->var_namelist[i]); if (v->ndim > 0) { print0("[%llu", v->dims[0]); for (j = 1; j < v->ndim; j++) print0(", %llu", v->dims[j]); print0("] :\n"); } else { print0("\tscalar\n"); } // determine subset we will write decompose (numproc, rank, v->ndim, v->dims, decomp_values, varinfo[i].count, varinfo[i].start, &sum_count); varinfo[i].writesize = sum_count * adios_type_size(v->type, v->value); if (varinfo[i].writesize != 0) { write_total += varinfo[i].writesize; if (largest_block < varinfo[i].writesize) largest_block = varinfo[i].writesize; } } // determine output buffer size and allocate it uint64_t bufsize = write_total + f->nvars*128 + f->nattrs*32 + 1024; if (bufsize > max_write_buffer_size) { print ("ERROR: rank %d: write buffer size needs to hold about %llu bytes, " "but max is set to %d\n", rank, bufsize, max_write_buffer_size); return 1; } print0 ("Rank %d: allocate %llu MB for output buffer\n", rank, bufsize/1048576+1); adios_allocate_buffer (ADIOS_BUFFER_ALLOC_NOW, bufsize/1048576+1); // allocate read buffer bufsize = largest_block + 128; if (bufsize > max_read_buffer_size) { print ("ERROR: rank %d: read buffer size needs to hold at least %llu bytes, " "but max is set to %d\n", rank, bufsize, max_read_buffer_size); return 1; } print0 ("Rank %d: allocate %g MB for input buffer\n", rank, (double)bufsize/1048576.0); readbuf = (char *) malloc ((size_t)bufsize); if (!readbuf) { print ("ERROR: rank %d: cannot allocate %llu bytes for read buffer\n", rank, bufsize); return 1; } // Select output method adios_select_method (gh, wmethodname, wmethodparams, ""); // Define variables for output based on decomposition char *vpath, *vname; for (i=0; i<f->nvars; i++) { v = varinfo[i].v; if (varinfo[i].writesize != 0) { // define variable for ADIOS writes getbasename (f->var_namelist[i], &vpath, &vname); if (v->ndim > 0) { int64s_to_str (v->ndim, v->dims, gdims); int64s_to_str (v->ndim, varinfo[i].count, ldims); int64s_to_str (v->ndim, varinfo[i].start, offs); print ("rank %d: Define variable path=\"%s\" name=\"%s\" " "gdims=%s ldims=%s offs=%s\n", rank, vpath, vname, gdims, ldims, offs); adios_define_var (gh, vname, vpath, v->type, ldims, gdims, offs); } else { print ("rank %d: Define scalar path=\"%s\" name=\"%s\"\n", rank, vpath, vname); adios_define_var (gh, vname, vpath, v->type, "", "", ""); } free(vpath); free(vname); } } if (rank == 0) { // get and define attributes enum ADIOS_DATATYPES attr_type; void * attr_value; char * attr_value_str; int attr_size; for (i=0; i<f->nattrs; i++) { adios_get_attr_byid (f, i, &attr_type, &attr_size, &attr_value); attr_value_str = (char *)value_to_string (attr_type, attr_value, 0); getbasename (f->attr_namelist[i], &vpath, &vname); if (vpath && !strcmp(vpath,"/__adios__")) { // skip on /__adios/... attributes print ("rank %d: Ignore this attribute path=\"%s\" name=\"%s\" value=\"%s\"\n", rank, vpath, vname, attr_value_str); } else { adios_define_attribute (gh, vname, vpath, attr_type, attr_value_str, ""); print ("rank %d: Define attribute path=\"%s\" name=\"%s\" value=\"%s\"\n", rank, vpath, vname, attr_value_str); free (attr_value); } } } return retval; }
int main (int argc, char ** argv) { int rank, size, i; MPI_Comm comm = MPI_COMM_WORLD; enum ADIOS_DATATYPES attr_type; enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP; int attr_size; void * data = NULL; MPI_Init (&argc, &argv); MPI_Comm_rank (comm, &rank); MPI_Comm_size (comm, &size); adios_read_init_method (method, comm, "verbose=3"); adios_logger_open ("log_read_C", rank); ADIOS_FILE * f = adios_read_open ("attributes_C.bp", method, comm, ADIOS_LOCKMODE_NONE, 0.0); if (f == NULL) { log_error ("%s\n", adios_errmsg()); return -1; } for (i = 0; i < f->nattrs; i++) { adios_get_attr (f, f->attr_namelist[i], &attr_type, &attr_size, &data); log_test("rank %d: attr: %s %s = ", rank, adios_type_to_string(attr_type), f->attr_namelist[i]); int type_size = adios_type_size (attr_type, data); int nelems = attr_size / type_size; int k; char *p = (char*)data; for (k=0; k<nelems; k++) { if (k>0) log_test(", "); switch (attr_type) { case adios_integer: log_test ("%d", *(int *)p); break; case adios_double: log_test ("%e", *(double *)p); break; case adios_string: log_test ("\"%s\"", (char *)p); break; case adios_string_array: log_test ("\"%s\"", *(char **)p); break; default: log_test ("??????\n"); } p=p+type_size; } log_test("\n"); free (data); data = 0; } adios_read_close (f); MPI_Barrier (comm); adios_read_finalize_method (ADIOS_READ_METHOD_BP); adios_logger_close(); MPI_Finalize (); return 0; }
int main (int argc, char ** argv) { int gidx, i, j, k,l; MPI_Comm comm_dummy = 0; /* MPI_Comm is defined through adios_read.h */ void * data = NULL; uint64_t start[] = {0,0,0,0,0,0,0,0,0,0}; uint64_t count[10]; int64_t bytes_read = 0; if (argc < 2) { printf("Usage: %s <BP-file>\n", argv[0]); return 1; } ADIOS_FILE * f; //int step; //for (step=0; step < 2; step++) { f = adios_fopen (argv[1], comm_dummy); if (f == NULL) { printf ("%s\n", adios_errmsg()); return -1; } /* For all groups */ for (gidx = 0; gidx < f->groups_count; gidx++) { printf("Group %s:\n", f->group_namelist[gidx]); ADIOS_GROUP * g = adios_gopen (f, f->group_namelist[gidx]); if (g == NULL) { printf ("%s\n", adios_errmsg()); return -1; } /* For all variables */ printf(" Variables=%d:\n", g->vars_count); for (i = 0; i < g->vars_count; i++) { ADIOS_VARINFO * v = adios_inq_var_byid (g, i); uint64_t total_size = adios_type_size (v->type, v->value); for (j = 0; j < v->ndim; j++) total_size *= v->dims[j]; printf(" %-9s %s", adios_type_to_string(v->type), g->var_namelist[i]); if (v->ndim == 0) { /* Scalars do not need to be read in, we get it from the metadata when using adios_inq_var */ printf(" = %s\n", value_to_string(v->type, v->value, 0)); } else { /* Arrays have to be read in from the file */ printf("[%" PRIu64,v->dims[0]); for (j = 1; j < v->ndim; j++) printf(", %" PRIu64,v->dims[j]); //printf("] = \n"); if (v->type == adios_integer) printf("] = min=%d max=%d timedim=%d\n", (*(int*)v->gmin), (*(int*)v->gmax), v->timedim); else if (v->type == adios_double) printf("] = min=%lg max=%lg timedim=%d\n", (*(double*)v->gmin), (*(double*)v->gmax), v->timedim); if (total_size > 1024*1024*1024) { printf(" // too big, do not read in\n"); } else { data = malloc (total_size); if (data == NULL) { fprintf (stderr, "malloc failed.\n"); return -1; } for (j = 0; j < v->ndim; j++) count[j] = v->dims[j]; bytes_read = adios_read_var_byid (g, i, start, count, data); if (bytes_read < 0) { printf ("%s\n", adios_errmsg()); } else if (bytes_read > 1024*1024) { printf ("Too big to print\n"); } else if (v->ndim == 1) { printf (" ["); for (j = 0; j < v->dims[0]; j++) printf("%s ", value_to_string(v->type, data, j)); printf ("]\n"); } else if (v->ndim == 2) { for (j = 0; j < v->dims[0]; j++) { printf (" row %d: [", j); for (k = 0; k < v->dims[1]; k++) printf("%s ", value_to_string(v->type, data, j*v->dims[1] + k)); printf ("]\n"); } } else if (v->ndim == 3) { for (j = 0; j < v->dims[0]; j++) { printf (" block %d: \n", j); for (k = 0; k < v->dims[1]; k++) { printf (" row %d: [", k); for (l = 0; l < v->dims[2]; l++) { printf("%s ", value_to_string(v->type, data, j*v->dims[1]*v->dims[2] + k*v->dims[1] + l)); } printf ("]\n"); } printf ("\n"); } } else { printf (" cannot print arrays with >3 dimensions\n"); } free (data); } } adios_free_varinfo (v); } /* variables */ /* For all attributes */ printf(" Attributes=%d:\n", g->attrs_count); for (i = 0; i < g->attrs_count; i++) { enum ADIOS_DATATYPES atype; int asize; void *adata; adios_get_attr_byid (g, i, &atype, &asize, &adata); printf(" %-9s %s = %s\n", adios_type_to_string(atype), g->attr_namelist[i], value_to_string(atype, adata, 0)); free(adata); } /* attributes */ adios_gclose (g); } /* groups */ adios_fclose (f); //} /* loop 'step' */ return 0; }
int main (int argc, char ** argv) { int i, j, k,l; MPI_Comm comm_dummy = 0; /* MPI_Comm is defined through adios_read.h */ if (argc < 2) { printf("Usage: %s <BP-file>\n", argv[0]); return 1; } adios_read_init_method (ADIOS_READ_METHOD_BP, comm_dummy, "show_hidden_attrs"); ADIOS_FILE * f; f = adios_read_open_file (argv[1], ADIOS_READ_METHOD_BP, comm_dummy); if (f == NULL) { printf ("%s\n", adios_errmsg()); return -1; } /* For all variables */ printf(" Variables=%d:\n", f->nvars); for (i = 0; i < f->nvars; i++) { ADIOS_VARINFO * v = adios_inq_var_byid (f, i); adios_inq_var_stat (f, v, 0, 1); adios_inq_var_blockinfo (f, v); uint64_t total_size = adios_type_size (v->type, v->value); for (j = 0; j < v->ndim; j++) total_size *= v->dims[j]; printf(" %-9s %s", adios_type_to_string(v->type), f->var_namelist[i]); if (v->ndim == 0) { /* Scalars do not need to be read in, we get it from the metadata when using adios_inq_var */ printf(" = %s\n", value_to_string(v->type, v->value, 0)); } else { /* Arrays, print min/max statistics*/ printf("[%lld",v->dims[0]); for (j = 1; j < v->ndim; j++) printf(", %lld",v->dims[j]); //printf("] = \n"); if (v->type == adios_integer) printf("] : min=%d max=%d\n", (*(int*)v->statistics->min), (*(int*)v->statistics->max)); else if (v->type == adios_double) printf("] : min=%lg max=%lg\n", (*(double*)v->statistics->min), (*(double*)v->statistics->max)); /* Print block info */ for (l=0; l<v->nsteps; l++) { printf(" step %3d: \n", l); for (j=0; j<v->nblocks[l]; j++) { printf(" block %3d: [", j); for (k=0; k<v->ndim; k++) { printf("%3lld:%3lld", v->blockinfo[j].start[k], v->blockinfo[j].start[k]+v->blockinfo[j].count[k]-1); if (k<v->ndim-1) printf(", "); } printf("]\n"); } } } adios_free_varinfo (v); } /* variables */ /* For all attributes */ printf(" Attributes=%d:\n", f->nattrs); for (i = 0; i < f->nattrs; i++) { enum ADIOS_DATATYPES atype; int asize; void *adata; adios_get_attr_byid (f, i, &atype, &asize, &adata); printf(" %-9s %s = %s\n", adios_type_to_string(atype), f->attr_namelist[i], value_to_string(atype, adata, 0)); free(adata); } /* attributes */ adios_read_close (f); return 0; }
void performQuery(ADIOS_QUERY_TEST_INFO *queryInfo, ADIOS_FILE *f, int use_streaming, int print_points, int read_results) { int timestep = 0 ; ADIOS_VARINFO * tempVar = adios_inq_var(f, queryInfo->varName); adios_inq_var_blockinfo(f, tempVar); if (use_streaming) for (timestep = 0; timestep < queryInfo->fromStep; ++timestep) assert(adios_advance_step(f, 0, 0) == 0); fprintf(stderr,"times steps for variable is: [%d, %d], batch size is %" PRIu64 "\n", queryInfo->fromStep, queryInfo->fromStep + queryInfo->numSteps, queryInfo->batchSize); for (timestep = queryInfo->fromStep; timestep < queryInfo->fromStep + queryInfo->numSteps; timestep ++) { fprintf(stderr, "querying on timestep %d \n", timestep); while (1) { ADIOS_QUERY_RESULT *currBatch = NULL; currBatch = adios_query_evaluate( queryInfo->query, queryInfo->outputSelection, use_streaming ? 0 : timestep, queryInfo->batchSize ); if (currBatch == NULL) { fprintf(stderr, "Unexpected error status in querying evaluation, returned NULL as result. " "ADIOS error message: %s \n", adios_errmsg()); break; } if (currBatch->status == ADIOS_QUERY_RESULT_ERROR) { fprintf(stderr, "ERROR in querying evaluation: %s \n", adios_errmsg()); break; } if (currBatch->nselections == 0) { /* this is normal, we processed everything in previous loop or there is 0 total result */ break; } int n; if (currBatch->selections->type == ADIOS_SELECTION_POINTS) { for (n = 0; n < currBatch->nselections; n++) { const ADIOS_SELECTION_POINTS_STRUCT * retrievedPts = &(currBatch->selections[n].u.points); /* fprintf(stderr,"retrieved points %" PRIu64 " \n", retrievedPts->npoints); */ uint64_t * wboffs = calloc (retrievedPts->ndim, sizeof(uint64_t)); if (retrievedPts->container_selection && retrievedPts->container_selection->type == ADIOS_SELECTION_WRITEBLOCK) { int i; int blockidx = retrievedPts->container_selection->u.block.index; if (use_streaming) { adios_inq_var_blockinfo(f, tempVar); } else { for (i = 0; i < timestep-1; i++) blockidx += tempVar->nblocks[i]; } for (i = 0; i < retrievedPts->ndim; ++i) { wboffs[i] = tempVar->blockinfo[blockidx].start[i]; } } if (print_points) { printPoints(retrievedPts, timestep, wboffs); } free (wboffs); if (read_results) { int elmSize = adios_type_size(tempVar->type, NULL); void *data = malloc(retrievedPts->npoints * elmSize); // read returned temp data adios_schedule_read (f, &currBatch->selections[n], queryInfo->varName, use_streaming ? 0 : timestep, 1, data); adios_perform_reads(f, 1); free(data); } fprintf(stderr,"Total points retrieved %"PRIu64" in %d blocks\n", currBatch->npoints, currBatch->nselections); /* if (tempVar->type == adios_double) { */ /* for (i = 0; i < retrievedPts->npoints; i++) { */ /* fprintf(stderr,"%.6f\t", ((double*)data)[i]); */ /* } */ /* fprintf(stderr,"\n"); */ /* } */ /* else if (tempVar->type == adios_real) { */ /* for (i = 0; i < retrievedPts->npoints; i++) { */ /* fprintf(stderr,"%.6f\t", ((float*)data)[i]); */ /* } */ /* fprintf(stderr,"\n"); */ /* } */ } } else if (currBatch->selections->type == ADIOS_SELECTION_WRITEBLOCK) { fprintf(stderr,"Number of blocks retrieved: %d\n", currBatch->nselections); if (print_points) { for (n = 0; n < currBatch->nselections; n++) { fprintf(stdout,"%d %d\n", timestep, currBatch->selections[n].u.block.index); } } } free(currBatch->selections); if (currBatch->status == ADIOS_QUERY_NO_MORE_RESULTS) { free (currBatch); break; } free(currBatch); } if (use_streaming) { const int err = adios_advance_step(f, 0, 0); if (timestep < queryInfo->fromStep + queryInfo->numSteps - 1) { assert(err == 0); } else { assert(err == err_end_of_stream || err == err_step_notready); } } } adios_query_free(queryInfo->query); }
void buildIndex_mpi(ADIOS_FILE* f, ADIOS_VARINFO* v, int rank, int size) { adios_inq_var_blockinfo(f, v); int i=0; int j=0; int k=0; printf("building index on variable %d, binning op=%s\n", v->varid, gBinningOption); int blockCounter = -1; FastBitDataType ft = fastbit_adios_util_getFastbitDataType(v->type); #ifdef SINGLE_BLOCK for (i=0; i < v->nsteps; i++) { int nblocks = v->nblocks[i]; for (j=0; j < nblocks; j++) { blockCounter++; if (blockCounter % size == rank) { onBlock(rank, f, v, i, j, blockCounter, ft); fastbit_cleanup(); } } printf(" rank %d, varid %d, timestep %d total index created = %" PRIu64 ", %" PRIu64 ", %" PRIu64 ", \n", rank, v->varid, i, sum_nb, sum_nk, sum_no); printf(" rank %d, varid %d, timestep %d total bytes = %" PRIu64 ", %" PRIu64 ", %" PRIu64 ", \n", rank, v->varid, i, adios_type_size(adios_unsigned_integer , NULL)*sum_nb, adios_type_size(adios_double, NULL)*sum_nk, adios_type_size(adios_long, NULL)*sum_no); printf("\n"); } #endif #ifdef MULTI_BLOCK for (i=0; i<v->nsteps; i++) { int nblocks = v->nblocks[i]; int blockStart = 0; int blockEnd = blockStart+pack; while (blockStart < nblocks) { if (blockEnd > nblocks) { blockEnd = nblocks; } printf("block start=%d, end=%d, max=%d\n", blockStart, blockEnd, nblocks); if (workCounter % size == rank) { printf("%ld mod %ld == %d \n", workCounter, size, rank); onMultiBlock(rank, f, v, i, blockStart, blockEnd, ft); fastbit_cleanup(); } workCounter ++; blockStart += pack; blockEnd = blockStart+pack; } fastbit_cleanup(); } #endif #ifdef BOX for (i=0; i<v->nsteps; i++) { uint64_t s = 0; uint64_t dataSize = 1; uint64_t start[v->ndim]; for (s=0; s<v->ndim; s++) { dataSize *= v->dims[s]; start[s] = 0; } uint64_t split = dataSize/recommended_index_ele; uint64_t startRef = 0; if (split == 0) { if (rank == 0) { onBox(rank, f, v, i, start, v->dims, ft); } } else { while (startRef < v->dims[0]) { uint64_t count[v->ndim]; start[0] = startRef; startRef += v->dims[0]/split; if (startRef >= v->dims[0]) { startRef = v->dims[0]; } count[0] = startRef - start[0]; for (s=1; s<v->ndim; s++) { start[s] = 0; count[s] = v->dims[s]; } if (workCounter % size == rank) { onBox(rank, f, v, i, start, count, ft); } workCounter ++; } } } #endif }
int main (int argc, char ** argv) { int i, j, k, l, t; MPI_Comm comm_dummy = 0; /* MPI_Comm is defined through adios_read.h */ void * data = NULL; uint64_t start[] = {0,0,0,0,0,0,0,0,0,0}; uint64_t count[10]; ADIOS_SELECTION *sel; if (argc < 2) { printf("Usage: %s <BP-file>\n", argv[0]); return 1; } ADIOS_FILE * f; //int step; //for (step=0; step < 2; step++) { f = adios_read_open_file (argv[1], ADIOS_READ_METHOD_BP, comm_dummy); if (f == NULL) { printf ("%s\n", adios_errmsg()); return -1; } /* For all variables */ printf(" Variables=%d:\n", f->nvars); for (i = 0; i < f->nvars; i++) { ADIOS_VARINFO * v = adios_inq_var_byid (f, i); adios_inq_var_stat (f, v, 0, 0); uint64_t total_size = adios_type_size (v->type, v->value); for (j = 0; j < v->ndim; j++) total_size *= v->dims[j]; printf(" %-9s %s", adios_type_to_string(v->type), f->var_namelist[i]); if (v->ndim == 0) { /* Scalars do not need to be read in, we get it from the metadata when using adios_inq_var */ printf(" = %s\n", value_to_string(v->type, v->value, 0)); } else { /* Arrays have to be read in from the file */ if (v->nsteps > 1) { printf(" %d*",v->nsteps); } printf("[%" PRIu64,v->dims[0]); for (j = 1; j < v->ndim; j++) printf(", %" PRIu64,v->dims[j]); //printf("] = \n"); if (v->type == adios_integer) printf("] = min=%d max=%d\n", (*(int*)v->statistics->min), (*(int*)v->statistics->max)); else if (v->type == adios_double) printf("] = min=%lg max=%lg\n", (*(double*)v->statistics->min), (*(double*)v->statistics->max)); if (total_size > 1024*1024*1024) { printf(" // too big, do not read in\n"); } else { data = malloc (total_size); if (data == NULL) { fprintf (stderr, "malloc failed.\n"); return -1; } for (j = 0; j < v->ndim; j++) count[j] = v->dims[j]; for (t=0; t<v->nsteps; t++) { sel = adios_selection_boundingbox (v->ndim, start, count); adios_schedule_read_byid (f, sel, i, t, 1, data); adios_perform_reads (f, 1); printf(" Step %d:\n", t); if (adios_errno) { printf ("%s\n", adios_errmsg()); } else if (total_size > 1024*1024) { printf ("Too big to print\n"); } else if (v->ndim == 1) { printf (" ["); for (j = 0; j < v->dims[0]; j++) printf("%s ", value_to_string(v->type, data, j)); printf ("]\n"); } else if (v->ndim == 2) { for (j = 0; j < v->dims[0]; j++) { printf (" row %d: [", j); for (k = 0; k < v->dims[1]; k++) printf("%s ", value_to_string(v->type, data, j*v->dims[1] + k)); printf ("]\n"); } } else if (v->ndim == 3) { for (j = 0; j < v->dims[0]; j++) { printf (" block %d: \n", j); for (k = 0; k < v->dims[1]; k++) { printf (" row %d: [", k); for (l = 0; l < v->dims[2]; l++) { // NCSU ALACRITY-ADIOS - Fixed bug, k*v->dims[1] changed to k*v->dims[2] printf("%s ", value_to_string(v->type, data, j*v->dims[1]*v->dims[2] + k*v->dims[2] + l)); } printf ("]\n"); } printf ("\n"); } } else { printf (" cannot print arrays with >3 dimensions\n"); } } free (data); } } adios_free_varinfo (v); } /* variables */ /* For all attributes */ printf(" Attributes=%d:\n", f->nattrs); for (i = 0; i < f->nattrs; i++) { enum ADIOS_DATATYPES atype; int asize; void *adata; adios_get_attr_byid (f, i, &atype, &asize, &adata); int type_size = adios_type_size (atype, adata); int nelems = asize / type_size; printf(" %-9s %s = ", adios_type_to_string(atype), f->attr_namelist[i]); char *p = (char*)adata; if (nelems>1) printf("{"); for (j=0; j<nelems; j++) { if (j>0) printf(", "); printf ("%s", value_to_string(atype, p, 0)); p += type_size; } if (nelems>1) printf("}"); printf("\n"); free(adata); } /* attributes */ adios_read_close (f); //} /* loop 'step' */ return 0; }
void performQuery(ADIOS_QUERY_TEST_INFO *queryInfo, ADIOS_FILE *f, int use_streaming, int print_points, int read_results) { int timestep = 0 ; ADIOS_VARINFO * tempVar = adios_inq_var(f, queryInfo->varName); if (use_streaming) for (timestep = 0; timestep < queryInfo->fromStep; ++timestep) assert(adios_advance_step(f, 0, 0) == 0); fprintf(stderr,"times steps for variable is: [%d, %d], batch size is %" PRIu64 "\n", queryInfo->fromStep, queryInfo->fromStep + queryInfo->numSteps, queryInfo->batchSize); for (timestep = queryInfo->fromStep; timestep < queryInfo->fromStep + queryInfo->numSteps; timestep ++) { fprintf(stderr, "querying on timestep %d \n", timestep); ADIOS_SELECTION* currBatch = NULL; while (adios_query_evaluate(queryInfo->query, queryInfo->outputSelection, use_streaming ? 0 : timestep, queryInfo->batchSize, &currBatch) >= 0) { if (currBatch == NULL) { break; } assert(currBatch->type ==ADIOS_SELECTION_POINTS); const ADIOS_SELECTION_POINTS_STRUCT * retrievedPts = &(currBatch->u.points); /* fprintf(stderr,"retrieved points %" PRIu64 " \n", retrievedPts->npoints); */ if (print_points) { printPoints(retrievedPts, timestep); } if (read_results) { int elmSize = adios_type_size(tempVar->type, NULL); void *data = malloc(retrievedPts->npoints * elmSize); // read returned temp data adios_schedule_read (f, currBatch, queryInfo->varName, use_streaming ? 0 : timestep, 1, data); adios_perform_reads(f, 1); free(data); } fprintf(stderr,"Total data retrieved:%"PRIu64"\n", retrievedPts->npoints); /* if (tempVar->type == adios_double) { */ /* for (i = 0; i < retrievedPts->npoints; i++) { */ /* fprintf(stderr,"%.6f\t", ((double*)data)[i]); */ /* } */ /* fprintf(stderr,"\n"); */ /* } */ /* else if (tempVar->type == adios_real) { */ /* for (i = 0; i < retrievedPts->npoints; i++) { */ /* fprintf(stderr,"%.6f\t", ((float*)data)[i]); */ /* } */ /* fprintf(stderr,"\n"); */ /* } */ adios_selection_delete(currBatch); currBatch = NULL; } if (use_streaming) { const int err = adios_advance_step(f, 0, 0); if (timestep < queryInfo->fromStep + queryInfo->numSteps - 1) { assert(err == 0); } else { assert(err == err_end_of_stream || err == err_step_notready); } } } adios_query_free(queryInfo->query); }
int main (int argc, char ** argv) { char filename [256]; int rank, size, gidx, i, j, k,l; MPI_Comm comm_dummy = MPI_COMM_WORLD; /* MPI_Comm is defined through adios_read.h */ enum ADIOS_DATATYPES attr_type; void * data = NULL; uint64_t start[] = {0,0,0,0,0,0,0,0,0,0}; uint64_t count[MAX_DIMS], hcount[MAX_DIMS], bytes_read = 0; herr_t h5_err; char h5name[256],aname[256],fname[256]; int dims [MAX_DIMS]; int h5rank[MAX_DIMS]; int h5i, level; hid_t grp_id [GMAX+1], space_id, dataset_id; hid_t memspace_id, dataspace_id, att_id; char ** grp_name; hid_t type_id; hid_t h5_type_id; hsize_t adims; if (argc < 2) { printf("Usage: %s <BP-file> <HDF5-file>\n", argv[0]); return 1; } MPI_Init(&argc, &argv); h5_err = H5Eset_auto(NULL, NULL ); ADIOS_FILE * f = adios_fopen (argv[1], comm_dummy); HDF5_FILE = H5Fcreate(argv[2],H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); /* create the complex types for HDF5 */ complex_real_id = H5Tcreate (H5T_COMPOUND, sizeof (complex_real_t)); H5Tinsert (complex_real_id, "real", HOFFSET(complex_real_t,re), H5T_NATIVE_FLOAT); H5Tinsert (complex_real_id, "imaginary", HOFFSET(complex_real_t,im), H5T_NATIVE_FLOAT); complex_double_id = H5Tcreate (H5T_COMPOUND, sizeof (complex_double_t)); H5Tinsert (complex_double_id, "real", HOFFSET(complex_double_t,re), H5T_NATIVE_DOUBLE); H5Tinsert (complex_double_id, "imaginary", HOFFSET(complex_double_t,im), H5T_NATIVE_DOUBLE); if (f == NULL) { if (DEBUG) printf ("%s\n", adios_errmsg()); return -1; } /* For all groups */ for (gidx = 0; gidx < f->groups_count; gidx++) { if (DEBUG) printf("Group %s:\n", f->group_namelist[gidx]); ADIOS_GROUP * g = adios_gopen (f, f->group_namelist[gidx]); if (g == NULL) { if (DEBUG) printf ("%s\n", adios_errmsg()); return -1; } /* First create all of the groups */ grp_id [0] = HDF5_FILE; for (i = 0; i < g->vars_count; i++) { ADIOS_VARINFO * v = adios_inq_var_byid (g, i); strcpy(h5name,g->var_namelist[i]); grp_name = bp_dirparser (h5name, &level); for (j = 0; j < level-1; j++) { grp_id [j + 1] = H5Gopen (grp_id [j], grp_name [j]); if (grp_id [j + 1] < 0) { grp_id [j + 1] = H5Gcreate (grp_id [j], grp_name [j], 0); } } for (j=1; j<level; j++) { H5Gclose(grp_id[j]); } } /* Now we can write data into these scalars */ /* For all variables */ if (DEBUG) printf(" Variables=%d:\n", g->vars_count); for (i = 0; i < g->vars_count; i++) { ADIOS_VARINFO * v = adios_inq_var_byid (g, i); uint64_t total_size = adios_type_size (v->type, v->value); for (j = 0; j < v->ndim; j++) total_size *= v->dims[j]; strcpy(h5name,g->var_namelist[i]); if (DEBUG) printf(" %-9s %s", adios_type_to_string(v->type), g->var_namelist[i]); h5_err = bp_getH5TypeId (v->type, &h5_type_id); if (v->type==adios_string) H5Tset_size(h5_type_id,strlen(v->value)); if (v->ndim == 0) { /* Scalars do not need to be read in, we get it from the metadata when using adios_inq_var */ if (DEBUG) printf(" = %s\n", value_to_string(v->type, v->value, 0)); // add the hdf5 dataset, these are scalars for (h5i = 0;h5i<MAX_DIMS;h5i++) count[0] = 0; count[0] = 1; // we are writing just 1 element, RANK=1 h5_err = bp_getH5TypeId (v->type, &h5_type_id); H5LTmake_dataset(HDF5_FILE,h5name,1,count,h5_type_id,v->value); } else { h5_err = readVar(g, v, h5name); } adios_free_varinfo (v); } /* variables */ /* For all attributes */ if (DEBUG) printf(" Attributes=%d:\n", g->attrs_count); for (i = 0; i < g->attrs_count; i++) { enum ADIOS_DATATYPES atype; int asize; void *adata; adios_get_attr_byid (g, i, &atype, &asize, &adata); grp_name = bp_dirparser (g->attr_namelist[i], &level); strcpy(aname,grp_name[level-1]); // the name of the attribute is the last in the array // we then need to concat the rest together strcpy(fname,"/"); for (j=0;j<level-1;j++) { strcat(fname,grp_name[j]); } h5_err = bp_getH5TypeId (atype, &h5_type_id); // let's create the attribute adims = 1; if (atype==adios_string) H5Tset_size(h5_type_id,strlen(adata)); space_id = H5Screate(H5S_SCALAR); // just a scalar att_id = H5Acreate(HDF5_FILE, g->attr_namelist[i], h5_type_id, space_id,H5P_DEFAULT); h5_err = H5Awrite(att_id, h5_type_id, adata); h5_err = H5Aclose(att_id); h5_err = H5Sclose(space_id); if (DEBUG) printf(" %-9s %s = %s\n", adios_type_to_string(atype), g->attr_namelist[i], value_to_string(atype, adata, 0)); free(adata); } /* attributes */ adios_gclose (g); } /* groups */ adios_fclose (f); h5_err = H5Fclose(HDF5_FILE); MPI_Finalize(); return 0; }