Beispiel #1
0
void ToolsAdiosParallel::listAvailableDatasets()
{
    ADIOS_VARINFO *pVarInfo;

    //available data sets in this file
    m_outStream << "Number of available data sets: ";
    m_outStream << pFile->nvars << std::endl;

    for(int i = 0; i < pFile->nvars; i++)
    {
        m_outStream << pFile->var_namelist[i] << " ";

        pVarInfo = adios_inq_var(pFile, pFile->var_namelist[i]);

        if(pVarInfo->ndim > 0)
        {
            // print number of elements per dimension to m_outstream
            m_outStream << "[";
            for(int j = 0; j < pVarInfo->ndim; j++)
            {
                m_outStream << pVarInfo->dims[j];
                if(j < pVarInfo->ndim-1)
                {
                    m_outStream << ",";
                }
            }
            m_outStream << "]" << std::endl;
        }
        else
        {
            m_outStream << "[scalar]" << std::endl;
        }
        adios_free_varinfo(pVarInfo);
    }
}
Beispiel #2
0
int64_t getByteEstimation(ADIOS_FILE* f, int rank, int argc, char** argv)
{
  uint64_t bytes = 0;

  if (argc >= 3) {
    int i=2;
    while (i<argc) {
      const char* varName = argv[i];      
      if(strstr(varName, "<binning prec") != NULL) {
	if (gBinningOption == NULL) {
	   gBinningOption = argv[i];
	}
	if (argc == 3) {
	  return getByteEstimationOnFile(f, rank);
	}
	i++;
	continue;
      }
      ADIOS_VARINFO * v = adios_inq_var(f, varName);
      if (v == NULL) {
	printf("Invalid variable: %s\n", varName);	
	return -1;
      }
      uint64_t varSize = estimateBytesOnVar(f, v);
      printf(" var: %s has size: %" PRId64 "\n", varName, varSize);
      bytes += varSize;
      adios_free_varinfo(v);
      i++;
    } 
    return bytes;
  } else { 
    return getByteEstimationOnFile(f,rank);
  }
}
Beispiel #3
0
void adios_checkpoint_verify_variables(ADIOS_FILE* fp, const char* name, int* origin)
{
  ADIOS_VARINFO *vi;
  int count = 1;
  int size;
  vi = adios_inq_var(fp, name);
  if (vi->ndim > 0)
  {
    cout<<name<<" verification not passed, not a scalar"<<endl;
    return;
  }
  size = count*adios_type_size(vi->type, vi->value);
  int * mem= (int * )malloc(size);
  ADIOS_SELECTION *sel = adios_selection_writeblock(OHMMS::Controller->rank());
  adios_schedule_read(fp, sel, name, 0, 1, mem);
  //adios_schedule_read(fp, NULL, name, 0, 1, mem);
  adios_perform_reads(fp, 1);
  if(mem[0] == *origin)
  {
    cout<<name<<" verification passed "<<mem[0]<<endl;
  }
  else
  {
    cout<<name<<" verification not passed, readin: "<<mem[0]<<" writeout: "<<*origin<<endl;
  }
  adios_free_varinfo (vi);
  adios_selection_delete(sel);
  free(mem);
}
Beispiel #4
0
int getJobCounter(ADIOS_FILE* f)
{
  int numVars = f->nvars;
  int counter = 0;

  int i=0, k=0;
  for (i=0; i<numVars; i++) {
    char* varName = f->var_namelist[i];
    ADIOS_VARINFO* v = adios_inq_var(f, varName);
    if (v->ndim == 0) {
      continue;
    }
    for (k=0; k<v->nsteps; k++) {
#ifdef MULTI_BOX
      int nblocks = v->nblocks[k];
      printf("var = %s, timestep = %ld,  nblocks=%ld\n", varName, k, nblocks);
      int remainder = nblocks % pack;
      counter += nblocks/pack;
#endif
#ifdef BOX
      int j=0;
      uint64_t totalElements =1;
      for (j=0; j<v->ndim; j++) {
	totalElements *= v->dims[j];
      }
      int remainder = totalElements % recommended_index_ele;      
      counter += totalElements/recommended_index_ele;      
#endif
      if (remainder > 0) {
	counter += 1;
      }
    }
  }
  return counter;
}
Beispiel #5
0
void adios_checkpoint_verify_variables(ADIOS_FILE* fp, const char* name, RealType* origin)
{
  ADIOS_VARINFO *vi;
  int count = 1;
  int size;
  vi = adios_inq_var(fp, name);
  adios_inq_var_blockinfo (fp, vi);
  if (vi->ndim > 0)
  {
    count*=vi->dims[0];
    for (int j = 1; j < vi->ndim; j++)
    {
      count *= vi->dims[j];
    }
  }
  size = count*adios_type_size(vi->type, vi->value);
  RealType *mem= (RealType *)malloc(size);
  adios_schedule_read(fp, NULL, name, 0, 1, mem);
  adios_perform_reads(fp, 1);
  for(int i=0; i<count; i++)
  {
    if(mem[i] == origin[i])
    {
      cout<<name<<"["<<i<<"]verification passed "<<mem[i]<<endl;
    }
    else
    {
      cout<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl;
    }
  }
  adios_free_varinfo (vi);
  free(mem);
}
Beispiel #6
0
void adios_checkpoint_verify_variables(ADIOS_FILE* fp, const char* name, unsigned long origin)
{
  ADIOS_VARINFO *vi;
  int count = 1;
  int size;
  vi = adios_inq_var(fp, name);
  if (vi->ndim > 0)
  {
    cout<<name<<" verification not passed, not a scalar"<<endl;
    return;
  }
  size = count*adios_type_size(vi->type, vi->value);
  unsigned long* mem= (unsigned long* )malloc(size);
  adios_schedule_read(fp, NULL, name, 0, 1, mem);
  adios_perform_reads(fp, 1);
  if(mem[0] == origin)
  {
    cout<<name<<" verification passed"<<mem[0]<<endl;
  }
  else
  {
    cout<<name<<" verification not passed, readin: "<<mem[0]<<" writeout: "<<origin<<endl;
  }
  adios_free_varinfo (vi);
  free(mem);
}
Beispiel #7
0
/// Constructor.
ADIOS_Var::ADIOS_Var(ADIOS_File* afile, const char* vn)
    : m_file(afile), m_name(vn), m_handle(0)
{
    if (afile != 0 && afile->getHandle() != 0)
        m_handle = adios_inq_var(afile->getHandle(), vn);
    LOGGER(ibis::gVerbose > 8)
        << "adios_inq_var is called for " << m_name << ": m_handle="
        << m_handle;
}
std::string AdiosCheckpointInput::getScalar(const std::string& var_name)
{
  //Scalars are all stored in the metadata so we can read them without disk access
  ADIOS_VARINFO* adios_inq = adios_inq_var(adios_file_handle, var_name.c_str());
  int string_size = adios_type_size(adios_inq->type, adios_inq->value);
  std::string string_val = static_cast<char*>(*adios_inq->value);
  adios_free_varinfo(adios_inq);
  return string_val;
}
Beispiel #9
0
void adios_checkpoint_verify_random_variables(ADIOS_FILE* fp, const char* name, uint_type* origin)
{
  ADIOS_VARINFO *vi;
  int count_int = 1;
  int size;
  uint64_t *start;
  uint64_t *count;
  vi = adios_inq_var(fp, name);
  adios_inq_var_blockinfo (fp, vi);
  if (vi->ndim > 0)
  {
    start = (uint64_t *)malloc(vi->ndim * sizeof(uint64_t));
    count = (uint64_t *)malloc(vi->ndim * sizeof(uint64_t));
  }
  for (int j=0; j<vi->nblocks[0]; j++)
  {
    if(j == OHMMS::Controller->rank())
    {
      for (int k=0; k<vi->ndim; k++)
      {
        start[k] = vi->blockinfo[j].start[k];
        count[k] = vi->blockinfo[j].count[k];
        count_int *= count[k];
        //cout<<OHMMS::Controller->rank()<<" count "<<start[k]<<" "<<count[k]<<endl;
      }
    }
  }
  size = count_int*adios_type_size(vi->type, vi->value);
  uint_type *mem= (uint_type*)malloc(size);
  ADIOS_SELECTION *sel = adios_selection_boundingbox(vi->ndim, start, count);
  adios_schedule_read(fp, sel, name, 0, 1, mem);
  adios_perform_reads(fp, 1);
  int flag = 0;
  for(int i=0; i<count_int; i++)
  {
    if(mem[i] == origin[i])
    {
      //cout<<name<<"["<<i<<"]verification passed "<<mem[i]<<endl;
    }
    else
    {
      flag = 1;
      cout<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl;
    }
  }
  if (flag == 0) cout<<name<<" verification passed "<<endl;
  else cout<<name<<" verification not passed "<<endl;
  adios_free_varinfo (vi);
  adios_selection_delete(sel);
  free(start);
  free(count);
  free(mem);
}
Beispiel #10
0
void ADIOS1CommonRead::ScheduleReadCommon(const std::string &name,
                                          const Dims &offs, const Dims &ldims,
                                          const int fromStep, const int nSteps,
                                          const bool readAsLocalValue,
                                          const bool readAsJoinedArray,
                                          void *data)
{
    if (readAsLocalValue)
    {
        /* Get all the requested values from metadata now */
        ADIOS_VARINFO *vi = adios_inq_var(m_fh, name.c_str());
        if (vi)
        {
            adios_inq_var_stat(m_fh, vi, 0, 1);
            int elemsize = adios_type_size(vi->type, nullptr);
            long long blockidx = 0;
            for (int i = 0; i < fromStep; i++)
            {
                blockidx += vi->nblocks[i];
            }
            char *dest = (char *)data;
            for (int i = fromStep; i < fromStep + nSteps; i++)
            {
                for (int j = 0; j < vi->nblocks[i]; j++)
                {
                    memcpy(dest, vi->statistics->blocks->mins[blockidx],
                           elemsize);
                    ++blockidx;
                    dest += elemsize;
                }
            }
            adios_free_varinfo(vi);
        }
    }
    else
    {
        uint64_t start[32], count[32];
        for (int i = 0; i < ldims.size(); i++)
        {
            start[i] = (uint64_t)offs[i];
            count[i] = (uint64_t)ldims[i];
        }
        ADIOS_SELECTION *sel = nullptr;
        if (ldims.size() > 0)
        {
            sel = adios_selection_boundingbox(ldims.size(), start, count);
        }
        adios_schedule_read(m_fh, sel, name.c_str(), (int)fromStep, (int)nSteps,
                            data);
        adios_selection_delete(sel);
    }
}
T AdiosCheckpointInput::getScalar(const std::string& var_name)
{
  //Scalars are all stored in the metadata so we can read them without disk access
  ADIOS_VARINFO* adios_inq = adios_inq_var(adios_file_handle, var_name.c_str());
  if (adios_type_size(adios_inq->type, adios_inq->value) != sizeof(T))
  {
    qmcplusplus::app_error() << "Data type does not match data type found in file: " << std::endl;
    return ;
  }
  T scalar_value = static_cast<T>(*adios_inq->value);
  adios_free_varinfo(adios_inq);
  return scalar_value;
}
Beispiel #12
0
void ToolsAdiosParallel::convertToText()
{
    if(m_options.data.size() == 0)
        throw std::runtime_error("No datasets requested");

    for (size_t i = 0; i < m_options.data.size(); ++i)
    {
        ADIOS_VARINFO *pVarInfo;

        //get name of dataset to print
        std::string nodeName = m_options.data[i];

        uint8_t *P;
        int varElement = 1;
        int varTypeSize = 0;

        adios_read_init_method(ADIOS_READ_METHOD_BP, comm, nodeName.c_str());

        pVarInfo = adios_inq_var(pFile, nodeName.c_str());

        varTypeSize = adios_type_size(pVarInfo->type, NULL);

        // get number of elements combined in a dataset
        for(int j = 0; j < pVarInfo->ndim; j++)
        {
            varElement = varElement * pVarInfo->dims[j];
        }
        // allocate memory
        P = (uint8_t*) malloc (sizeof(uint8_t) * varTypeSize * varElement);

        adios_schedule_read(pFile, NULL, nodeName.c_str(), 0, 1, P);

        adios_perform_reads(pFile, 1);

        if(pVarInfo->ndim > 0)
        {        
            for(int k = 0; k < varElement; k++)
            {
                printValue(pVarInfo->type, &P[k*varTypeSize]);
            }
        }
        else
        {
            printValue(pVarInfo->type, pVarInfo->value);
        }
        adios_free_varinfo(pVarInfo);
    }
}
Beispiel #13
0
    /** Read the skalar field and optionally the attribute into the values referenced by the pointers */
    void operator()(ThreadParams& params,
                const std::string& name, T_Scalar* value,
                const std::string& attrName = "", T_Attribute* attribute = nullptr)
    {
        log<picLog::INPUT_OUTPUT> ("ADIOS: read %1%D scalars: %2%") % simDim % name;
        std::string datasetName = params.adiosBasePath + name;

        ADIOS_VARINFO* varInfo;
        ADIOS_CMD_EXPECT_NONNULL( varInfo = adios_inq_var(params.fp, datasetName.c_str()) );
        if(varInfo->ndim != simDim)
            throw std::runtime_error(std::string("Invalid dimensionality for ") + name);
        if(varInfo->type != traits::PICToAdios<T_Scalar>().type)
            throw std::runtime_error(std::string("Invalid type for ") + name);

        DataSpace<simDim> gridPos = Environment<simDim>::get().GridController().getPosition();
        uint64_t start[varInfo->ndim];
        uint64_t count[varInfo->ndim];
        for(int d = 0; d < varInfo->ndim; ++d)
        {
            /* \see adios_define_var: z,y,x in C-order */
            start[d] = gridPos.revert()[d];
            count[d] = 1;
        }

        ADIOS_SELECTION* fSel = adios_selection_boundingbox(varInfo->ndim, start, count);

        // avoid deadlock between not finished pmacc tasks and mpi calls in adios
        __getTransactionEvent().waitForFinished();

        /* specify what we want to read, but start reading at below at `adios_perform_reads` */
        /* magic parameters (0, 1): `from_step` (not used in streams), `nsteps` to read (must be 1 for stream) */
        log<picLog::INPUT_OUTPUT > ("ADIOS: Schedule read skalar %1%)") % datasetName;
        ADIOS_CMD( adios_schedule_read(params.fp, fSel, datasetName.c_str(), 0, 1, (void*)value) );

        /* start a blocking read of all scheduled variables */
        ADIOS_CMD( adios_perform_reads(params.fp, 1) );

        adios_selection_delete(fSel);
        adios_free_varinfo(varInfo);

        if(!attrName.empty())
        {
            log<picLog::INPUT_OUTPUT> ("ADIOS: read attribute %1% for scalars: %2%") % attrName % name;
            *attribute = readAttribute<T_Attribute>(params.fp, datasetName, attrName);
        }
    }
Beispiel #14
0
int print_varinfo (ADIOS_FILE *f, int start_step) 
{
    ADIOS_VARINFO * v;
    int i,j,k;

    v = adios_inq_var (f, "t");
    adios_inq_var_blockinfo (f, v);

    printf ("ndim = %d\n",  v->ndim);
    printf ("dims[%llu]",  v->dims[0]);
    if (v->dims[0] != gdims[start_step]) 
    {
        printf ("\tERROR: expected [%llu]", gdims[start_step]);
        nerrors++;
    }
    printf("\n");
    printf ("nsteps = %d\n",  v->nsteps);
    printf ("sum_nblocks = %d\n",  v->sum_nblocks);
    k = 0; // blockinfo is a contigous 1D array of elements from 0 to v->sum_nblocks-1
    for (i = 0; i < v->nsteps; i++) {
        printf ("  nblocks[%d] = %d\n", i, v->nblocks[i]);
        for (j = 0; j < v->nblocks[i]; j++) {
            printf("    block %2d: [%llu:%llu]", j,
                        v->blockinfo[k].start[0],
                        v->blockinfo[k].start[0] + v->blockinfo[k].count[0]-1);
            
            if (v->blockinfo[k].start[0] != block_offset [(start_step+i)*nblocks_per_step*size + j] ||
                v->blockinfo[k].count[0] != block_count  [(start_step+i)*nblocks_per_step*size + j] ) 
            {
                nerrors++;
                printf ("\tERROR: expected [%llu:%llu]",
                    block_offset [(start_step+i)*nblocks_per_step*size + j],
                    block_offset [(start_step+i)*nblocks_per_step*size + j] + 
                      block_count  [(start_step+i)*nblocks_per_step*size + j] -1
                );
            }
            printf("\n");
            k++;
        }
    }
    adios_free_varinfo (v);
}
Beispiel #15
0
static void check_xform_var(ADIOS_GROUP *ag, double *arr) {
    int64_t readlen;
    double *readarr = malloc(N * sizeof(double));
    memset(readarr, 0, N*sizeof(double));

    // Check the transformed data
    ADIOS_VARINFO *vi_xform = adios_inq_var(ag, XFORM_VAR);
    assert(vi_xform);
    assert(vi_xform->type == adios_double);
    assert(vi_xform->ndim == 1); // Apparently the "dummy" dimension has gone away for timed arrays that are written only once
    assert(vi_xform->dims[0] == N);

    readlen = adios_read_var_byid(ag, vi_xform->varid, (uint64_t[]){0}, vi_xform->dims, readarr);

    assert(readlen == N * sizeof(double));
    assert(memcmp(readarr, arr, N * sizeof(double)) == 0);

    // Cleanup
    adios_free_varinfo(vi_xform);
    free(readarr);
}
void AdiosCheckpointInput::getVector(const std::string& var_name, vector<T>& buffer)
{
  ADIOS_VARINFO* adios_inq = adios_inq_var(adios_file_handle, var_name.c_str());
  //Check to make sure the T is the same size as the data type on the disk
  if (adios_type_size(adios_inq->type, NULL) != sizeof(T))
  {
    qmcplusplus::app_error() << "Data type does not match data type found in file: " << std::endl;
    APP_ABORT("qmcapp");
    return ;
  }
  uint64_t total_size = 1;
  for (int i = 0; i < adios_inq->ndim; i++)
    total_size *= adios_inq->dims[i];
  //make sure we have enough space to copy all the data from the file
  buffer.reserve(total_size);
  //schedule the read
  adios_schedule_read(adios_file_handle, sel, var_name.c_str(), 0, 1, &(buffer[0]));
  //perform the read
  adios_perform_reads(adios_file_handle, 1);
  //Don't need the information about the variable anymore
  adios_free_varinfo(adios_inq);
}
Beispiel #17
0
int64_t getByteEstimationOnFile(ADIOS_FILE* f, int rank) 
{
  // check all vars
  int numVars = f->nvars;
  uint64_t bytes = 0;

  int i=0;
  for (i=0; i<numVars; i++) {
    char* varName = f->var_namelist[i];
    ADIOS_VARINFO* v = adios_inq_var(f, varName);

    uint64_t varSize = estimateBytesOnVar(f, v);
    if (rank == 0) {
      printf(" var: %s has size: %" PRId64 "\n", varName, varSize);
    }
    bytes += varSize;

    adios_free_varinfo(v);
  }
  return bytes;

}
void AdiosCheckpointInput::performReads()
{
  for (int i = 0; i < read_queue.size(); i++)
  {
    const char* var_name = read_queue[i].c_str();
    //grab the information from the metadata about this variable
    ADIOS_VARINFO* adios_inq = adios_inq_var(adios_file_handle, var_name);
    //Calculate the total size of the vector based on the metadata
    uint64_t total_size = adios_type_size(adios_inq->type, NULL);
    for (int i = 0; i < adios_inq->ndim; i++)
      total_size *= adios_inq->dims[i];
    //allocate and then store for when retrieve
    void* buff = malloc(total_size);
    buffers.push_back(buff);
    //schedule the read
    adios_schedule_read(adios_file_handle, sel, var_name, 0, 1, buff);
    //Don't need the information about the variable anymore
    adios_free_varinfo(adios_inq);
  }
  //perform the read
  adios_perform_reads(adios_file_handle, 1);
}
Beispiel #19
0
void adios_trace_verify_local_variables(ADIOS_FILE* fp, const char* name, double* origin)
{
  ADIOS_VARINFO *vi;
  int count = 1;
  unsigned long size = 1;
  vi = adios_inq_var(fp, name);
  adios_inq_var_blockinfo(fp, vi);
  for (int j=0; j<vi->nblocks[0]; j++)
  {
    if(OHMMS::Controller->rank() == j)
    {
      for (int k=0; k<vi->ndim; k++)
      {
        count *= vi->blockinfo[j].count[k];
      }
    }
  }
  size = count * adios_type_size(vi->type, vi->value);
  double* mem = (double*)malloc(size);
  ADIOS_SELECTION *sel = adios_selection_writeblock(OHMMS::Controller->rank());
  adios_schedule_read(fp, sel, name, 0, 1, mem);
  adios_perform_reads(fp, 1);
  int flag = 0;
  for(int i=0; i<count; i++)
  {
    if(mem[i] == origin[i])
    {
    }
    else
    {
      flag = 1;
      cout<<OHMMS::Controller->rank()<<" "<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl;
    }
  }
  if(flag == 0) cout<<OHMMS::Controller->rank()<<" "<<name<<" verification passed"<<endl;
  adios_selection_delete(sel);
  adios_free_varinfo (vi);
  free(mem);
}
Beispiel #20
0
void buildIndexOnAllVar(ADIOS_FILE* f, int rank, int size) 
{
  int numVars = f->nvars;
  
  int i=0;
  for (i=0; i<numVars; i++) {
    char* varName = f->var_namelist[i];
    ADIOS_VARINFO* v = adios_inq_var(f, varName);

    if (rank == 0) {
      printf("\n==> building fastbit index on  %dth variable: %s, %s ", i, varName, gBinningOption);
    }

    if (v->ndim > 0) {
      buildIndex_mpi(f, v, rank, size);
    } else {
      if (rank == 0) {
	printf("\t ... skipping scalar ...\n");
      }
    }
    adios_free_varinfo(v);
  }
}
static void test_file_mode_reads_on_var(ADIOS_FILE *fp, const char *bp_filename, const char *varname) {
	int i;

	ADIOS_VARINFO *varinfo = adios_inq_var(fp, varname);
	MPI_Assert(COMM, varinfo);

	if (varinfo->value != NULL) {
		//if (rank == 0) fprintf(stderr, "(skipping scalar variable '%s')\n", varname);
		adios_free_varinfo(varinfo);
		return;
	}

	fprintf(stderr, "[rank %d/%d] Starting file-mode writeblock reads on %s:/%s\n", rank, size, bp_filename, varname);

	adios_inq_var_blockinfo(fp, varinfo);
	MPI_Assert(COMM, varinfo->blockinfo);

	const enum ADIOS_DATATYPES datatype = varinfo->type;
	const int datatypesize = adios_get_type_size(datatype, NULL);

	int timestep, timestep_blockidx, blockidx = 0;
	for (timestep = 0; timestep < varinfo->nsteps; ++timestep) {
		for (timestep_blockidx = 0; timestep_blockidx < varinfo->nblocks[timestep]; ++timestep_blockidx, ++blockidx) {
			if (blockidx % size != rank) continue;

			const ADIOS_VARBLOCK *vb = &varinfo->blockinfo[blockidx];

			ADIOS_SELECTION *block_bb = adios_selection_boundingbox(varinfo->ndim, vb->start, vb->count);
			ADIOS_SELECTION *block_wb = adios_selection_writeblock(timestep_blockidx);
			ADIOS_SELECTION *block_abs_wb = adios_selection_writeblock(blockidx);
			block_abs_wb->u.block.is_absolute_index = 1;

			uint64_t blocksize = datatypesize;
			for (i = 0; i < varinfo->ndim; ++i)
				blocksize *= vb->count[i];

			void *buf_bb = malloc(blocksize);
			void *buf_wb = malloc(blocksize);
			void *buf_abs_wb = malloc(blocksize);
			memset(buf_bb,     0, blocksize);
			memset(buf_wb,     1, blocksize);
			memset(buf_abs_wb, 2, blocksize);
			MPI_Assert(COMM, buf_bb && buf_wb && buf_abs_wb);

			adios_schedule_read(fp, block_bb,     varname, timestep, 1, buf_bb    );
			adios_schedule_read(fp, block_wb,     varname, timestep, 1, buf_wb    );
			adios_schedule_read(fp, block_abs_wb, varname, timestep, 1, buf_abs_wb);
			adios_perform_reads(fp, 1);

			fprintf(stderr, "[rank %d/%d] Checking file-mode blockidx %d BB vs. WB...\n", rank, size, blockidx);
			MPI_Assert(COMM, memcmp(buf_bb, buf_wb, blocksize) == 0);
			fprintf(stderr, "[rank %d/%d] Checking file-mode blockidx %d BB vs. abs-WB...\n", rank, size, blockidx);
			MPI_Assert(COMM, memcmp(buf_bb, buf_abs_wb, blocksize) == 0);

			free(buf_bb); free(buf_wb); free(buf_abs_wb);
			adios_selection_delete(block_bb);
			adios_selection_delete(block_wb);
			adios_selection_delete(block_abs_wb);
		}
	}

	adios_free_varinfo(varinfo);

	fprintf(stderr, "[rank %d/%d] Finished file-mode writeblock reads on %s:/%s\n", rank, size, bp_filename, varname);
}
Beispiel #22
0
int main (int argc, char ** argv) 
{
    int         rank, size, i, j;
    MPI_Comm    comm = MPI_COMM_WORLD;
    enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP;

    MPI_Init (&argc, &argv);
    MPI_Comm_rank (comm, &rank);
    MPI_Comm_size (comm, &size);

    adios_read_init_method (method, comm, "verbose=3");
    ADIOS_FILE * f = adios_read_open_file("adios_stat.bp", method, comm);
    if (f == NULL)
    {
        fprintf (stderr, "%s\n", adios_errmsg());
        return -1;
    }

    ADIOS_VARINFO * v = adios_inq_var (f, "temperature");
    if (v) {
        /* get statistics, for each individual time-step */ 
        adios_inq_var_stat (f, v, 1, 0);


        printf("Global MIN of temperature: %lf\n", * (double *) v->statistics->min);
        printf("Global MAX of temperature: %lf\n", * (double *) v->statistics->max);
        printf("Global AVG of temperature: %lf\n", * (double *) v->statistics->avg);
        printf("Global STD DEV of temperature: %lf\n", * (double *) v->statistics->std_dev);

        printf("\n");
        printf("---------------------------------------------------------------------------\n");
        if (v->statistics->steps) {
            printf("MIN\t\tMAX\t\tAVG\t\tSTD_DEV\t\tHISTOGRAM\n");
            for(i = 0; i < v->nsteps; i++)
            {
                if (v->statistics->steps->mins[i]) 
                    printf("%lf\t", * (double *) v->statistics->steps->mins[i]);
                else
                    printf("--\t\t");
                if (v->statistics->steps->maxs[i]) 
                    printf("%lf\t", * (double *) v->statistics->steps->maxs[i]);
                else
                    printf("--\t\t");
                if (v->statistics->steps->avgs[i]) 
                    printf("%lf\t", * (double *) v->statistics->steps->avgs[i]);
                else
                    printf("--\t\t");
                if (v->statistics->steps->std_devs[i]) 
                    printf("%lf\t", * (double *) v->statistics->steps->std_devs[i]);
                else
                    printf("--\t\t");
                if (v->statistics->histogram) {
                    for(j = 0; j <= v->statistics->histogram->num_breaks; j++) {
                        printf("%d ", v->statistics->histogram->frequencies[i][j]);
                    }
                    printf("\n");
                } else {
                    printf("--\t\t");
                }
                printf("\n");
            }	
        } else {
            printf ("Per step statistics is missing\n"); 
        }
        printf("---------------------------------------------------------------------------\n");
        printf("\n");

        if (v->statistics->histogram) {
            printf("Break points:\t\t\t");
            for(j = 0; j < v->statistics->histogram->num_breaks; j++)
                printf("%6.2lf\t", v->statistics->histogram->breaks[j]);

            printf("\n");

            printf("Frequencies:\t\t\t");
            for(j = 0; j <= v->statistics->histogram->num_breaks; j++)
                printf("%6d\t", v->statistics->histogram->gfrequencies[j]);
        }

        printf("\n\n");

#if 0 
        printf ("Auto covariance of MIN values of temperature over time: %lf\n", adios_stat_cov (v, v, "min", 0, 12, 0));
        printf ("Auto correlation of MAX values of temperature over time, with lag 2 units: %lf\n", adios_stat_cor (v, v, "max", 0, 8, 2));
        printf("\n\n");
#endif 

        adios_free_varinfo (v);

    } else {
        fprintf (stderr, "ERROR: Cannot inquire statistics of variable 'temperature': %s\n", adios_errmsg());
    }

    v = adios_inq_var (f, "complex");
    if (v) {
        adios_inq_var_stat (f, v, 1, 0);
        double *C = v->statistics->min;
        printf("Global Minimum of variable complex - Magnitude: %lf\n", C[0]);
        printf("Global Minimum of variable complex - Real part: %lf\n", C[1]);
        printf("Global Minimum of variable complex - Imaginary part: %lfi\n", C[2]);

        double ** Cmin;
        Cmin = (double **) v->statistics->steps->mins;

        printf("\nMagnitude\t\tReal\t\t\tImaginary\n");
        for (j = 0; j < v->nsteps; j++)
        {
            printf ("%lf\t\t%lf\t\t%lf\n", Cmin[j][0], Cmin[j][1], Cmin[j][2]);
        }
        printf("\n");
        adios_free_varinfo (v);

    } else {
        fprintf (stderr, "ERROR: Cannot inquire statistics of variable 'complex': %s\n", adios_errmsg());
    }

    adios_read_close (f);
    MPI_Barrier (comm);
    adios_read_finalize_method (method);
    MPI_Finalize ();
    return 0;
}
int main (int argc, char ** argv) 
{
    int         rank, size, i, j;
    MPI_Comm    comm = MPI_COMM_WORLD;
    ADIOS_FILE * f;
    ADIOS_VARINFO * v;
    ADIOS_SELECTION * sel;
    int steps = 0;
    int retval = 0;
    float timeout_sec = 1.0; 

    void * data = NULL;
    uint64_t start[2], count[2];

    MPI_Init (&argc, &argv);

    MPI_Comm_rank (comm, &rank);
    MPI_Comm_size (comm, &size);

    adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3");

    f = adios_read_open ("adios_globaltime.bp", ADIOS_READ_METHOD_BP,
                          comm, ADIOS_LOCKMODE_NONE, timeout_sec);
    if (adios_errno == err_file_not_found)
    {
        printf ("rank %d: Stream not found after waiting %f seconds: %s\n",
                rank, timeout_sec, adios_errmsg());
        retval = adios_errno;
    }
    else if (adios_errno == err_end_of_stream)
    {
        printf ("rank %d: Stream terminated before open. %s\n", rank, adios_errmsg());
        retval = adios_errno;
    }
    else if (f == NULL) {
        printf ("rank %d: Error at opening stream: %s\n", rank, adios_errmsg());
        retval = adios_errno;
    }
    else
    {
        /* process file here... */
        v = adios_inq_var (f, "temperature");
        adios_inq_var_blockinfo (f, v);

        printf ("ndim = %d\n",  v->ndim);
        //printf ("nsteps = %d\n",  v->nsteps);
        printf ("dims[%llu][%llu]\n",  v->dims[0], v->dims[1]);

        uint64_t slice_size = v->dims[0]/size;
        if (rank == size-1)
            slice_size = slice_size + v->dims[0]%size;

        start[0] = rank * slice_size;
        count[0] = slice_size;
        start[1] = 0;
        count[1] = v->dims[1];

        data = malloc (slice_size * v->dims[1] * 8);

        /* Processing loop over the steps (we are already in the first one) */
        while (adios_errno != err_end_of_stream) {
            steps++; // steps start counting from 1

            sel = adios_selection_boundingbox (v->ndim, start, count);
            adios_schedule_read (f, sel, "temperature", 0, 1, data);
            adios_perform_reads (f, 1);

            if (rank == 0)
                printf ("--------- Step: %d --------------------------------\n", 
                        f->current_step);

            printf("rank=%d: [0:%lld,0:%lld] = [", rank, v->dims[0], v->dims[1]);
            for (i = 0; i < slice_size; i++) {
                printf (" [");
                for (j = 0; j < v->dims[1]; j++) {
                    printf ("%g ", *((double *)data + i * v->dims[1] + j));
                }
                printf ("]");
            }
            printf (" ]\n\n");

            // advance to 1) next available step with 2) blocking wait
            adios_advance_step (f, 0, timeout_sec);
            if (adios_errno == err_step_notready)
            {
                printf ("rank %d: No new step arrived within the timeout. Quit. %s\n",
                        rank, adios_errmsg());
                break; // quit while loop
            }

        }

        adios_read_close (f);
    }

    if (rank==0) 
        printf ("We have processed %d steps\n", steps);

    adios_read_finalize_method (ADIOS_READ_METHOD_BP);
    free (data);
    MPI_Finalize ();

    return retval;
}
void
eavlXGCParticleImporter::Initialize()
{
    ephase.clear();
    iphase.clear();
    egid.clear();
    igid.clear();
    last_available_timestep = fp->last_step;

    if(readingRestartFile) //check for restart file format
    {
        if(fp->nvars <= 25)
            readingRestartFile = 0;
    }


    if(readingRestartFile)
    {
        int numVarsPerGroup = 25;
        nvars = fp->nvars/numVarsPerGroup;
        if(nvars <= mpiRank)
        {
            printf("Warning! :: Thread[%i] is wasting cycles :: too many processors for data\n", mpiRank);
            return;
        }
        //----Set indexes for each reader if there is more than one
        int endIndex;
        int startIndex = (nvars / numMPITasks) * mpiRank;
        if (nvars % numMPITasks > mpiRank)
        {
            startIndex += mpiRank;
            endIndex = startIndex + (nvars / numMPITasks) + 1;
        }
        else
        {
            startIndex += nvars % numMPITasks;
            endIndex = startIndex + (nvars / numMPITasks);
        }
        startIndex *= numVarsPerGroup;
        endIndex *= numVarsPerGroup;
        //--
        for(int i = startIndex; i < endIndex; i++)
        {
            ADIOS_VARINFO *avi = adios_inq_var_byid(fp, i);
            cerr << __LINE__ << endl;
            string longvarNm(&fp->var_namelist[i][0]);   //!!This changed to not remove first char
            cerr << __LINE__ << endl;
            string varNm = longvarNm.substr(longvarNm.find("/",1,1)+1,longvarNm.length());

            if(varNm == "ephase")
            {
                ephase[longvarNm] = avi;
                ephaseAvail = 1;
            }
            else if(varNm == "egid")
            {
                egid[longvarNm] = avi;
            }
            else if(varNm == "iphase")
            {
                iphase[longvarNm] = avi;
                iphaseAvail = 1;
            }
            else if(varNm == "igid")
            {
                igid[longvarNm] = avi;
            }
            else if(varNm == "inum")
            {
                int ipart;
                AdiosGetValue (fp, i, ipart);
                totalIParticles += ipart;
                //totalIParticles += (int)(*(int *)avi->value);
                adios_free_varinfo(avi);
            }
            else if(varNm == "enum")
            {
                int epart;
                AdiosGetValue (fp, i, epart);
                totalEParticles += epart;
                //totalEParticles += (int)(*(int *)avi->value);
                adios_free_varinfo(avi);
            }
            else if(i < startIndex + numVarsPerGroup)
            {
                if (varNm == "timestep")
                {
                    //timestep = (int)(*(int *)avi->value);
                    AdiosGetValue (fp, i, timestep);
                }
                else if(varNm == "time")
                {
                    AdiosGetValue (fp, i, time);
                    //time = (double)(*(double *)avi->value);
                }
                else if(varNm == "maxnum")
                {
                    AdiosGetValue (fp, i, maxnum);
                    //maxnum = (int)(*(int *)avi->value);
                }
                else if (varNm == "inphase")
                {
                    AdiosGetValue (fp, i, inphase);
                    //inphase = (int)(*(int *)avi->value);
                }
                else if(varNm == "enphase")
                {
                    AdiosGetValue (fp, i, enphase);
                    //enphase = (int)(*(int *)avi->value);
                }
                else if(varNm == "emaxgid")
                {
                    AdiosGetValue (fp, i, emaxgid);
                    //emaxgid = (long long)(*(long long *)avi->value);
                }
                else if(varNm == "imaxgid")
                {
                    AdiosGetValue (fp, i, imaxgid);
                    //imaxgid = (long long)(*(long long *)avi->value);
                }
                adios_free_varinfo(avi);
            }
            else
            {
                adios_free_varinfo(avi);
            }
        }
    }
    else //reading particle file or aggregated restart file
    {
        for(int i = 0; i < fp->nvars; i++)
        {
            string varNm(&fp->var_namelist[i][0]);
            if(varNm == "ephase")
            {
                ephaseAvail = 1;
                //----Set indexes for each reader for the ELECTRONS
                ADIOS_VARINFO *avi = adios_inq_var(fp, "ephase");
                nvars = avi->dims[0];
                int endIndex;
                int startIndex = (nvars / numMPITasks) * mpiRank;
                if (nvars % numMPITasks > mpiRank)
                {
                    startIndex += mpiRank;
                    endIndex = startIndex + (nvars / numMPITasks) + 1;
                }
                else
                {
                    startIndex += nvars % numMPITasks;
                    endIndex = startIndex + (nvars / numMPITasks);
                }
                ELECTRONendIndex = endIndex;
                ELECTRONstartIndex = startIndex;
                adios_free_varinfo(avi);
                //--
            }
            else if(varNm == "iphase")
            {
                iphaseAvail = 1;
                //----Set indexes for each reader for the IONS
                ADIOS_VARINFO *avi = adios_inq_var(fp, "iphase");
                nvars = avi->dims[0];

                int endIndex;
                int startIndex = (nvars / numMPITasks) * mpiRank;
                if (nvars % numMPITasks > mpiRank)
                {
                    startIndex += mpiRank;
                    endIndex = startIndex + (nvars / numMPITasks) + 1;
                }
                else
                {
                    startIndex += nvars % numMPITasks;
                    endIndex = startIndex + (nvars / numMPITasks);
                }
                IONendIndex = endIndex;
                IONstartIndex = startIndex;
                adios_free_varinfo(avi);
                //--
            }
        }

        for(int i = 0; i < fp->nvars; i++)
        {
            ADIOS_VARINFO *avi = adios_inq_var_byid(fp, i);
            string varNm(&fp->var_namelist[i][0]);

            if(varNm == "ephase")
            {
                ephase[varNm] = avi;
            }
            else if(varNm == "egid")
            {
                egid[varNm] = avi;
            }
            else if(varNm == "iphase")
            {
                iphase[varNm] = avi;
            }
            else if(varNm == "igid")
            {
                igid[varNm] = avi;
            }
            else if(varNm == "inum")
            {
                totalIParticles = IONendIndex - IONstartIndex;
                adios_free_varinfo(avi);
            }
            else if(varNm == "enum")
            {
                totalEParticles = ELECTRONendIndex - ELECTRONstartIndex;
                adios_free_varinfo(avi);
            }
            else if (varNm == "timestep")
            {
                //timestep = (int)(*(int *)avi->value);
                AdiosGetValue (fp, i, timestep);
                adios_free_varinfo(avi);
            }
            else if(varNm == "time")
            {
                AdiosGetValue (fp, i, time);
                //time = (double)(*(double *)avi->value);
                adios_free_varinfo(avi);
            }
            else if (varNm == "inphase")
            {
                AdiosGetValue (fp, i, inphase);
                //inphase = (int)(*(int *)avi->value);
                adios_free_varinfo(avi);
            }
            else if(varNm == "enphase")
            {
                AdiosGetValue (fp, i, enphase);
                //enphase = (int)(*(int *)avi->value);
                adios_free_varinfo(avi);
            }
            else if(varNm == "enum_total")
            {
                AdiosGetValue (fp, i, emaxgid);
                //emaxgid = (long long)(*(long long *)avi->value);
                adios_free_varinfo(avi);
            }
            else if(varNm == "inum_total")
            {
                AdiosGetValue (fp, i, imaxgid);
                //imaxgid = (long long)(*(long long *)avi->value);
                adios_free_varinfo(avi);
            }
            else
            {
                adios_free_varinfo(avi);
            }
        }
    }
}
Beispiel #25
0
int read_scalar_stepbystep ()
{
    ADIOS_FILE * f;
    float timeout_sec = 0.0; 
    int steps = 0;
    int retval = 0;
    MPI_Comm    comm = MPI_COMM_SELF;

    adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3");
    printf ("\n--------- Read scalar in stream using varinfo->value  ------------\n");
    f = adios_read_open (fname, ADIOS_READ_METHOD_BP,
                          comm, ADIOS_LOCKMODE_NONE, timeout_sec);
    if (adios_errno == err_file_not_found)
    {
        printf ("Stream not found after waiting %f seconds: %s\n",
                timeout_sec, adios_errmsg());
        retval = adios_errno;
    }
    else if (adios_errno == err_end_of_stream)
    {
        printf ("Stream terminated before open. %s\n", adios_errmsg());
        retval = adios_errno;
    }
    else if (f == NULL) {
        printf ("Error at opening stream: %s\n", adios_errmsg());
        retval = adios_errno;
    }
    else
    {
        /* Processing loop over the steps (we are already in the first one) */
        while (adios_errno != err_end_of_stream) {
            steps++; // steps start counting from 1
            printf ("Step: %d\n", f->current_step);

            /* Check the scalar O with varinfo->value */
            ADIOS_VARINFO * v = adios_inq_var (f, "NX");
            int value =  *(int*)v->value;
            printf ("Scalar NX = %d", value);
            if (value != 
                    block_count [f->current_step*nblocks_per_step*size]) 
            {
                printf ("\tERROR expected = %llu", 
                        block_count [f->current_step*nblocks_per_step*size]);
                nerrors++;
            }
            printf ("\n");

            // advance to 1) next available step with 2) blocking wait
            adios_advance_step (f, 0, timeout_sec);
            if (adios_errno == err_step_notready)
            {
                //printf ("No new step arrived within the timeout. Quit. %s\n",
                //        adios_errmsg());
                break; // quit while loop
            }
        }
        adios_read_close (f);
    }
    adios_read_finalize_method (ADIOS_READ_METHOD_BP);
    //printf ("We have processed %d steps\n", steps);
    return retval;
}
Beispiel #26
0
int print_scalar (ADIOS_FILE *f, char * name) 
{
    ADIOS_VARINFO * v;
    int i,j,k;

    v = adios_inq_var (f, name);
    adios_inq_var_blockinfo (f, v);

    printf ("Scalar '%s':\n",  name);
    printf ("nsteps = %d\n",  v->nsteps);
    printf ("nblocks per step = %d\n",  v->nblocks[0]);

    int err;

    /* Read one writeblock across all timesteps */
    int *data = (int*) calloc (v->nsteps, sizeof(int));
    ADIOS_SELECTION *s;
    printf ("Read same instance across all timesteps:\n");
    for (i=0; i < v->nblocks[0]; i++) {
        s = adios_selection_writeblock(i);
        err = adios_schedule_read_byid(f, s, v->varid, 0, v->nsteps, data);
        if (!err) 
        { 
            err = adios_perform_reads(f, 1);
            if (!err) 
            { 
                err = 0;
                printf ("  block %d = [",  i);
                for (j=0; j < v->nsteps; j++) {
                    printf ("%d", data[j]);
                    if (data[j] != 
                        block_offset [j*nblocks_per_step*size + i]) 
                    {
                        err = 1;
                    }
                    if (j < v->nsteps-1) printf(",");
                }
                printf("]");

                if (err) 
                {
                    nerrors++;
                    printf ("\tERROR expected = [");
                    for (j=0; j < v->nsteps; j++) {
                        printf ("%llu", block_offset [j*nblocks_per_step*size + i]);
                        if (j < v->nsteps-1) printf(",");
                    }
                    printf("]");
                }
                printf("\n");

            } else {
                printf ("ERROR at reading scalar '%s': %s\n", name, adios_errmsg());
            } 
        } else {
                printf ("ERROR at scheduling read for scalar '%s': %s\n", name, adios_errmsg());
        }
        adios_selection_delete(s);
    }

    /* Now read piecewise, one writeblock at a time */
    printf ("Read each instance individually:\n");
    for (j=0; j < v->nsteps; j++) {
        printf ("  step %d: \n",  j);
        for (i=0; i < v->nblocks[j]; i++) {
            s = adios_selection_writeblock(i);
            err = adios_schedule_read_byid(f, s, v->varid, j, 1, data);
            if (!err) 
            { 
                err = adios_perform_reads(f, 1);
                if (!err) 
                { 
                    printf ("    block %d = %d", i, data[0]);
                    if (data[0] != 
                        block_offset [j*nblocks_per_step*size + i]) 
                    {
                        printf ("\tERROR expected = %llu", 
                                block_offset [j*nblocks_per_step*size + i]);
                        nerrors++;
                    }
                    printf ("\n");
                } else {
                    printf ("ERROR at reading scalar '%s': %s\n", name, adios_errmsg());
                } 
            } else {
                printf ("ERROR at scheduling read for scalar '%s': %s\n", name, adios_errmsg());
            }
            adios_selection_delete(s);
        }
    }

    /* Now get them piecewise, but not with reading but through statistics */
    printf ("Get each instance individually from available statistics:\n");
    adios_inq_var_stat (f, v, 0, 1);
    if (v->statistics && v->statistics->blocks) {
        ADIOS_VARSTAT *stat = v->statistics;
        int blockid = 0;
        for (j=0; j < v->nsteps; j++) {
            printf ("  step %d: \n",  j);
            for (i=0; i < v->nblocks[j]; i++) {
                printf ("    block %d = %d", i, *(int*)stat->blocks->mins[blockid]);
                if (*(int*)stat->blocks->mins[blockid] != 
                        block_offset [j*nblocks_per_step*size + i]) 
                {
                    printf ("\tERROR expected = %llu", 
                            block_offset [j*nblocks_per_step*size + i]);
                    nerrors++;
                }
                printf ("\n");
                blockid++;
            }
        }
    }

    adios_free_varinfo (v);
    free(data);
}
Beispiel #27
0
int main (int argc, char ** argv)
{
    int         i, j, datasize;
    MPI_Comm    comm = MPI_COMM_WORLD;
    enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP;
    ADIOS_SELECTION * sel1;
    double * data = NULL;
    uint64_t start[2], count[2];

    MPI_Init (&argc, &argv);
#ifdef WITH_NCSU_TIMER
    timer_init();
#endif

    adios_read_init_method (method, comm, NULL);

    ADIOS_FILE * f = adios_read_open_file ("adios_global.bp", method, comm);
    ADIOS_VARINFO * varinfo = adios_inq_var (f, "temperature");
    if (varinfo)
    {
        int nranks;

        assert(varinfo->ndim == 2);

        nranks = varinfo->dims[0];
        assert(nranks % 4 == 0);
        assert(varinfo->dims[1] == 10);

        datasize = (nranks / 2) * varinfo->dims[1] * sizeof(double);
        data = malloc (datasize);

        start[0] = nranks / 4;
        start[1] = 2;
        count[0] = nranks / 2;
        count[1] = 6;

        sel1 = adios_selection_boundingbox (varinfo->ndim, start, count);

        adios_schedule_read (f, sel1, "temperature", 0, 1, data);
        adios_perform_reads (f, 1);

        printf("Subvolume at (%" PRIu64 ",%" PRIu64 ") of size (%" PRIu64 ",%" PRIu64 "):\n", start[0], start[1], count[0], count[1]);
        for (i = 0; i < count[0]; i++) {
            printf("[ ");
            for (j = 0; j < count[1]; j++) {
                printf("%.0lf ", data[i * count[1] + j]);
            }
            printf("]\n");
        }

        adios_selection_delete (sel1);
    }

    adios_free_varinfo (varinfo);
    adios_read_close (f);

    adios_read_finalize_method (ADIOS_READ_METHOD_BP);


#ifdef WITH_NCSU_TIMER
    printf("[TIMERS] ");
    timer_result_t *results = timer_get_results_sorted();
    for (i = 0; i < timer_get_num_timers(); i++) {
        printf("%s: %0.4lf ", results[i].name, results[i].time);
    }
    printf("\n");
    free(results);
#endif

#ifdef WITH_NCSU_TIMER
    timer_finalize();
#endif
    MPI_Finalize ();
    return 0;
}
Beispiel #28
0
int main (int argc, char ** argv) 
{
    char        filename [256] = "stream.bp";
    int         rank, size;
    int         NX, NY; 
    int         len, off;
    double      *t = NULL;
    MPI_Comm    comm = MPI_COMM_WORLD;

    int64_t     adios_handle;
	uint64_t    adios_groupsize, adios_totalsize;

    uint64_t    start[2], count[2];

    ADIOS_SELECTION *sel;
    int         steps = 0;

    MPI_Init (&argc, &argv);
    MPI_Comm_rank (comm, &rank);
    MPI_Comm_size (comm, &size);
    
    // ADIOS read init
    adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3");
    
    ADIOS_FILE* fp = adios_read_open_file ("kstar.bp", 
                                           ADIOS_READ_METHOD_BP,
                                           comm);
    assert(fp != NULL);

    ADIOS_VARINFO* nx_info = adios_inq_var( fp, "N");
    ADIOS_VARINFO* ny_info = adios_inq_var( fp, "L");

    NX = *((int *)nx_info->value);
    NY= *((int*)ny_info->value);

    len = NX / size;
    off = len * rank;

    if (rank == size-1)
        len = len + NX % size;

    printf("\trank=%d: NX,NY,len,off = %d\t%d\t%d\t%d\n", rank, NX, NY, len, off);
    assert(len > 0);

    t = (double *) malloc(sizeof(double) * len * NY);
    memset(t, '\0', sizeof(double) * len * NY);
    assert(t != NULL);

    start[0] = off;
    start[1] = 0;
    count[0] = len;
    count[1] = NY;
    sel = adios_selection_boundingbox (2, start, count);

    // ADIOS write init
    adios_init ("adios.xml", comm);
    
    remove (filename);
    //int ii;
    //for(ii = 0; ii<10; ii++){
    //    for (i = 0; i < len * NY; i++)
    //        t[i] = ii*1000 + rank;

    while(adios_errno != err_end_of_stream && adios_errno != err_step_notready)
    {
        steps++;
        // Reading
        adios_schedule_read (fp, sel, "var", 0, 1, t);
        adios_perform_reads (fp, 1);

        // Debugging
        //for (i = 0; i < len*NY; i++)  t[i] = off * NY + i;

        printf("step=%d\trank=%d\t[%d,%d]\n", steps, rank, len, NY);

        // Writing
        adios_open (&adios_handle, "writer", filename, "a", comm);
        adios_groupsize = 4*4 + 8*len*NY;
        adios_group_size (adios_handle, adios_groupsize, &adios_totalsize);
        adios_write (adios_handle, "NX", &NX);
        adios_write (adios_handle, "NY", &NY);
        adios_write (adios_handle, "len", &len);
        adios_write (adios_handle, "off", &off);
        adios_write (adios_handle, "var_2d_array", t);
        adios_close (adios_handle);


        // Advance
        MPI_Barrier (comm);
        adios_advance_step(fp, 0, TIMEOUT_SEC);
    }
    free(t);

    MPI_Barrier (comm);
    adios_read_close(fp);

    if (rank==0) 
        printf ("We have processed %d steps\n", steps);

    MPI_Barrier (comm);
    adios_read_finalize_method(ADIOS_READ_METHOD_BP);

    adios_finalize (rank);

    MPI_Finalize ();

    return 0;
}
Beispiel #29
0
int main (int argc, char ** argv) 
{
    int         rank, size;
    int         NX, NY; 
    int         len, off;
    double      *t = NULL;
    MPI_Comm    comm = MPI_COMM_WORLD;

    uint64_t    start[2], count[2];

    ADIOS_SELECTION *sel;
    int         steps = 0;

#ifdef _USE_GNUPLOT
    int         i, j;
    double      *tmp;
    FILE        *pipe;
#else
    // Variables for ADIOS write
    int64_t     adios_handle;
    uint64_t    adios_groupsize, adios_totalsize;
    char        outfn[256];
#endif

    MPI_Init (&argc, &argv);
    MPI_Comm_rank (comm, &rank);
    MPI_Comm_size (comm, &size);

    adios_read_init_method(ADIOS_READ_METHOD_FLEXPATH, comm, "");

    ADIOS_FILE* fp = adios_read_open("stream.bp", 
                                     ADIOS_READ_METHOD_FLEXPATH, 
                                     comm, ADIOS_LOCKMODE_NONE, 0.0);
    assert(fp != NULL);
    
    ADIOS_VARINFO* nx_info = adios_inq_var( fp, "NX");
    ADIOS_VARINFO* ny_info = adios_inq_var( fp, "NY");

    NX = *((int *)nx_info->value);
    NY= *((int*)ny_info->value);
    
    len = NX / size;
    off = len * rank;

    if (rank == size-1)
        len = len + NX % size;
    
    printf("\trank=%d: NX,NY,len,off = %d\t%d\t%d\t%d\n", rank, NX, NY, len, off);
    assert(len > 0);

    t = (double *) malloc(sizeof(double) * len * NY);
    memset(t, '\0', sizeof(double) * len * NY);
    assert(t != NULL);

    start[0] = off;
    start[1] = 0;
    count[0] = len;
    count[1] = NY;
    // Not working ... 
    //sel = adios_selection_boundingbox (2, start, count);

    sel = malloc(sizeof(ADIOS_SELECTION));
    sel->type=ADIOS_SELECTION_WRITEBLOCK;
    sel->u.block.index = rank;

#ifdef _USE_GNUPLOT
    if ((NX % size) > 0)
    {
        fprintf(stderr, "Equal distribution is required\n");
        return -1;
    }

    if (rank == 0) {
        pipe = popen("gnuplot", "w");
        fprintf(pipe, "set view map\n");
        fprintf(pipe, "set xrange [0:%d]\n", NX-1);

        tmp = (double *) malloc(sizeof(double) * NX * NY);
        assert(tmp != NULL);
    }

#else
    // ADIOS write init
    adios_init ("adios.xml", comm);
#endif

    //while(adios_errno != err_end_of_stream && adios_errno != err_step_notready)
    while(1)
    {
        steps++;
        // Reading
        adios_schedule_read (fp, sel, "var_2d_array", 0, 1, t);
        adios_perform_reads (fp, 1);
        
        printf("step=%d\trank=%d\tfp->current_step=%d\t[%d,%d]\n", 
                steps, rank, fp->current_step, len, NY);
        /*
        // Debugging
        for (i=0; i<len; i++) {
            printf("%d: rank=%d: t[%d,0:4] = ", steps, rank, off+i);
            for (j=0; j<5; j++) {
                printf(", %g", t[i*NY + j]);
            }
            printf(" ...\n");
        }
        */

        // Do something 
#ifdef _USE_GNUPLOT         // Option 1: plotting

        MPI_Gather(t, len * NY, MPI_DOUBLE, tmp, len * NY, MPI_DOUBLE, 0, comm);
        
        if (rank == 0)
        {
            fprintf(pipe, "set title 'Soft X-Rray Signal (shot #%d)'\n", steps);
            fprintf(pipe, "set xlabel 'Channel#'\n");
            fprintf(pipe, "set ylabel 'Timesteps'\n");
            fprintf(pipe, "set cblabel 'Voltage (eV)'\n");

#  ifndef _GNUPLOT_INTERACTIVE
            fprintf(pipe, "set terminal png\n");
            fprintf(pipe, "set output 'fig%03d.png'\n", steps);
#  endif

            fprintf(pipe, "splot '-' matrix with image\n");
            //fprintf(pipe, "plot '-' with lines, '-' with lines, '-' with lines\n");

            double *sum = calloc(NX, sizeof(double));

            for (j = 0; j < NY; j++) {
                for (i = 0; i < NX; i++) {
                    sum[i] += tmp[i * NY + j];
                }
            }

            for (j = 0; j < NY; j++) {
                for (i = 0; i < NX; i++) {
                    fprintf (pipe, "%g ", (-tmp[i * NY + j] + sum[i]/NY)/3276.8);
                }
                fprintf(pipe, "\n");
            }
            fprintf(pipe, "e\n");
            fprintf(pipe, "e\n");
            fflush (pipe);

#  ifdef _GNUPLOT_INTERACTIVE
            printf ("Press [Enter] to continue . . .");
            fflush (stdout);
            getchar ();
#  endif

            free(sum);
        }


#else        // Option 2: BP writing

        snprintf (outfn, sizeof(outfn), "reader_%3.3d.bp", steps);
        adios_open (&adios_handle, "reader", outfn, "w", comm);
        adios_groupsize = 4 * sizeof(int) + sizeof(double) * len * NY; 
        adios_group_size (adios_handle, adios_groupsize, &adios_totalsize);
        adios_write (adios_handle, "NX", &NX);
        adios_write (adios_handle, "NY", &NY);
        adios_write (adios_handle, "len", &len);
        adios_write (adios_handle, "off", &off);
        adios_write (adios_handle, "var", t);
        adios_close (adios_handle);

#endif        

        // Advance
        MPI_Barrier (comm);
        adios_advance_step(fp, 0, TIMEOUT_SEC);
        if (adios_errno == err_end_of_stream)
        {
            printf("rank %d, Stream terminated. Quit\n", rank);
            break; // quit while loop
        }
        else if (adios_errno == err_step_notready)
        {
            printf ("rank %d: No new step arrived within the timeout. Quit.\n", rank);
            break; // quit while loop
        }
        else if (adios_errno != err_no_error) {
            printf("ADIOS returned code=%d, msg:%s\n", 
                    adios_errno, adios_get_last_errmsg()); 
            break; // quit while loop
        }
    }
    //
    free(t);

    adios_read_close(fp);
    //printf("rank %d, Successfully closed stream\n", rank);

    adios_read_finalize_method(ADIOS_READ_METHOD_FLEXPATH);
    //printf("rank %d, Successfully finalized read method\n", rank);

#ifndef _USE_GNUPLOT
    adios_finalize (rank);
    //printf("rank %d, Successfully finalized adios\n", rank);
#else
    if (rank==0) {
        free(tmp);
        pclose(pipe);
    }
#endif

    MPI_Finalize ();

    return 0;
}
int main(int argc, char ** argv)
{
    int rank, size, varid, numvars;
    int bins, step, mod;
    char *filename, *in_stream, *data_var_name;
    MPI_Comm comm = MPI_COMM_WORLD;
    enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_FLEXPATH;
    //enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP;
    ADIOS_SELECTION * global_range_select;
    double *data;
    uint64_t tstep, global_size, mysize, mystart, sz;

    MPI_Init (&argc, &argv);
    MPI_Comm_rank(comm, &rank);
    MPI_Comm_size(comm, &size);

    /* Command line parsing */
    if (rank == 0 && argc < 4) {
      fprintf(stderr, "\nHistogram usage: <exec> input-stream-name num-bins" 
         " arr1 [arr2] [arr3] [...]\n"
         "\t where arr1, arr2, arr3 ... are the names of the arrays to be analyzed.\n");
      MPI_Abort(comm, -1);
    }
    MPI_Barrier(comm); 
    in_stream = argv[1];

    //Parse cmd line
    bins = atoi(argv[2]);
    numvars = argc - 3;
    const char *vars[numvars];
    for (varid=0; varid < numvars; varid++)
    {
      vars[varid] = argv[varid + 3];
    }

    /* Adios open and init */
    adios_read_init_method (method, comm, "verbose=1");
    ADIOS_FILE * f = adios_read_open (in_stream, method, comm, ADIOS_LOCKMODE_ALL, -1);
    step = 0; //not used now
    while (adios_errno != err_end_of_stream){
      //resource monitor

      /*loop over different arrays inside stream*/
      for (varid = 0; varid < numvars; varid++){         
#ifdef ENABLE_MONITOR
        //double t1 = wfgettimeofday();
        lib_mem_init();
        ind_timer_start(0, "whole timestep");
#endif
        //Init variables....
        global_size = 0; tstep = 0;
        mod = 0; mysize = 0; mystart = 0;
        adios_schedule_read (f, NULL, "ntimestep", 0, 1, &tstep);
        adios_perform_reads (f, 1);
        ADIOS_VARINFO * glob_info = adios_inq_var (f, vars[varid]);
        global_size = glob_info->dims[0];
        //printf("[DEBUG] global_size = %" PRIu64 " ntimestep = %" PRIu64 "\n", 
        //  global_size, tstep);

        //printf("[HIST%d] received data for timestep %" PRIu64 " with ndim: %d and globalsize:%"
        //    PRIu64 " \n", rank, tstep, ndim, global_size); 
        //sleep(800);
        //debug
        //Array slice computation
        mod = global_size % size;//size = MPI size
        if (mod == 0){
          mysize = global_size / size;
          mystart = mysize * rank;
        }
        else {
          mysize = global_size / (size);
          if (rank < mod){
            mysize++;
            mystart = mysize * rank;
          }
          else {
            mystart = (mod * (mysize + 1)) +
                      ((rank - mod) * mysize);
          }
        }

#ifdef ENABLE_MONITOR
        nohandler_mem(rank);
#endif

        //printf("[HISTO%d]: mysize = %" PRIu64" mystart = %" PRIu64 "\n", rank, mysize, mystart);
        //debug
        //if (step == 0) sleep(800);
        uint64_t starts[] = {mystart};
        uint64_t counts[] = {mysize};
        global_range_select = adios_selection_boundingbox (1, starts, counts);
        
        //Allocate space for arrays
        uint64_t msize = ((uint64_t) sizeof(double) * mysize);
        //printf("[DEBUG] mysize = %" PRIu64 " msize= %" PRIu64" \n", mysize, msize);
        //data = (double *) malloc(sizeof(double) * mysize);
        data = new double[mysize];
        if (data == NULL){
          //printf("DEBUG: malloc returned NULL, size was %d\n", msize);
        }
        else {
          if (rank == 0)
            printf("[HIST0] DEBUG: malloc successful, size was %d\n", mysize);
        }
        //memset (data, 0, sizeof(double) * mysize);

        //Read data
        adios_schedule_read (f,
                             global_range_select,
                             vars[varid],
                             0, 1, data);
        adios_perform_reads 

        (f, 1);
        
#ifdef ENABLE_MONITOR
        nohandler_mem(rank);
#endif

        //printf("PERFORM_READS success of variable: %s\n", vars[varid]); 
        /*
           Data check


        if (step == 4) {
          FILE *fp;
          char *log;
          asprintf(&log, "histo-input%d-%d.log", step, rank);
          fp = fopen(log, "w");
          fprintf(fp, "timestep: %" PRIu64 " mysize: %"PRIu64 "\n", 
              tstep, mysize);
          for (i=0; i<(int)mysize; i++){
            fprintf(fp, "%lf\n", data[i]);
          }
          fclose(fp);
          sleep(800);
        }
        */

         
        // find max and min
        sz = 0;
        sz = mysize; 
        double min = data[0];
        double max = data[0];
        for (uint64_t i = 1; i < sz; ++i)
        {
            if (data[i] > max) max = data[i];
            if (data[i] < min) min = data[i];
        }//local max, min found.
        //local data should just use shared mem.


        double g_min, g_max;

        // Find the global max/min
        MPI_Allreduce (&min, &g_min, 1, MPI_DOUBLE, MPI_MIN, comm);
        MPI_Allreduce (&max, &g_max, 1, MPI_DOUBLE, MPI_MAX, comm);

        //printf("[HIST%d] glob-min: %f, glob-max: %f\n", rank, g_min, g_max);
        
        nohandler_mem(rank);

        double width = (g_max - g_min)/bins;
        std::vector<uint64_t>   hist(bins);
        for (uint64_t i = 0; i < sz; ++i)//fill local bins
        {
            //printf("[HISTO%d] local filling adding index %" PRIu64 "\n", rank, i);
            int idx = int((data[i] - g_min)/width);//discover index
            if (idx == bins)        // we hit the max
                --idx;
            //printf("[%d]: %f -> %d\n", rank, data[i], idx);
            ++hist[idx];
        }

        delete[] data;
        // Global reduce histograms
        std::vector<uint64_t> g_hist(bins);
        MPI_Reduce(&hist[0], &g_hist[0], bins, MPI_UINT64_T, MPI_SUM, 0, comm);
        
        //debug
        //printf("[Completed histogram routine]\n");

        if (rank == 0) //print histogram to file
        {
          FILE *fp;
          const char *log = "histograms.log";
          fp = fopen(log, "a");
          fprintf(fp, "Histogram for %s, timestep %" PRIu64"\n", vars[varid], tstep);
          for (int i = 0; i < bins; ++i)
            fprintf(fp, "  %f-%f: %" PRIu64 "\n", g_min + i*width, g_min + (i+1)*width, g_hist[i]);
          fclose (fp);
        }

#ifdef ENABLE_MONITOR
        nohandler_mem(rank);
#endif

        if (rank == 0) //print histogram to terminal
        {
          printf("Histogram for %s, timestep %" PRIu64"\n", vars[varid], tstep);
          for (int i = 0; i < bins; ++i)
            printf("  %f-%f: %" PRIu64 "\n", 
              g_min + i*width, g_min + (i+1)*width, g_hist[i]);
        }
        
        //resource monitor
#ifdef ENABLE_MONITOR
        //double t2 = wfgettimeofday();
        ind_timer_end(0);
        char monitor_title[40];
        sprintf(monitor_title, "histogram-%s", vars[varid]);
        monitor_out (rank, size, tstep, msize, t1, t2, comm, monitor_title);
#endif
      }
      //end of read + analysis for 3 variables


      
      adios_release_step(f);
      //delete[] data;
      if (rank == 0) printf("[HIST%d] read and wrote data for timestep %" PRIu64 "\n", rank, tstep);

      step++;
      adios_advance_step(f, 0, -1);
      /*
      if (step == 6){ 
        double t1 = wfgettimeofday();
        FILE *tfp;
        tfp = fopen("time.log", "a");
        fprintf(tfp, "rank %d histogram end time: %f\n", rank, t1);
        fclose(tfp);
      }
      */ 
    }//end of adios stream while loop
    if (rank == 0) printf("[HIST%d] out of read loop\n", rank);
    /* performance measurement */
    /*
    if (rank == 0){ 
      double t3 = wfgettimeofday();
      FILE *tfp;
      tfp = fopen("time.log", "a");
      fprintf(tfp, "master histogram end time: %f\n", t3);
      fclose(tfp);
    }
    */
#ifdef ENABLE_MONITOR
    outer_timer_end(rank, "histogram");
#endif
    adios_read_close(f);
    adios_read_finalize_method(method);
    MPI_Finalize();
    return 0;
}