void InfiniteDimensionalMCMCSampler::_append_scalar_dataset(hid_t dset, double data)
{
// Only subprocess with rank 0 manipulates the output file
if ((this->m_env).subRank() == 0) {
int err;
// Create a memory dataspace for data to append
const int ndims = 1;
hsize_t dims[ndims] = { 1 }; // Only writing one double
hid_t mem_space = H5Screate_simple(ndims, dims, NULL);
// Extend the dataset
// Set dims to be the *new* dimension of the extended dataset
dims[0] = { this->_iteration / this->m_ov->m_save_freq };
err = H5Dset_extent(dset, dims);
// Select hyperslab on file dataset
hid_t file_space = H5Dget_space(dset);
hsize_t start[1] = {(this->_iteration / this->m_ov->m_save_freq) - 1};
hsize_t count[1] = {1};
err = H5Sselect_hyperslab(file_space, H5S_SELECT_SET, start, NULL, count, NULL);
// hsize_t size[1];
// size[0] = this->_iteration / this->m_ov->m_save_freq;
// Write the data
H5Dwrite(dset, H5T_NATIVE_DOUBLE, mem_space, file_space, H5P_DEFAULT, &data);
// Close a bunch of stuff
H5Sclose(file_space);
H5Sclose(mem_space);
}
}
/**
Append a vector to a specified dataset and return the error status
of the write operation. */
herr_t HDF5DataWriter::appendToDataset(hid_t dataset_id, const vector< double >& data)
{
herr_t status;
if (dataset_id < 0){
return -1;
}
hid_t filespace = H5Dget_space(dataset_id);
if (filespace < 0){
return -1;
}
if (data.size() == 0){
return 0;
}
hsize_t size = H5Sget_simple_extent_npoints(filespace) + data.size();
status = H5Dset_extent(dataset_id, &size);
if (status < 0){
return status;
}
filespace = H5Dget_space(dataset_id);
hsize_t size_increment = data.size();
hid_t memspace = H5Screate_simple(1, &size_increment, NULL);
hsize_t start = size - data.size();
H5Sselect_hyperslab(filespace, H5S_SELECT_SET, &start, NULL, &size_increment, NULL);
status = H5Dwrite(dataset_id, H5T_NATIVE_DOUBLE, memspace, filespace, H5P_DEFAULT, &data[0]);
return status;
}
int_f
h5dset_extent_c ( hid_t_f *dset_id , hsize_t_f *dims)
/******/
{
hid_t c_space_id;
hsize_t c_dims[H5S_MAX_RANK];
int rank;
int i;
int status;
int ret_value = -1;
if((c_space_id = H5Dget_space((hid_t)*dset_id)) < 0) return ret_value;
rank = H5Sget_simple_extent_ndims(c_space_id);
H5Sclose(c_space_id);
if(rank < 0 ) return ret_value;
/*
* Reverse dimensions due to C-FORTRAN storage order.
*/
for(i = 0; i < rank; i++)
c_dims[i] = (hsize_t)dims[rank - i - 1];
status = H5Dset_extent((hid_t)*dset_id, c_dims);
if(status >= 0)
ret_value = 0;
return ret_value;
}
void HDF5Output::flush() const {
hsize_t n = buffer.size();
if (n == 0)
return;
hid_t file_space = H5Dget_space(dset);
hsize_t count = H5Sget_simple_extent_npoints(file_space);
// resize dataset
hsize_t new_size[RANK] = {count + n};
H5Dset_extent(dset, new_size);
// get updated filespace
H5Sclose(file_space);
file_space = H5Dget_space(dset);
hsize_t offset[RANK] = {count};
hsize_t cnt[RANK] = {n};
H5Sselect_hyperslab(file_space, H5S_SELECT_SET, offset, NULL, cnt, NULL);
hid_t mspace_id = H5Screate_simple(RANK, cnt, NULL);
H5Dwrite(dset, sid, mspace_id, file_space, H5P_DEFAULT, buffer.data());
H5Sclose(mspace_id);
H5Sclose(file_space);
buffer.clear();
}
/**
* Appends along the last dimensions.
*/
static hid_t make_dataset(ndio_hdf5_t self,nd_type_id_t type_id,unsigned ndim,size_t *shape, hid_t* filespace)
{ hsize_t *sh=0,*ori=0,*ext=0;
TRY(self->isw);
STACK_ALLOC(hsize_t,sh ,ndim);
STACK_ALLOC(hsize_t,ori,ndim);
STACK_ALLOC(hsize_t,ext,ndim);
if(self->dataset>=0) // data set already exists...needs extending, append on slowest dim
{ HTRY(H5Sget_simple_extent_dims(space(self),sh,NULL));
ZERO(hsize_t,ori,ndim);
ori[0]=sh[0];
sh[0]+=shape[ndim-1];
reverse_hsz_sz(ndim,ext,shape);
HTRY(H5Dextend(self->dataset,sh));
HTRY(*filespace=H5Dget_space(self->dataset));
HTRY(H5Sselect_hyperslab(*filespace,H5S_SELECT_SET,ori,NULL,ext,NULL));
} else
{ HTRY(self->dataset=H5Dcreate(
self->file,name(self),
nd_to_hdf5_type(type_id),
make_space(self,ndim,shape),
H5P_DEFAULT,/*(rare) link creation props*/
dataset_creation_properties(
/*set_deflate*/(
set_chunk(self,ndim,shape))),
H5P_DEFAULT /*(rare) dataset access props*/
));
reverse_hsz_sz(ndim,sh,shape);
*filespace=H5S_ALL;
}
HTRY(H5Dset_extent(self->dataset,sh));
return self->dataset;
Error:
return -1;
}
void DCDataSet::append(size_t count, size_t offset, size_t stride, const void* data)
throw (DCException)
{
log_msg(2, "DCDataSet::append");
if (!opened)
throw DCException(getExceptionString("append: Dataset has not been opened/created."));
log_msg(3, "logical_size = %s", getLogicalSize().toString().c_str());
Dimensions target_offset(getLogicalSize());
// extend size (dataspace) of existing dataset with count elements
getLogicalSize()[0] += count;
hsize_t * max_dims = new hsize_t[ndims];
for (size_t i = 0; i < ndims; ++i)
max_dims[i] = H5F_UNLIMITED;
if (H5Sset_extent_simple(dataspace, 1, getLogicalSize().getPointer(), max_dims) < 0)
throw DCException(getExceptionString("append: Failed to set new extent"));
delete[] max_dims;
max_dims = NULL;
log_msg(3, "logical_size = %s", getLogicalSize().toString().c_str());
if (H5Dset_extent(dataset, getLogicalSize().getPointer()) < 0)
throw DCException(getExceptionString("append: Failed to extend dataset"));
// select the region in the target DataSpace to write to
Dimensions dim_data(count, 1, 1);
if (H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, target_offset.getPointer(),
NULL, dim_data.getPointer(), NULL) < 0 ||
H5Sselect_valid(dataspace) < 0)
throw DCException(getExceptionString("append: Invalid target hyperslap selection"));
// append data to the dataset.
// select the region in the source DataSpace to read from
Dimensions dim_src(offset + count * stride, 1, 1);
hid_t dsp_src = H5Screate_simple(1, dim_src.getPointer(), NULL);
if (dsp_src < 0)
throw DCException(getExceptionString("append: Failed to create src dataspace while appending"));
if (H5Sselect_hyperslab(dsp_src, H5S_SELECT_SET, Dimensions(offset, 0, 0).getPointer(),
Dimensions(stride, 1, 1).getPointer(), dim_data.getPointer(), NULL) < 0 ||
H5Sselect_valid(dsp_src) < 0)
throw DCException(getExceptionString("append: Invalid source hyperslap selection"));
if (!data || (count == 0))
{
H5Sselect_none(dataspace);
data = NULL;
}
if (H5Dwrite(dataset, this->datatype, dsp_src, dataspace, dsetWriteProperties, data) < 0)
throw DCException(getExceptionString("append: Failed to append dataset"));
H5Sclose(dsp_src);
}
void NSDFWriter::flush()
{
// We need to update the tend on each write since we do not know
// when the simulation is getting over and when it is just paused.
writeScalarAttr<string>(filehandle_, "tend", iso_time(NULL));
// append all uniform data
for (map< string, hid_t>::iterator it = classFieldToUniform_.begin();
it != classFieldToUniform_.end(); ++it){
map< string, vector < unsigned int > >::iterator idxit = classFieldToSrcIndex_.find(it->first);
if (idxit == classFieldToSrcIndex_.end()){
cerr << "Error: NSDFWriter::flush - could not find entry for " << it->first <<endl;
break;
}
if (data_.size() == 0 || data_[0].size() == 0){
break;
}
double * buffer = (double*)calloc(idxit->second.size() * steps_, sizeof(double));
vector< double > values;
for (unsigned int ii = 0; ii < idxit->second.size(); ++ii){
for (unsigned int jj = 0; jj < steps_; ++jj){
buffer[ii * steps_ + jj] = data_[idxit->second[ii]][jj];
}
data_[idxit->second[ii]].clear();
}
hid_t filespace = H5Dget_space(it->second);
if (filespace < 0){
break;
}
hsize_t dims[2];
hsize_t maxdims[2];
// retrieve current datset dimensions
herr_t status = H5Sget_simple_extent_dims(filespace, dims, maxdims);
hsize_t newdims[] = {dims[0], dims[1] + steps_}; // new column count
status = H5Dset_extent(it->second, newdims); // extend dataset to new column count
H5Sclose(filespace);
filespace = H5Dget_space(it->second); // get the updated filespace
hsize_t start[2] = {0, dims[1]};
dims[1] = steps_; // change dims for memspace & hyperslab
hid_t memspace = H5Screate_simple(2, dims, NULL);
H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, NULL, dims, NULL);
status = H5Dwrite(it->second, H5T_NATIVE_DOUBLE, memspace, filespace, H5P_DEFAULT, buffer);
H5Sclose(memspace);
H5Sclose(filespace);
free(buffer);
}
// append all event data
for (unsigned int ii = 0; ii < eventSrc_.size(); ++ii){
appendToDataset(getEventDataset(eventSrc_[ii], eventSrcFields_[ii]),
events_[ii]);
events_[ii].clear();
}
// flush HDF5 nodes.
HDF5DataWriter::flush();
}
herr_t H5VLARRAYappend_records( hid_t dataset_id,
hid_t type_id,
int nobjects,
hsize_t nrecords,
const void *data )
{
hid_t space_id;
hid_t mem_space_id;
hsize_t start[1];
hsize_t dataset_dims[1];
hsize_t dims_new[1] = {1}; /* Only a record on each append */
hvl_t wdata; /* Information to write */
/* Initialize VL data to write */
wdata.p=(void *)data;
wdata.len=nobjects;
/* Dimension for the new dataset */
dataset_dims[0] = nrecords + 1;
/* Extend the dataset */
if ( H5Dset_extent( dataset_id, dataset_dims ) < 0 )
goto out;
/* Create a simple memory data space */
if ( (mem_space_id = H5Screate_simple( 1, dims_new, NULL )) < 0 )
return -1;
/* Get the file data space */
if ( (space_id = H5Dget_space( dataset_id )) < 0 )
return -1;
/* Define a hyperslab in the dataset */
start[0] = nrecords;
if ( H5Sselect_hyperslab( space_id, H5S_SELECT_SET, start, NULL, dims_new, NULL) < 0 )
goto out;
if ( H5Dwrite( dataset_id, type_id, mem_space_id, space_id, H5P_DEFAULT, &wdata ) < 0 )
goto out;
/* Terminate access to the dataspace */
if ( H5Sclose( space_id ) < 0 )
goto out;
if ( H5Sclose( mem_space_id ) < 0 )
goto out;
return 1;
out:
return -1;
}
/** writeFile.
* Write the data using the HDF5 library calls.
* \param[in] pArray - The NDArray containing the data to write.
* \param[in] datatype - The HDF5 datatype of the data.
* \param[in] dataspace - A handle to the HDF5 dataspace for this dataset.
* \param[in] framesize - The size of the data to write.
*/
asynStatus NDFileHDF5Dataset::writeFile(NDArray *pArray, hid_t datatype, hid_t dataspace, hsize_t *framesize)
{
herr_t hdfstatus;
static const char *functionName = "writeFile";
// Increase the size of the dataset
asynPrint(this->pAsynUser_, ASYN_TRACE_FLOW,
"%s::%s: set_extent dims={%d,%d,%d}\n",
fileName, functionName, (int)this->dims_[0], (int)this->dims_[1], (int)this->dims_[2]);
hdfstatus = H5Dset_extent(this->dataset_, this->dims_);
if (hdfstatus){
asynPrint(this->pAsynUser_, ASYN_TRACE_ERROR,
"%s::%s ERROR Increasing the size of the dataset [%s] failed\n",
fileName, functionName, this->name_.c_str());
return asynError;
}
// Select a hyperslab.
hid_t fspace = H5Dget_space(this->dataset_);
if (fspace < 0){
asynPrint(this->pAsynUser_, ASYN_TRACE_ERROR,
"%s::%s ERROR Unable to get a copy of the dataspace for dataset [%s]\n",
fileName, functionName, this->name_.c_str());
return asynError;
}
hdfstatus = H5Sselect_hyperslab(fspace, H5S_SELECT_SET, this->offset_, NULL, framesize, NULL);
if (hdfstatus){
asynPrint(this->pAsynUser_, ASYN_TRACE_ERROR,
"%s::%s ERROR Unable to select hyperslab\n",
fileName, functionName);
return asynError;
}
// Write the data to the hyperslab.
hdfstatus = H5Dwrite(this->dataset_, datatype, dataspace, fspace, H5P_DEFAULT, pArray->pData);
if (hdfstatus){
asynPrint(this->pAsynUser_, ASYN_TRACE_ERROR,
"%s::%s ERROR Unable to write data to hyperslab\n",
fileName, functionName);
return asynError;
}
hdfstatus = H5Sclose(fspace);
if (hdfstatus){
asynPrint(this->pAsynUser_, ASYN_TRACE_ERROR,
"%s::%s ERROR Unable to close the dataspace\n",
fileName, functionName);
return asynError;
}
this->nextRecord_++;
return asynSuccess;
}
void OHDF5mpipp::updateSpikeDataSets(const double& t)
{
#pragma omp single
{
//std::cout << "\tspike_dataset.nodeoffset=" << spike_dataset.nodeoffset << std::endl;
//std::cout << "\tspike_dataset.window_entries=" << spike_dataset.window_entries << std::endl;
//std::cout << "\tspike_dataset.window_size=" << spike_dataset.window_size << std::endl;
//std::cout << "\tspike_dataset.next_window_size=" << spike_dataset.next_window_size << std::endl;
int new_window_size = 0;
int space_needed = predictNewSpikeWindowSize(t, spike_dataset);
if (space_needed>0 && !spike_dataset.extended2next) {
new_window_size = space_needed;
//std::cout << "new_window_size" << std::endl;
spike_dataset.next_window_size = new_window_size;
//spike_dataset.all_window_size += new_window_size;
}
int new_window_sizes[clientscount];
MPI_Allgather(&new_window_size, 1, MPI_INT, new_window_sizes, 1, MPI_INT, MPI_COMM_WORLD);
if (new_window_size>0) {
spike_dataset.next_nodeoffset=spike_dataset.all_window_size; // old all_window_size
for (int i=0; i<clientscount; i++) {
if (i<own_id)
spike_dataset.next_nodeoffset += new_window_sizes[i];
}
}
bool extendNecessary=false;
for (int i=0; i<clientscount; i++) {
extendNecessary = extendNecessary || new_window_sizes[i] > 0;
spike_dataset.all_window_size += new_window_sizes[i];
}
//std::cout << "\tspike_dataset.next_nodeoffset=" << spike_dataset.next_nodeoffset << std::endl;
//std::cout << "\tspike_dataset.all_window_size=" << spike_dataset.all_window_size << std::endl;
if (extendNecessary) {
hsize_t size[2]={spike_dataset.all_window_size,1};
H5Dset_extent(spike_dataset.dset_id, size);
status = H5Sclose (spike_dataset.filespace);
spike_dataset.filespace = H5Dget_space (spike_dataset.dset_id);
spike_dataset.extended2next=true;
#ifdef _DEBUG_MODE
std::cout << "H5Dset_extent:" << spike_dataset.all_window_size << std::endl;
#endif
}
}
}
void extendAndWrite(hid_t &file_id, char *datasetname, int rank, hsize_t *addDims, hid_t &type, const void *data, bool first) {
hid_t dset = H5Dopen(file_id, datasetname, H5P_DEFAULT);
// get the current size
hsize_t *currSize = new hsize_t[rank];
herr_t status;
hid_t space ;
if (first) {
for (int i = 1; i < rank; ++i) {
currSize[i] = addDims[i];
}
currSize[0] = 0;
} else {
space = H5Dget_space(dset);
H5Sget_simple_extent_dims(space, currSize, NULL);
status = H5Sclose(space);
}
hsize_t *size = new hsize_t[rank];
for (int i = 1; i < rank; ++i) {
size[i] = currSize[i];
}
size[0] = currSize[0] + addDims[0];
// extend it
status = H5Dset_extent(dset, size);
// get the offset
hsize_t *offset = new hsize_t[rank];
for (int i = 1; i < rank; ++i) {
offset[i] = 0;
}
offset[0] = currSize[0];
// get the new dataspace.
space = H5Dget_space(dset);
// get the hyperslab
status = H5Sselect_hyperslab(space, H5S_SELECT_SET, offset, NULL, addDims, NULL);
// get dataspace
hid_t dataspace = H5Screate_simple(rank, addDims, NULL);
// write the data
status = H5Dwrite(dset, type, dataspace, space, H5P_DEFAULT, data);
status = H5Sclose(dataspace);
status = H5Sclose(space);
status = H5Dclose(dset);
delete [] currSize;
delete [] size;
delete [] offset;
}
/* Truncate the dataset to at most size rows */
herr_t truncate_dset( hid_t dataset_id,
const int maindim,
const hsize_t size)
{
hid_t space_id;
hsize_t *dims = NULL;
int rank;
/* Get the dataspace handle */
if ( (space_id = H5Dget_space(dataset_id)) < 0 )
goto out;
/* Get the rank */
if ( (rank = H5Sget_simple_extent_ndims(space_id)) < 0 )
goto out;
if (rank) { /* multidimensional case */
/* Book some memory for the selections */
dims = (hsize_t *)malloc(rank*sizeof(hsize_t));
/* Get dataset dimensionality */
if ( H5Sget_simple_extent_dims(space_id, dims, NULL) < 0 )
goto out;
/* Truncate the EArray */
dims[maindim] = size;
if ( H5Dset_extent(dataset_id, dims) < 0 )
goto out;
/* Release resources */
free(dims);
}
else { /* scalar case (should never enter here) */
printf("A scalar Array cannot be truncated!.\n");
goto out;
}
/* Free resources */
if ( H5Sclose(space_id) < 0 )
return -1;
return 0;
out:
if (dims) free(dims);
return -1;
}
herr_t H5TBOappend_records( hid_t dataset_id,
hid_t mem_type_id,
hsize_t nrecords,
hsize_t nrecords_orig,
const void *data )
{
hid_t space_id = -1; /* Shut up the compiler */
hsize_t count[1];
hsize_t offset[1];
hid_t mem_space_id = -1; /* Shut up the compiler */
hsize_t dims[1];
/* Extend the dataset */
dims[0] = nrecords_orig;
dims[0] += nrecords;
if ( H5Dset_extent(dataset_id, dims) < 0 )
goto out;
/* Create a simple memory data space */
count[0]=nrecords;
if ( (mem_space_id = H5Screate_simple( 1, count, NULL )) < 0 )
return -1;
/* Get the file data space */
if ( (space_id = H5Dget_space(dataset_id)) < 0 )
return -1;
/* Define a hyperslab in the dataset */
offset[0] = nrecords_orig;
if ( H5Sselect_hyperslab( space_id, H5S_SELECT_SET, offset, NULL, count, NULL) < 0 )
goto out;
if ( H5Dwrite( dataset_id, mem_type_id, mem_space_id, space_id, H5P_DEFAULT, data ) < 0 )
goto out;
/* Terminate access to the dataspace */
if ( H5Sclose( mem_space_id ) < 0 )
goto out;
if ( H5Sclose( space_id ) < 0 )
goto out;
return 0;
out:
return -1;
}
int H5mdfile::H5_Dextend(int argc, char **argv, Tcl_Interp *interp)
{
/* Extend dataset to higher dimensions */
H5Sclose(dataspace_id);
for(int i=0;i<dataset_rank;i++)
{
if(atoi(argv[3+i])>(int)dims[i])
{
dims[i]=atoi(argv[3+i])-dimstotal[i];
}
dimstotal[i] = atoi(argv[3+i]);
}
status = H5Dset_extent(dataset_id, dimstotal);
dataspace_id = H5Dget_space(dataset_id);
return TCL_OK;
}
asynStatus NDFileHDF5AttributeDataset::writeAttributeDataset(hdf5::When_t whenToSave, hsize_t *offsets, NDAttribute *ndAttr, int flush, int indexed)
{
asynStatus status = asynSuccess;
char * stackbuf[MAX_ATTRIBUTE_STRING_SIZE];
void* pDatavalue = stackbuf;
int ret;
//check if the attribute is meant to be saved at this time
if (whenToSave_ == whenToSave) {
// Extend the dataset as required to store the data
if (indexed == -1){
extendDataSet(offsets);
} else {
extendIndexDataSet(offsets[indexed]);
}
// find the data based on datatype
ret = ndAttr->getValue(ndAttr->getDataType(), pDatavalue, MAX_ATTRIBUTE_STRING_SIZE);
if (ret == ND_ERROR) {
memset(pDatavalue, 0, MAX_ATTRIBUTE_STRING_SIZE);
}
// Work with HDF5 library to select a suitable hyperslab (one element) and write the new data to it
H5Dset_extent(dataset_, dims_);
filespace_ = H5Dget_space(dataset_);
// Select the hyperslab
H5Sselect_hyperslab(filespace_, H5S_SELECT_SET, offset_, NULL, elementSize_, NULL);
// Write the data to the hyperslab.
H5Dwrite(dataset_, datatype_, memspace_, filespace_, H5P_DEFAULT, pDatavalue);
// Check if we are being asked to flush
if (flush == 1){
status = this->flushDataset();
}
H5Sclose(filespace_);
nextRecord_++;
}
return status;
}
void pyne::Material::write_hdf5(std::string filename, std::string datapath,
std::string nucpath, float row, int chunksize) {
int row_num = (int) row;
// Turn off annoying HDF5 errors
H5Eset_auto2(H5E_DEFAULT, NULL, NULL);
//Set file access properties so it closes cleanly
hid_t fapl;
fapl = H5Pcreate(H5P_FILE_ACCESS);
H5Pset_fclose_degree(fapl,H5F_CLOSE_STRONG);
// Create new/open datafile.
hid_t db;
if (pyne::file_exists(filename)) {
bool ish5 = H5Fis_hdf5(filename.c_str());
if (!ish5)
throw h5wrap::FileNotHDF5(filename);
db = H5Fopen(filename.c_str(), H5F_ACC_RDWR, fapl);
}
else
db = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
//
// Read in nuclist if available, write it out if not
//
bool nucpath_exists = h5wrap::path_exists(db, nucpath);
std::vector<int> nuclides;
int nuc_size;
hsize_t nuc_dims[1];
if (nucpath_exists) {
nuclides = h5wrap::h5_array_to_cpp_vector_1d<int>(db, nucpath, H5T_NATIVE_INT);
nuc_size = nuclides.size();
nuc_dims[0] = nuc_size;
} else {
nuclides = std::vector<int>();
for (pyne::comp_iter i = comp.begin(); i != comp.end(); i++)
nuclides.push_back(i->first);
nuc_size = nuclides.size();
// Create the data if it doesn't exist
int nuc_data [nuc_size];
for (int n = 0; n != nuc_size; n++)
nuc_data[n] = nuclides[n];
nuc_dims[0] = nuc_size;
hid_t nuc_space = H5Screate_simple(1, nuc_dims, NULL);
hid_t nuc_set = H5Dcreate2(db, nucpath.c_str(), H5T_NATIVE_INT, nuc_space,
H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Dwrite(nuc_set, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, nuc_data);
H5Fflush(db, H5F_SCOPE_GLOBAL);
};
//
// Write out the data itself to the file
//
hid_t data_set, data_space, data_hyperslab;
int data_rank = 1;
hsize_t data_dims[1] = {1};
hsize_t data_max_dims[1] = {H5S_UNLIMITED};
hsize_t data_offset[1] = {0};
size_t material_struct_size = sizeof(pyne::material_struct) + sizeof(double)*nuc_size;
hid_t desc = H5Tcreate(H5T_COMPOUND, material_struct_size);
hid_t comp_values_array_type = H5Tarray_create2(H5T_NATIVE_DOUBLE, 1, nuc_dims);
// make the data table type
H5Tinsert(desc, "mass", HOFFSET(pyne::material_struct, mass), H5T_NATIVE_DOUBLE);
H5Tinsert(desc, "density", HOFFSET(pyne::material_struct, density),
H5T_NATIVE_DOUBLE);
H5Tinsert(desc, "atoms_per_molecule", HOFFSET(pyne::material_struct, atoms_per_mol),
H5T_NATIVE_DOUBLE);
H5Tinsert(desc, "comp", HOFFSET(pyne::material_struct, comp),
comp_values_array_type);
material_struct * mat_data = new material_struct[material_struct_size];
(*mat_data).mass = mass;
(*mat_data).density = density;
(*mat_data).atoms_per_mol = atoms_per_molecule;
for (int n = 0; n != nuc_size; n++) {
if (0 < comp.count(nuclides[n]))
(*mat_data).comp[n] = comp[nuclides[n]];
else
(*mat_data).comp[n] = 0.0;
};
// get / make the data set
bool datapath_exists = h5wrap::path_exists(db, datapath);
if (datapath_exists) {
data_set = H5Dopen2(db, datapath.c_str(), H5P_DEFAULT);
data_space = H5Dget_space(data_set);
data_rank = H5Sget_simple_extent_dims(data_space, data_dims, data_max_dims);
// Determine the row size.
if (std::signbit(row))
row_num = data_dims[0] + row; // careful, row is negative
if (data_dims[0] <= row_num) {
// row == -0, extend to data set so that we can append, or
// row_num is larger than current dimension, resize to accomodate.
data_dims[0] = row_num + 1;
H5Dset_extent(data_set, data_dims);
}
data_offset[0] = row_num;
} else {
// Get full space
data_space = H5Screate_simple(1, data_dims, data_max_dims);
// Make data set properties to enable chunking
hid_t data_set_params = H5Pcreate(H5P_DATASET_CREATE);
hsize_t chunk_dims[1] ={chunksize};
H5Pset_chunk(data_set_params, 1, chunk_dims);
H5Pset_deflate(data_set_params, 1);
material_struct * data_fill_value = new material_struct[material_struct_size];
(*data_fill_value).mass = -1.0;
(*data_fill_value).density= -1.0;
(*data_fill_value).atoms_per_mol = -1.0;
for (int n = 0; n != nuc_size; n++)
(*data_fill_value).comp[n] = 0.0;
H5Pset_fill_value(data_set_params, desc, &data_fill_value);
// Create the data set
data_set = H5Dcreate2(db, datapath.c_str(), desc, data_space, H5P_DEFAULT,
data_set_params, H5P_DEFAULT);
H5Dset_extent(data_set, data_dims);
// Add attribute pointing to nuc path
hid_t nuc_attr_type = H5Tcopy(H5T_C_S1);
H5Tset_size(nuc_attr_type, nucpath.length());
hid_t nuc_attr_space = H5Screate(H5S_SCALAR);
hid_t nuc_attr = H5Acreate2(data_set, "nucpath", nuc_attr_type, nuc_attr_space,
H5P_DEFAULT, H5P_DEFAULT);
H5Awrite(nuc_attr, nuc_attr_type, nucpath.c_str());
H5Fflush(db, H5F_SCOPE_GLOBAL);
// Remember to de-allocate
delete[] data_fill_value;
};
// Get the data hyperslab
data_hyperslab = H5Dget_space(data_set);
hsize_t data_count[1] = {1};
H5Sselect_hyperslab(data_hyperslab, H5S_SELECT_SET, data_offset, NULL, data_count, NULL);
// Get a memory space for writing
hid_t mem_space = H5Screate_simple(1, data_count, data_max_dims);
// Write the row...
H5Dwrite(data_set, desc, mem_space, data_hyperslab, H5P_DEFAULT, mat_data);
// Close out the Dataset
H5Fflush(db, H5F_SCOPE_GLOBAL);
H5Dclose(data_set);
H5Sclose(data_space);
H5Tclose(desc);
//
// Write out the metadata to the file
//
std::string attrpath = datapath + "_metadata";
hid_t metadatapace, attrtype, metadataet, metadatalab, attrmemspace;
int attrrank;
attrtype = H5Tvlen_create(H5T_NATIVE_CHAR);
// get / make the data set
bool attrpath_exists = h5wrap::path_exists(db, attrpath);
if (attrpath_exists) {
metadataet = H5Dopen2(db, attrpath.c_str(), H5P_DEFAULT);
metadatapace = H5Dget_space(metadataet);
attrrank = H5Sget_simple_extent_dims(metadatapace, data_dims, data_max_dims);
if (data_dims[0] <= row_num) {
// row == -0, extend to data set so that we can append, or
// row_num is larger than current dimension, resize to accomodate.
data_dims[0] = row_num + 1;
H5Dset_extent(metadataet, data_dims);
}
data_offset[0] = row_num;
} else {
hid_t metadataetparams;
hsize_t attrchunkdims [1];
// Make data set properties to enable chunking
metadataetparams = H5Pcreate(H5P_DATASET_CREATE);
attrchunkdims[0] = chunksize;
H5Pset_chunk(metadataetparams, 1, attrchunkdims);
H5Pset_deflate(metadataetparams, 1);
hvl_t attrfillvalue [1];
attrfillvalue[0].len = 3;
attrfillvalue[0].p = (char *) "{}\n";
H5Pset_fill_value(metadataetparams, attrtype, &attrfillvalue);
// make dataset
metadatapace = H5Screate_simple(1, data_dims, data_max_dims);
metadataet = H5Dcreate2(db, attrpath.c_str(), attrtype, metadatapace,
H5P_DEFAULT, metadataetparams, H5P_DEFAULT);
H5Dset_extent(metadataet, data_dims);
};
// set the attr string
hvl_t attrdata [1];
Json::FastWriter writer;
std::string metadatatr = writer.write(metadata);
attrdata[0].p = (char *) metadatatr.c_str();
attrdata[0].len = metadatatr.length();
// write the attr
metadatalab = H5Dget_space(metadataet);
H5Sselect_hyperslab(metadatalab, H5S_SELECT_SET, data_offset, NULL, data_count, NULL);
attrmemspace = H5Screate_simple(1, data_count, data_max_dims);
H5Dwrite(metadataet, attrtype, attrmemspace, metadatalab, H5P_DEFAULT, attrdata);
// close attr data objects
H5Fflush(db, H5F_SCOPE_GLOBAL);
H5Dclose(metadataet);
H5Sclose(metadatapace);
H5Tclose(attrtype);
// Close out the HDF5 file
H5Fclose(db);
// Remember the milk!
// ...by which I mean to deallocate
delete[] mat_data;
};
/*
* Append planes, each of (1,2*chunksize,2*chunksize) to the dataset.
* In other words, 4 chunks are appended to the dataset at a time.
* Fill each plane with the plane number and then write it at the nth plane.
* Increase the plane number and repeat till the end of dataset, when it
* reaches chunksize long. End product is a (2*chunksize)^3 cube.
*
* Return: 0 succeed; -1 fail.
*/
static int
write_file(void)
{
hid_t fid; /* File ID for new HDF5 file */
hid_t dsid; /* dataset ID */
hid_t fapl; /* File access property list */
hid_t dcpl; /* Dataset creation property list */
char *name;
UC_CTYPE *buffer, *bufptr; /* data buffer */
hsize_t cz=chunksize_g; /* Chunk size */
hid_t f_sid; /* dataset file space id */
hid_t m_sid; /* memory space id */
int rank; /* rank */
hsize_t chunk_dims[3]; /* Chunk dimensions */
hsize_t dims[3]; /* Dataspace dimensions */
hsize_t memdims[3]; /* Memory space dimensions */
hsize_t start[3] = {0,0,0}, count[3]; /* Hyperslab selection values */
hbool_t disabled; /* Object's disabled status */
hsize_t i, j, k;
name = filename_g;
/* Open the file */
if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
return -1;
if(use_swmr_g)
if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
return -1;
if((fid = H5Fopen(name, H5F_ACC_RDWR | (use_swmr_g ? H5F_ACC_SWMR_WRITE : 0), fapl)) < 0){
fprintf(stderr, "H5Fopen failed\n");
return -1;
}
/* Open the dataset of the program name */
if((dsid = H5Dopen2(fid, progname_g, H5P_DEFAULT)) < 0){
fprintf(stderr, "H5Dopen2 failed\n");
return -1;
}
/* Disabled mdc flushed for the dataset */
if(H5Odisable_mdc_flushes(dsid) < 0) {
fprintf(stderr, "H5Odisable_mdc_flushes failed\n");
return -1;
}
/* Get mdc disabled status of the dataset */
if(H5Oare_mdc_flushes_disabled(dsid, &disabled) < 0) {
fprintf(stderr, "H5Oare_mdc_flushes_disabled failed\n");
return -1;
} else if(disabled)
printf("Dataset has disabled mdc flushes.\n");
else
printf("Dataset should have disabled its mdc flushes.\n");
/* Find chunksize used */
if ((dcpl = H5Dget_create_plist(dsid)) < 0){
fprintf(stderr, "H5Dget_create_plist failed\n");
return -1;
}
if (H5D_CHUNKED != H5Pget_layout(dcpl)){
fprintf(stderr, "storage layout is not chunked\n");
return -1;
}
if ((rank = H5Pget_chunk(dcpl, 3, chunk_dims)) != 3){
fprintf(stderr, "storage rank is not 3\n");
return -1;
}
/* verify chunk_dims against set paramenters */
if (chunk_dims[0]!= chunkdims_g[0] || chunk_dims[1] != cz || chunk_dims[2] != cz){
fprintf(stderr, "chunk size is not as expected. Got dims=(%llu,%llu,%llu)\n",
(unsigned long long)chunk_dims[0], (unsigned long long)chunk_dims[1],
(unsigned long long)chunk_dims[2]);
return -1;
}
/* allocate space for data buffer 1 X dims[1] X dims[2] of UC_CTYPE */
memdims[0]=1;
memdims[1] = dims_g[1];
memdims[2] = dims_g[2];
if ((buffer=(UC_CTYPE*)HDmalloc((size_t)memdims[1]*(size_t)memdims[2]*sizeof(UC_CTYPE)))==NULL) {
fprintf(stderr, "malloc: failed\n");
return -1;
};
/*
* Get dataset rank and dimension.
*/
f_sid = H5Dget_space(dsid); /* Get filespace handle first. */
rank = H5Sget_simple_extent_ndims(f_sid);
if (rank != UC_RANK){
fprintf(stderr, "rank(%d) of dataset does not match\n", rank);
return -1;
}
if (H5Sget_simple_extent_dims(f_sid, dims, NULL) < 0){
fprintf(stderr, "H5Sget_simple_extent_dims got error\n");
return -1;
}
printf("dataset rank %d, dimensions %llu x %llu x %llu\n",
rank, (unsigned long long)(dims[0]), (unsigned long long)(dims[1]),
(unsigned long long)(dims[2]));
/* verify that file space dims are as expected and are consistent with memory space dims */
if (dims[0] != 0 || dims[1] != memdims[1] || dims[2] != memdims[2]){
fprintf(stderr, "dataset is not empty. Got dims=(%llu,%llu,%llu)\n",
(unsigned long long)dims[0], (unsigned long long)dims[1],
(unsigned long long)dims[2]);
return -1;
}
/* setup mem-space for buffer */
if ((m_sid=H5Screate_simple(rank, memdims, NULL))<0){
fprintf(stderr, "H5Screate_simple for memory failed\n");
return -1;
};
/* write planes */
count[0]=1;
count[1]=dims[1];
count[2]=dims[2];
for (i=0; i<nplanes_g; i++){
/* fill buffer with value i+1 */
bufptr = buffer;
for (j=0; j<dims[1]; j++)
for (k=0; k<dims[2]; k++)
*bufptr++ = i;
/* extend the dataset by one for new plane */
dims[0]=i+1;
if(H5Dset_extent(dsid, dims) < 0){
fprintf(stderr, "H5Dset_extent failed\n");
return -1;
}
/* Get the dataset's dataspace */
if((f_sid = H5Dget_space(dsid)) < 0){
fprintf(stderr, "H5Dset_extent failed\n");
return -1;
}
start[0]=i;
/* Choose the next plane to write */
if(H5Sselect_hyperslab(f_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0){
fprintf(stderr, "Failed H5Sselect_hyperslab\n");
return -1;
}
/* Write plane to the dataset */
if(H5Dwrite(dsid, UC_DATATYPE, m_sid, f_sid, H5P_DEFAULT, buffer) < 0){
fprintf(stderr, "Failed H5Dwrite\n");
return -1;
}
/* Flush the dataset for every "chunkplanes_g" planes */
if(!((i + 1) % (hsize_t)chunkplanes_g)) {
if(H5Dflush(dsid) < 0) {
fprintf(stderr, "Failed to H5Dflush dataset\n");
return -1;
}
}
}
if(H5Dflush(dsid) < 0) {
fprintf(stderr, "Failed to H5Dflush dataset\n");
return -1;
}
/* Enable mdc flushes for the dataset */
/* Closing the dataset later will enable mdc flushes automatically if this is not done */
if(disabled)
if(H5Oenable_mdc_flushes(dsid) < 0) {
fprintf(stderr, "Failed to H5Oenable_mdc_flushes\n");
return -1;
}
/* Done writing. Free/Close all resources including data file */
HDfree(buffer);
if(H5Dclose(dsid) < 0){
fprintf(stderr, "Failed to close datasete\n");
return -1;
}
if(H5Sclose(m_sid) < 0){
fprintf(stderr, "Failed to close memory space\n");
return -1;
}
if(H5Sclose(f_sid) < 0){
fprintf(stderr, "Failed to close file space\n");
return -1;
}
if(H5Pclose(fapl) < 0){
fprintf(stderr, "Failed to property list\n");
return -1;
}
if(H5Fclose(fid) < 0){
fprintf(stderr, "Failed to close file id\n");
return -1;
}
return 0;
} /* write_file() */
int
main (void)
{
hid_t vfile, file, src_space, mem_space, vspace,
vdset, dset; /* Handles */
hid_t dcpl;
herr_t status;
hsize_t vdsdims[3] = {4*DIM0_1, VDSDIM1, VDSDIM2},
vdsdims_max[3] = {VDSDIM0, VDSDIM1, VDSDIM2},
dims[3] = {DIM0_1, DIM1, DIM2},
extdims[3] = {2*DIM0_1, DIM1, DIM2},
chunk_dims[3] = {DIM0_1, DIM1, DIM2},
dims_max[3] = {DIM0, DIM1, DIM2},
vdsdims_out[3],
vdsdims_max_out[3],
start[3], /* Hyperslab parameters */
stride[3],
count[3],
src_count[3],
block[3];
hsize_t start_out[3], /* Hyperslab parameter out */
stride_out[3],
count_out[3],
block_out[3];
int i, j, k;
H5D_layout_t layout; /* Storage layout */
size_t num_map; /* Number of mappings */
ssize_t len; /* Length of the string; also a return value */
char *filename;
char *dsetname;
int wdata[DIM0_1*DIM1*DIM2];
int rdata[80][10][10];
int a_rdata[20][10][10];
/*
* Create source files and datasets. This step is optional.
*/
for (i=0; i < PLANE_STRIDE; i++) {
/*
* Initialize data for i-th source dataset.
*/
for (j = 0; j < DIM0_1*DIM1*DIM2; j++) wdata[j] = i+1;
/*
* Create the source files and datasets. Write data to each dataset and
* close all resources.
*/
file = H5Fcreate (SRC_FILE[i], H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
src_space = H5Screate_simple (RANK, dims, dims_max);
dcpl = H5Pcreate(H5P_DATASET_CREATE);
status = H5Pset_chunk (dcpl, RANK, chunk_dims);
dset = H5Dcreate2 (file, SRC_DATASET[i], H5T_NATIVE_INT, src_space, H5P_DEFAULT,
dcpl, H5P_DEFAULT);
status = H5Dwrite (dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT,
wdata);
status = H5Sclose (src_space);
status = H5Pclose (dcpl);
status = H5Dclose (dset);
status = H5Fclose (file);
}
vfile = H5Fcreate (VFILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
/* Create VDS dataspace. */
vspace = H5Screate_simple (RANK, vdsdims, vdsdims_max);
/* Create dataspaces for the source dataset. */
src_space = H5Screate_simple (RANK, dims, dims_max);
/* Create VDS creation property */
dcpl = H5Pcreate (H5P_DATASET_CREATE);
/* Initialize hyperslab values */
start[0] = 0;
start[1] = 0;
start[2] = 0;
stride[0] = PLANE_STRIDE; /* we will select every fifth plane in VDS */
stride[1] = 1;
stride[2] = 1;
count[0] = H5S_UNLIMITED;
count[1] = 1;
count[2] = 1;
src_count[0] = H5S_UNLIMITED;
src_count[1] = 1;
src_count[2] = 1;
block[0] = 1;
block[1] = DIM1;
block[2] = DIM2;
/*
* Build the mappings
*
*/
status = H5Sselect_hyperslab (src_space, H5S_SELECT_SET, start, NULL, src_count, block);
for (i=0; i < PLANE_STRIDE; i++) {
status = H5Sselect_hyperslab (vspace, H5S_SELECT_SET, start, stride, count, block);
status = H5Pset_virtual (dcpl, vspace, SRC_FILE[i], SRC_DATASET[i], src_space);
start[0]++;
}
H5Sselect_none(vspace);
/* Create a virtual dataset */
vdset = H5Dcreate2 (vfile, DATASET, H5T_NATIVE_INT, vspace, H5P_DEFAULT,
dcpl, H5P_DEFAULT);
status = H5Sclose (vspace);
status = H5Sclose (src_space);
status = H5Pclose (dcpl);
/* Let's get space of the VDS and its dimension; we should get 40x10x10 */
vspace = H5Dget_space (vdset);
H5Sget_simple_extent_dims (vspace, vdsdims_out, vdsdims_max_out);
printf ("VDS dimensions first time \n");
printf (" Current: ");
for (i=0; i<RANK; i++)
printf (" %d ", (int)vdsdims_out[i]);
printf ("\n");
/* Let's add data to the source datasets and check new dimensions for VDS */
for (i=0; i < PLANE_STRIDE; i++) {
/*
* Initialize data for i-th source dataset.
*/
for (j = 0; j < DIM0_1*DIM1*DIM2; j++) wdata[j] = 10*(i+1);
/*
* Create the source files and datasets. Write data to each dataset and
* close all resources.
*/
file = H5Fopen (SRC_FILE[i], H5F_ACC_RDWR, H5P_DEFAULT);
dset = H5Dopen2 (file, SRC_DATASET[i], H5P_DEFAULT);
status = H5Dset_extent (dset, extdims);
src_space = H5Dget_space (dset);
start[0] = DIM0_1;
start[1] = 0;
start[2] = 0;
count[0] = 1;
count[1] = 1;
count[2] = 1;
block[0] = DIM0_1;
block[1] = DIM1;
block[2] = DIM2;
mem_space = H5Screate_simple(RANK, dims, NULL);
status = H5Sselect_hyperslab (src_space, H5S_SELECT_SET, start, NULL, count, block);
status = H5Dwrite (dset, H5T_NATIVE_INT, mem_space, src_space, H5P_DEFAULT,
wdata);
status = H5Sclose (src_space);
status = H5Dclose (dset);
status = H5Fclose (file);
}
status = H5Dclose (vdset);
status = H5Fclose (vfile);
/*
* Now we begin the read section of this example.
*/
/*
* Open file and dataset using the default properties.
*/
vfile = H5Fopen (VFILE, H5F_ACC_RDONLY, H5P_DEFAULT);
vdset = H5Dopen2 (vfile, DATASET, H5P_DEFAULT);
/*
* Get creation property list and mapping properties.
*/
dcpl = H5Dget_create_plist (vdset);
/*
* Get storage layout.
*/
layout = H5Pget_layout (dcpl);
if (H5D_VIRTUAL == layout)
printf(" Dataset has a virtual layout \n");
else
printf(" Wrong layout found \n");
/*
* Find number of mappings.
*/
status = H5Pget_virtual_count (dcpl, &num_map);
printf(" Number of mappings is %lu\n", (unsigned long)num_map);
/*
* Get mapping parameters for each mapping.
*/
for (i = 0; i < (int)num_map; i++) {
printf(" Mapping %d \n", i);
printf(" Selection in the virtual dataset \n");
/* Get selection in the virttual dataset */
vspace = H5Pget_virtual_vspace (dcpl, (size_t)i);
if (H5Sget_select_type(vspace) == H5S_SEL_HYPERSLABS) {
if (H5Sis_regular_hyperslab(vspace)) {
status = H5Sget_regular_hyperslab (vspace, start_out, stride_out, count_out, block_out);
printf(" start = [%llu, %llu, %llu] \n", (unsigned long long)start_out[0], (unsigned long long)start_out[1], (unsigned long long)start_out[2]);
printf(" stride = [%llu, %llu, %llu] \n", (unsigned long long)stride_out[0], (unsigned long long)stride_out[1], (unsigned long long)stride_out[2]);
printf(" count = [%llu, %llu, %llu] \n", (unsigned long long)count_out[0], (unsigned long long)count_out[1], (unsigned long long)count_out[2]);
printf(" block = [%llu, %llu, %llu] \n", (unsigned long long)block_out[0], (unsigned long long)block_out[1], (unsigned long long)block_out[2]);
}
}
/* Get source file name */
len = H5Pget_virtual_filename (dcpl, (size_t)i, NULL, 0);
filename = (char *)malloc((size_t)len*sizeof(char)+1);
H5Pget_virtual_filename (dcpl, (size_t)i, filename, len+1);
printf(" Source filename %s\n", filename);
/* Get source dataset name */
len = H5Pget_virtual_dsetname (dcpl, (size_t)i, NULL, 0);
dsetname = (char *)malloc((size_t)len*sizeof(char)+1);
H5Pget_virtual_dsetname (dcpl, (size_t)i, dsetname, len+1);
printf(" Source dataset name %s\n", dsetname);
/* Get selection in the source dataset */
printf(" Selection in the source dataset \n");
src_space = H5Pget_virtual_srcspace (dcpl, (size_t)i);
if (H5Sget_select_type(src_space) == H5S_SEL_HYPERSLABS) {
if (H5Sis_regular_hyperslab(src_space)) {
status = H5Sget_regular_hyperslab (src_space, start_out, stride_out, count_out, block_out);
printf(" start = [%llu, %llu, %llu] \n", (unsigned long long)start_out[0], (unsigned long long)start_out[1], (unsigned long long)start_out[2]);
printf(" stride = [%llu, %llu, %llu] \n", (unsigned long long)stride_out[0], (unsigned long long)stride_out[1], (unsigned long long)stride_out[2]);
printf(" count = [%llu, %llu, %llu] \n", (unsigned long long)count_out[0], (unsigned long long)count_out[1], (unsigned long long)count_out[2]);
printf(" block = [%llu, %llu, %llu] \n", (unsigned long long)block_out[0], (unsigned long long)block_out[1], (unsigned long long)block_out[2]);
}
}
H5Sclose(vspace);
H5Sclose(src_space);
free(filename);
free(dsetname);
}
/*
* Read data from VDS.
*/
vspace = H5Dget_space (vdset);
H5Sget_simple_extent_dims (vspace, vdsdims_out, vdsdims_max_out);
printf ("VDS dimensions second time \n");
printf (" Current: ");
for (i=0; i<RANK; i++)
printf (" %d ", (int)vdsdims_out[i]);
printf ("\n");
/* Read all VDS data */
/* EIP We should be able to do it by using H5S_ALL instead of making selection
* or using H5Sselect_all from vspace.
*/
start[0] = 0;
start[1] = 0;
start[2] = 0;
count[0] = 1;
count[1] = 1;
count[2] = 1;
block[0] = vdsdims_out[0];
block[1] = vdsdims_out[1];
block[2] = vdsdims_out[2];
status = H5Sselect_hyperslab (vspace, H5S_SELECT_SET, start, NULL, count, block);
mem_space = H5Screate_simple(RANK, vdsdims_out, NULL);
status = H5Dread (vdset, H5T_NATIVE_INT, mem_space, vspace, H5P_DEFAULT,
rdata);
printf (" All data: \n");
for (i=0; i < (int)vdsdims_out[0]; i++) {
for (j=0; j < (int)vdsdims_out[1]; j++) {
printf ("(%d, %d, 0)", i, j);
for (k=0; k < (int)vdsdims_out[2]; k++)
printf (" %d ", rdata[i][j][k]);
printf ("\n");
}
}
/* Read VDS, but only data mapeed to dataset a.h5 */
start[0] = 0;
start[1] = 0;
start[2] = 0;
stride[0] = PLANE_STRIDE;
stride[1] = 1;
stride[2] = 1;
count[0] = 2*DIM0_1;
count[1] = 1;
count[2] = 1;
block[0] = 1;
block[1] = vdsdims_out[1];
block[2] = vdsdims_out[2];
dims[0] = 2*DIM0_1;
status = H5Sselect_hyperslab (vspace, H5S_SELECT_SET, start, stride, count, block);
mem_space = H5Screate_simple(RANK, dims, NULL);
status = H5Dread (vdset, H5T_NATIVE_INT, mem_space, vspace, H5P_DEFAULT,
a_rdata);
printf (" All data: \n");
for (i=0; i < 2*DIM0_1; i++) {
for (j=0; j < (int)vdsdims_out[1]; j++) {
printf ("(%d, %d, 0)", i, j);
for (k=0; k < (int)vdsdims_out[2]; k++)
printf (" %d ", a_rdata[i][j][k]);
printf ("\n");
}
}
/*
* Close and release resources.
*/
status = H5Sclose(mem_space);
status = H5Pclose (dcpl);
status = H5Dclose (vdset);
status = H5Fclose (vfile);
return 0;
}
int
main (void)
{
hid_t vfile, file, src_space, mem_space, vspace,
vdset, dset; /* Handles */
hid_t dcpl, dapl;
herr_t status;
hsize_t vdsdims[3] = {4*DIM0_1, VDSDIM1, VDSDIM2},
vdsdims_max[3] = {VDSDIM0, VDSDIM1, VDSDIM2},
dims[3] = {DIM0_1, DIM1, DIM2},
memdims[3] = {DIM0_1, DIM1, DIM2},
extdims[3] = {0, DIM1, DIM2}, /* Dimensions of the extended source datasets */
chunk_dims[3] = {DIM0_1, DIM1, DIM2},
dims_max[3] = {DIM0, DIM1, DIM2},
vdsdims_out[3],
vdsdims_max_out[3],
start[3], /* Hyperslab parameters */
stride[3],
count[3],
src_count[3],
block[3];
hsize_t start_out[3], /* Hyperslab parameter out */
stride_out[3],
count_out[3],
block_out[3];
int i, j;
H5D_layout_t layout; /* Storage layout */
size_t num_map; /* Number of mappings */
ssize_t len; /* Length of the string; also a return value */
char *filename;
char *dsetname;
int wdata[DIM0_1*DIM1*DIM2];
/*
* Create source files and datasets. This step is optional.
*/
for (i=0; i < PLANE_STRIDE; i++) {
/*
* Initialize data for i-th source dataset.
*/
for (j = 0; j < DIM0_1*DIM1*DIM2; j++) wdata[j] = i+1;
/*
* Create the source files and datasets. Write data to each dataset and
* close all resources.
*/
file = H5Fcreate (SRC_FILE[i], H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
src_space = H5Screate_simple (RANK, dims, dims_max);
dcpl = H5Pcreate(H5P_DATASET_CREATE);
status = H5Pset_chunk (dcpl, RANK, chunk_dims);
dset = H5Dcreate2 (file, SRC_DATASET[i], H5T_NATIVE_INT, src_space, H5P_DEFAULT,
dcpl, H5P_DEFAULT);
status = H5Dwrite (dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT,
wdata);
status = H5Sclose (src_space);
status = H5Pclose (dcpl);
status = H5Dclose (dset);
status = H5Fclose (file);
}
vfile = H5Fcreate (VFILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
/* Create VDS dataspace. */
vspace = H5Screate_simple (RANK, vdsdims, vdsdims_max);
/* Create dataspaces for the source dataset. */
src_space = H5Screate_simple (RANK, dims, dims_max);
/* Create VDS creation property */
dcpl = H5Pcreate (H5P_DATASET_CREATE);
/* Initialize hyperslab values */
start[0] = 0;
start[1] = 0;
start[2] = 0;
stride[0] = PLANE_STRIDE; /* we will select every fifth plane in VDS */
stride[1] = 1;
stride[2] = 1;
count[0] = H5S_UNLIMITED;
count[1] = 1;
count[2] = 1;
src_count[0] = H5S_UNLIMITED;
src_count[1] = 1;
src_count[2] = 1;
block[0] = 1;
block[1] = DIM1;
block[2] = DIM2;
/*
* Build the mappings
*
*/
status = H5Sselect_hyperslab (src_space, H5S_SELECT_SET, start, NULL, src_count, block);
for (i=0; i < PLANE_STRIDE; i++) {
status = H5Sselect_hyperslab (vspace, H5S_SELECT_SET, start, stride, count, block);
status = H5Pset_virtual (dcpl, vspace, SRC_FILE[i], SRC_DATASET[i], src_space);
start[0]++;
}
H5Sselect_none(vspace);
/* Create a virtual dataset */
vdset = H5Dcreate2 (vfile, DATASET, H5T_NATIVE_INT, vspace, H5P_DEFAULT,
dcpl, H5P_DEFAULT);
status = H5Sclose (vspace);
status = H5Sclose (src_space);
status = H5Pclose (dcpl);
/* Let's add data to the source datasets and check new dimensions for VDS */
/* We will add only one plane to the first source dataset, two planes to the
second one, three to the third, and four to the forth. */
for (i=0; i < PLANE_STRIDE; i++) {
/*
* Initialize data for i-th source dataset.
*/
for (j = 0; j < (i+1)*DIM1*DIM2; j++) wdata[j] = 10*(i+1);
/*
* Open the source files and datasets. Appen data to each dataset and
* close all resources.
*/
file = H5Fopen (SRC_FILE[i], H5F_ACC_RDWR, H5P_DEFAULT);
dset = H5Dopen2 (file, SRC_DATASET[i], H5P_DEFAULT);
extdims[0] = DIM0_1+i+1;
status = H5Dset_extent (dset, extdims);
src_space = H5Dget_space (dset);
start[0] = DIM0_1;
start[1] = 0;
start[2] = 0;
count[0] = 1;
count[1] = 1;
count[2] = 1;
block[0] = i+1;
block[1] = DIM1;
block[2] = DIM2;
memdims[0] = i+1;
mem_space = H5Screate_simple(RANK, memdims, NULL);
status = H5Sselect_hyperslab (src_space, H5S_SELECT_SET, start, NULL, count, block);
status = H5Dwrite (dset, H5T_NATIVE_INT, mem_space, src_space, H5P_DEFAULT,
wdata);
status = H5Sclose (src_space);
status = H5Dclose (dset);
status = H5Fclose (file);
}
status = H5Dclose (vdset);
status = H5Fclose (vfile);
/*
* Now we begin the read section of this example.
*/
/*
* Open file and dataset using the default properties.
*/
vfile = H5Fopen (VFILE, H5F_ACC_RDONLY, H5P_DEFAULT);
/*
* Open VDS using different access properties to use max or
* min extents depending on the sizes of the underlying datasets
*/
dapl = H5Pcreate (H5P_DATASET_ACCESS);
for(i = 0; i < 2; i++) {
status = H5Pset_virtual_view (dapl, i ? H5D_VDS_LAST_AVAILABLE : H5D_VDS_FIRST_MISSING);
vdset = H5Dopen2 (vfile, DATASET, dapl);
/* Let's get space of the VDS and its dimension; we should get 32(or 20)x10x10 */
vspace = H5Dget_space (vdset);
H5Sget_simple_extent_dims (vspace, vdsdims_out, vdsdims_max_out);
printf ("VDS dimensions, bounds = H5D_VDS_%s: ", i ? "LAST_AVAILABLE" : "FIRST_MISSING");
for (j=0; j<RANK; j++)
printf (" %d ", (int)vdsdims_out[j]);
printf ("\n");
/* Close */
status = H5Dclose (vdset);
status = H5Sclose (vspace);
}
status = H5Pclose (dapl);
vdset = H5Dopen2 (vfile, DATASET, H5P_DEFAULT);
/*
* Get creation property list and mapping properties.
*/
dcpl = H5Dget_create_plist (vdset);
/*
* Get storage layout.
*/
layout = H5Pget_layout (dcpl);
if (H5D_VIRTUAL == layout)
printf(" Dataset has a virtual layout \n");
else
printf(" Wrong layout found \n");
/*
* Find number of mappings.
*/
status = H5Pget_virtual_count (dcpl, &num_map);
printf(" Number of mappings is %lu\n", (unsigned long)num_map);
/*
* Get mapping parameters for each mapping.
*/
for (i = 0; i < (int)num_map; i++) {
printf(" Mapping %d \n", i);
printf(" Selection in the virtual dataset \n");
/* Get selection in the virttual dataset */
vspace = H5Pget_virtual_vspace (dcpl, (size_t)i);
if (H5Sget_select_type(vspace) == H5S_SEL_HYPERSLABS) {
if (H5Sis_regular_hyperslab(vspace)) {
status = H5Sget_regular_hyperslab (vspace, start_out, stride_out, count_out, block_out);
printf(" start = [%llu, %llu, %llu] \n", (unsigned long long)start_out[0], (unsigned long long)start_out[1], (unsigned long long)start_out[2]);
printf(" stride = [%llu, %llu, %llu] \n", (unsigned long long)stride_out[0], (unsigned long long)stride_out[1], (unsigned long long)stride_out[2]);
printf(" count = [%llu, %llu, %llu] \n", (unsigned long long)count_out[0], (unsigned long long)count_out[1], (unsigned long long)count_out[2]);
printf(" block = [%llu, %llu, %llu] \n", (unsigned long long)block_out[0], (unsigned long long)block_out[1], (unsigned long long)block_out[2]);
}
}
/* Get source file name */
len = H5Pget_virtual_filename (dcpl, (size_t)i, NULL, 0);
filename = (char *)malloc((size_t)len*sizeof(char)+1);
H5Pget_virtual_filename (dcpl, (size_t)i, filename, len+1);
printf(" Source filename %s\n", filename);
/* Get source dataset name */
len = H5Pget_virtual_dsetname (dcpl, (size_t)i, NULL, 0);
dsetname = (char *)malloc((size_t)len*sizeof(char)+1);
H5Pget_virtual_dsetname (dcpl, (size_t)i, dsetname, len+1);
printf(" Source dataset name %s\n", dsetname);
/* Get selection in the source dataset */
printf(" Selection in the source dataset \n");
src_space = H5Pget_virtual_srcspace (dcpl, (size_t)i);
if (H5Sget_select_type(src_space) == H5S_SEL_HYPERSLABS) {
if (H5Sis_regular_hyperslab(src_space)) {
status = H5Sget_regular_hyperslab (src_space, start_out, stride_out, count_out, block_out);
printf(" start = [%llu, %llu, %llu] \n", (unsigned long long)start_out[0], (unsigned long long)start_out[1], (unsigned long long)start_out[2]);
printf(" stride = [%llu, %llu, %llu] \n", (unsigned long long)stride_out[0], (unsigned long long)stride_out[1], (unsigned long long)stride_out[2]);
printf(" count = [%llu, %llu, %llu] \n", (unsigned long long)count_out[0], (unsigned long long)count_out[1], (unsigned long long)count_out[2]);
printf(" block = [%llu, %llu, %llu] \n", (unsigned long long)block_out[0], (unsigned long long)block_out[1], (unsigned long long)block_out[2]);
}
}
H5Sclose(vspace);
H5Sclose(src_space);
free(filename);
free(dsetname);
}
/*
* Close and release resources.
*/
status = H5Pclose (dcpl);
status = H5Dclose (vdset);
status = H5Fclose (vfile);
return 0;
}
// Informational }}}
// Resizing {{{
void Dataset::resize(hsize_t const* dimensions) const {
assertSuccess(
"resizing dataset",
H5Dset_extent(getId(),dimensions)
);
}
herr_t H5ARRAYappend_records( hid_t dataset_id,
hid_t type_id,
const int rank,
hsize_t *dims_orig,
hsize_t *dims_new,
int extdim,
const void *data )
{
hid_t space_id;
hid_t mem_space_id;
hsize_t *dims = NULL; /* Shut up the compiler */
hsize_t *start = NULL; /* Shut up the compiler */
int i;
/* Compute the arrays for new dimensions and coordinates and extents */
dims = malloc(rank*sizeof(hsize_t));
start = malloc(rank*sizeof(hsize_t));
for(i=0;i<rank;i++) {
dims[i] = dims_orig[i];
start[i] = 0;
}
dims[extdim] += dims_new[extdim];
start[extdim] = (hsize_t )dims_orig[extdim];
/* Extend the dataset */
if ( H5Dset_extent( dataset_id, dims ) < 0 )
goto out;
/* Create a simple memory data space */
if ( (mem_space_id = H5Screate_simple( rank, dims_new, NULL )) < 0 )
return -1;
/* Get the file data space */
if ( (space_id = H5Dget_space( dataset_id )) < 0 )
return -1;
/* Define a hyperslab in the dataset */
if ( H5Sselect_hyperslab( space_id, H5S_SELECT_SET, start, NULL, dims_new, NULL) < 0 )
goto out;
if ( H5Dwrite( dataset_id, type_id, mem_space_id, space_id, H5P_DEFAULT, data ) < 0 )
goto out;
/* Update the original dimensions of the array after a successful append */
dims_orig[extdim] += dims_new[extdim];
/* Terminate access to the dataspace */
if ( H5Sclose( mem_space_id ) < 0 )
goto out;
if ( H5Sclose( space_id ) < 0 )
goto out;
/* Release resources */
free(start);
free(dims);
return 0;
out:
if (start) free(start);
if (dims) free(dims);
return -1;
}
/*
* This program performs three different types of parallel access. It writes on
* the entire dataset, it extends the dataset to nchunks*CHUNK_SIZE, and it only
* opens the dataset. At the end, it verifies the size of the dataset to be
* consistent with argument 'chunk_factor'.
*/
static void
parallel_access_dataset(const char *filename, int chunk_factor, access_type action, hid_t *file_id, hid_t *dataset)
{
/* HDF5 gubbins */
hid_t memspace, dataspace; /* HDF5 file identifier */
hid_t access_plist; /* HDF5 ID for file access property list */
herr_t hrc; /* HDF5 return code */
hsize_t size[1];
hsize_t chunk_dims[1] ={CHUNK_SIZE};
hsize_t count[1];
hsize_t stride[1];
hsize_t block[1];
hsize_t offset[1]; /* Selection offset within dataspace */
hsize_t dims[1];
hsize_t maxdims[1];
/* Variables used in reading data back */
char buffer[CHUNK_SIZE];
int i;
long nchunks;
/* MPI Gubbins */
MPI_Offset filesize, /* actual file size */
est_filesize; /* estimated file size */
/* Initialize MPI */
MPI_Comm_size(MPI_COMM_WORLD,&mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank);
nchunks=chunk_factor*mpi_size;
/* Set up MPIO file access property lists */
access_plist = H5Pcreate(H5P_FILE_ACCESS);
VRFY((access_plist >= 0), "");
hrc = H5Pset_fapl_mpio(access_plist, MPI_COMM_WORLD, MPI_INFO_NULL);
VRFY((hrc >= 0), "");
/* Open the file */
if (*file_id<0){
*file_id = H5Fopen(filename, H5F_ACC_RDWR, access_plist);
VRFY((*file_id >= 0), "");
}
/* Open dataset*/
if (*dataset<0){
*dataset = H5Dopen2(*file_id, DSET_NAME, H5P_DEFAULT);
VRFY((*dataset >= 0), "");
}
memspace = H5Screate_simple(1, chunk_dims, NULL);
VRFY((memspace >= 0), "");
dataspace = H5Dget_space(*dataset);
VRFY((dataspace >= 0), "");
size[0] = nchunks*CHUNK_SIZE;
switch (action) {
/* all chunks are written by all the processes in an interleaved way*/
case write_all:
memset(buffer, mpi_rank+1, CHUNK_SIZE);
count[0] = 1;
stride[0] = 1;
block[0] = chunk_dims[0];
for (i=0; i<nchunks/mpi_size; i++){
offset[0] = (i*mpi_size+mpi_rank)*chunk_dims[0];
hrc = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, stride, count, block);
VRFY((hrc >= 0), "");
/* Write the buffer out */
hrc = H5Dwrite(*dataset, H5T_NATIVE_UCHAR, memspace, dataspace, H5P_DEFAULT, buffer);
VRFY((hrc >= 0), "H5Dwrite");
}
break;
/* only extends the dataset */
case extend_only:
/* check if new size is larger than old size */
hrc = H5Sget_simple_extent_dims(dataspace, dims, maxdims);
VRFY((hrc >= 0), "");
/* Extend dataset*/
if (size[0] > dims[0]) {
hrc = H5Dset_extent(*dataset, size);
VRFY((hrc >= 0), "");
}
break;
/* only opens the *dataset */
case open_only:
break;
}
/* Close up */
hrc = H5Dclose(*dataset);
VRFY((hrc >= 0), "");
*dataset = -1;
hrc = H5Sclose (dataspace);
VRFY((hrc >= 0), "");
hrc = H5Sclose (memspace);
VRFY((hrc >= 0), "");
hrc = H5Fclose(*file_id);
VRFY((hrc >= 0), "");
*file_id = -1;
/* verify file size */
filesize = get_filesize(filename);
est_filesize = nchunks*CHUNK_SIZE*sizeof(unsigned char);
VRFY((filesize >= est_filesize), "file size check");
/* Can close some plists */
hrc = H5Pclose(access_plist);
VRFY((hrc >= 0), "");
/* Make sure all processes are done before exiting this routine. Otherwise,
* other tests may start and change the test data file before some processes
* of this test are still accessing the file.
*/
MPI_Barrier(MPI_COMM_WORLD);
}
int
main (void)
{
hid_t file; /* handles */
hid_t dataspace, dataset;
hid_t filespace;
hid_t cparms;
hsize_t dims[2] = { 3, 3}; /*
* dataset dimensions
* at the creation time
*/
hsize_t dims1[2] = { 3, 3}; /* data1 dimensions */
hsize_t dims2[2] = { 7, 1}; /* data2 dimensions */
hsize_t dims3[2] = { 2, 2}; /* data3 dimensions */
hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED};
hsize_t chunk_dims[2] ={2, 5};
hsize_t size[2];
hsize_t offset[2];
herr_t status;
int data1[3][3] = { {1, 1, 1}, /* data to write */
{1, 1, 1},
{1, 1, 1} };
int data2[7] = { 2, 2, 2, 2, 2, 2, 2};
int data3[2][2] = { {3, 3},
{3, 3} };
int fillvalue = 0;
/*
* Create the data space with unlimited dimensions.
*/
dataspace = H5Screate_simple(RANK, dims, maxdims);
/*
* Create a new file. If file exists its contents will be overwritten.
*/
file = H5Fcreate(H5FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
/*
* Modify dataset creation properties, i.e. enable chunking.
*/
cparms = H5Pcreate(H5P_DATASET_CREATE);
status = H5Pset_chunk( cparms, RANK, chunk_dims);
status = H5Pset_fill_value (cparms, H5T_NATIVE_INT, &fillvalue );
/*
* Create a new dataset within the file using cparms
* creation properties.
*/
dataset = H5Dcreate2(file, DATASETNAME, H5T_NATIVE_INT, dataspace, H5P_DEFAULT,
cparms, H5P_DEFAULT);
/*
* Extend the dataset. This call assures that dataset is at least 3 x 3.
*/
size[0] = 3;
size[1] = 3;
status = H5Dset_extent(dataset, size);
/*
* Select a hyperslab.
*/
filespace = H5Dget_space(dataset);
offset[0] = 0;
offset[1] = 0;
status = H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL,
dims1, NULL);
/*
* Write the data to the hyperslab.
*/
status = H5Dwrite(dataset, H5T_NATIVE_INT, dataspace, filespace,
H5P_DEFAULT, data1);
/*
* Extend the dataset. Dataset becomes 10 x 3.
*/
dims[0] = dims1[0] + dims2[0];
size[0] = dims[0];
size[1] = dims[1];
status = H5Dset_extent(dataset, size);
/*
* Select a hyperslab.
*/
filespace = H5Dget_space(dataset);
offset[0] = 3;
offset[1] = 0;
status = H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL,
dims2, NULL);
/*
* Define memory space
*/
dataspace = H5Screate_simple(RANK, dims2, NULL);
/*
* Write the data to the hyperslab.
*/
status = H5Dwrite(dataset, H5T_NATIVE_INT, dataspace, filespace,
H5P_DEFAULT, data2);
/*
* Extend the dataset. Dataset becomes 10 x 5.
*/
dims[1] = dims1[1] + dims3[1];
size[0] = dims[0];
size[1] = dims[1];
status = H5Dset_extent(dataset, size);
/*
* Select a hyperslab
*/
filespace = H5Dget_space(dataset);
offset[0] = 0;
offset[1] = 3;
status = H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL,
dims3, NULL);
/*
* Define memory space.
*/
dataspace = H5Screate_simple(RANK, dims3, NULL);
/*
* Write the data to the hyperslab.
*/
status = H5Dwrite(dataset, H5T_NATIVE_INT, dataspace, filespace,
H5P_DEFAULT, data3);
/*
* Resulting dataset
*
* 1 1 1 3 3
* 1 1 1 3 3
* 1 1 1 0 0
* 2 0 0 0 0
* 2 0 0 0 0
* 2 0 0 0 0
* 2 0 0 0 0
* 2 0 0 0 0
* 2 0 0 0 0
* 2 0 0 0 0
*/
/*
* Close/release resources.
*/
H5Dclose(dataset);
H5Sclose(dataspace);
H5Sclose(filespace);
H5Pclose(cparms);
H5Fclose(file);
return 0;
}
/*
* Function: dset_write
* Purpose: Write buffer into the dataset.
* Return: SUCCESS or FAIL
* Programmer: Christian Chilan, April, 2008
* Modifications:
*/
static herr_t
dset_write(int local_dim, file_descr *fd, parameters *parms, void *buffer)
{
int cur_dim = order[local_dim]-1;
int ret_code = SUCCESS;
int k;
hsize_t dims[MAX_DIMS], maxdims[MAX_DIMS];
hsize_t i;
int j;
herr_t hrc;
/* iterates according to the dimensions in order array */
for (i=0; i < parms->dset_size[cur_dim]; i += parms->buf_size[cur_dim]){
h5offset[cur_dim] = (hssize_t)i;
offset[cur_dim] = (HDoff_t)i;
if (local_dim > 0){
dset_write(local_dim-1, fd, parms, buffer);
}else{
switch (parms->io_type) {
case POSIXIO:
/* initialize POSIX offset in the buffer */
for(j = 0; j < parms->rank; j++)
buf_offset[j] = 0;
buf_p = (unsigned char *)buffer;
/* write POSIX buffer */
posix_buffer_write(0, fd, parms, buffer);
break;
case HDF5:
/* if dimensions are extendable, extend them as needed during access */
if (parms->h5_use_chunks && parms->h5_extendable) {
hrc=H5Sget_simple_extent_dims(h5dset_space_id,dims,maxdims);
VRFY((hrc >= 0), "H5Sget_simple_extent_dims");
for (k=0; k < parms->rank; k++){
HDassert(h5offset[k] >= 0);
if (dims[k] <= (hsize_t)h5offset[k]) {
dims[k] = dims[k]+h5count[k];
hrc=H5Sset_extent_simple(h5dset_space_id,parms->rank,dims,maxdims);
VRFY((hrc >= 0), "H5Sset_extent_simple");
hrc=H5Dset_extent(h5ds_id,dims);
VRFY((hrc >= 0), "H5Dextend");
}
}
}
/* applies offset */
hrc = H5Soffset_simple(h5dset_space_id, h5offset);
VRFY((hrc >= 0), "H5Soffset_simple");
/* Write the buffer out */
hrc=H5Sget_simple_extent_dims(h5dset_space_id,dims,maxdims);
hrc = H5Dwrite(h5ds_id, ELMT_H5_TYPE, h5mem_space_id,
h5dset_space_id, h5dxpl, buffer);
VRFY((hrc >= 0), "H5Dwrite");
break;
default:
/* unknown request */
HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type);
HDassert(0 && "Unknown IO type");
break;
} /* switch (parms->io_type) */
}
}
done:
return ret_code;
}
PetscErrorCode ISView_General_HDF5(IS is, PetscViewer viewer)
{
hid_t filespace; /* file dataspace identifier */
hid_t chunkspace; /* chunk dataset property identifier */
hid_t plist_id; /* property list identifier */
hid_t dset_id; /* dataset identifier */
hid_t memspace; /* memory dataspace identifier */
hid_t inttype; /* int type (H5T_NATIVE_INT or H5T_NATIVE_LLONG) */
hid_t file_id, group;
herr_t status;
hsize_t dim, maxDims[3], dims[3], chunkDims[3], count[3],offset[3];
PetscInt bs, N, n, timestep, low;
const PetscInt *ind;
const char *isname;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = ISGetBlockSize(is,&bs);CHKERRQ(ierr);
ierr = PetscViewerHDF5OpenGroup(viewer, &file_id, &group);CHKERRQ(ierr);
ierr = PetscViewerHDF5GetTimestep(viewer, ×tep);CHKERRQ(ierr);
/* Create the dataspace for the dataset.
*
* dims - holds the current dimensions of the dataset
*
* maxDims - holds the maximum dimensions of the dataset (unlimited
* for the number of time steps with the current dimensions for the
* other dimensions; so only additional time steps can be added).
*
* chunkDims - holds the size of a single time step (required to
* permit extending dataset).
*/
dim = 0;
if (timestep >= 0) {
dims[dim] = timestep+1;
maxDims[dim] = H5S_UNLIMITED;
chunkDims[dim] = 1;
++dim;
}
ierr = ISGetSize(is, &N);CHKERRQ(ierr);
ierr = ISGetLocalSize(is, &n);CHKERRQ(ierr);
ierr = PetscHDF5IntCast(N/bs,dims + dim);CHKERRQ(ierr);
maxDims[dim] = dims[dim];
chunkDims[dim] = dims[dim];
++dim;
if (bs >= 1) {
dims[dim] = bs;
maxDims[dim] = dims[dim];
chunkDims[dim] = dims[dim];
++dim;
}
filespace = H5Screate_simple(dim, dims, maxDims);
if (filespace == -1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Cannot H5Screate_simple()");
#if defined(PETSC_USE_64BIT_INDICES)
inttype = H5T_NATIVE_LLONG;
#else
inttype = H5T_NATIVE_INT;
#endif
/* Create the dataset with default properties and close filespace */
ierr = PetscObjectGetName((PetscObject) is, &isname);CHKERRQ(ierr);
if (!H5Lexists(group, isname, H5P_DEFAULT)) {
/* Create chunk */
chunkspace = H5Pcreate(H5P_DATASET_CREATE);
if (chunkspace == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Pcreate()");
status = H5Pset_chunk(chunkspace, dim, chunkDims);CHKERRQ(status);
#if (H5_VERS_MAJOR * 10000 + H5_VERS_MINOR * 100 + H5_VERS_RELEASE >= 10800)
dset_id = H5Dcreate2(group, isname, inttype, filespace, H5P_DEFAULT, chunkspace, H5P_DEFAULT);
#else
dset_id = H5Dcreate(group, isname, inttype, filespace, H5P_DEFAULT);
#endif
if (dset_id == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Dcreate2()");
status = H5Pclose(chunkspace);CHKERRQ(status);
} else {
dset_id = H5Dopen2(group, isname, H5P_DEFAULT);
status = H5Dset_extent(dset_id, dims);CHKERRQ(status);
}
status = H5Sclose(filespace);CHKERRQ(status);
/* Each process defines a dataset and writes it to the hyperslab in the file */
dim = 0;
if (timestep >= 0) {
count[dim] = 1;
++dim;
}
ierr = PetscHDF5IntCast(n/bs,count + dim);CHKERRQ(ierr);
++dim;
if (bs >= 1) {
count[dim] = bs;
++dim;
}
if (n > 0) {
memspace = H5Screate_simple(dim, count, NULL);
if (memspace == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Screate_simple()");
} else {
/* Can't create dataspace with zero for any dimension, so create null dataspace. */
memspace = H5Screate(H5S_NULL);
if (memspace == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Screate()");
}
/* Select hyperslab in the file */
ierr = PetscLayoutGetRange(is->map, &low, NULL);CHKERRQ(ierr);
dim = 0;
if (timestep >= 0) {
offset[dim] = timestep;
++dim;
}
ierr = PetscHDF5IntCast(low/bs,offset + dim);CHKERRQ(ierr);
++dim;
if (bs >= 1) {
offset[dim] = 0;
++dim;
}
if (n > 0) {
filespace = H5Dget_space(dset_id);
if (filespace == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Dget_space()");
status = H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, count, NULL);CHKERRQ(status);
} else {
/* Create null filespace to match null memspace. */
filespace = H5Screate(H5S_NULL);
if (filespace == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Screate(H5S_NULL)");
}
/* Create property list for collective dataset write */
plist_id = H5Pcreate(H5P_DATASET_XFER);
if (plist_id == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Pcreate()");
#if defined(PETSC_HAVE_H5PSET_FAPL_MPIO)
status = H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE);CHKERRQ(status);
#endif
/* To write dataset independently use H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_INDEPENDENT) */
ierr = ISGetIndices(is, &ind);CHKERRQ(ierr);
status = H5Dwrite(dset_id, inttype, memspace, filespace, plist_id, ind);CHKERRQ(status);
status = H5Fflush(file_id, H5F_SCOPE_GLOBAL);CHKERRQ(status);
ierr = ISGetIndices(is, &ind);CHKERRQ(ierr);
/* Close/release resources */
if (group != file_id) {status = H5Gclose(group);CHKERRQ(status);}
status = H5Pclose(plist_id);CHKERRQ(status);
status = H5Sclose(filespace);CHKERRQ(status);
status = H5Sclose(memspace);CHKERRQ(status);
status = H5Dclose(dset_id);CHKERRQ(status);
ierr = PetscInfo1(is, "Wrote IS object with name %s\n", isname);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*-------------------------------------------------------------------------
* Function: addrem_records
*
* Purpose: Adds/removes a specified number of records to random datasets
* to the SWMR test file.
*
* Parameters: hid_t fid
* The file ID of the SWMR HDF5 file
*
* unsigned verbose
* Whether or not to emit verbose console messages
*
* unsigned long nops
* # of records to read/write in the datasets
*
* unsigned long flush_count
* # of records to write before flushing the file to disk
*
* Return: Success: 0
* Failure: -1
*
*-------------------------------------------------------------------------
*/
static int
addrem_records(hid_t fid, unsigned verbose, unsigned long nops, unsigned long flush_count)
{
hid_t tid; /* Datatype ID for records */
hid_t mem_sid; /* Memory dataspace ID */
hsize_t start[2] = {0, 0}, count[2] = {1, 1}; /* Hyperslab selection values */
hsize_t dim[2] = {1, 0}; /* Dataspace dimensions */
symbol_t buf[MAX_SIZE_CHANGE]; /* Write buffer */
H5AC_cache_config_t mdc_config_orig; /* Original metadata cache configuration */
H5AC_cache_config_t mdc_config_cork; /* Corked metadata cache configuration */
unsigned long op_to_flush; /* # of operations before flush */
unsigned long u, v; /* Local index variables */
assert(fid > 0);
/* Reset the buffer */
memset(&buf, 0, sizeof(buf));
/* Create a dataspace for the record to add */
if((mem_sid = H5Screate_simple(2, count, NULL)) < 0)
return -1;
/* Create datatype for appending records */
if((tid = create_symbol_datatype()) < 0)
return -1;
/* Get the current metadata cache configuration, and set up the corked
* configuration */
mdc_config_orig.version = H5AC__CURR_CACHE_CONFIG_VERSION;
if(H5Fget_mdc_config(fid, &mdc_config_orig) < 0)
return -1;
memcpy(&mdc_config_cork, &mdc_config_orig, sizeof(mdc_config_cork));
mdc_config_cork.evictions_enabled = FALSE;
mdc_config_cork.incr_mode = H5C_incr__off;
mdc_config_cork.flash_incr_mode = H5C_flash_incr__off;
mdc_config_cork.decr_mode = H5C_decr__off;
/* Add and remove records to random datasets, according to frequency
* distribution */
op_to_flush = flush_count;
for(u=0; u<nops; u++) {
symbol_info_t *symbol; /* Symbol to write record to */
hid_t file_sid; /* Dataset's space ID */
/* Get a random dataset, according to the symbol distribution */
symbol = choose_dataset();
/* Decide whether to shrink or expand, and by how much */
count[1] = (hsize_t)random() % (MAX_SIZE_CHANGE * 2) + 1;
if(count[1] > MAX_SIZE_CHANGE) {
/* Add records */
count[1] -= MAX_SIZE_CHANGE;
/* Set the buffer's IDs (equal to its position) */
for(v=0; v<count[1]; v++)
buf[v].rec_id = (uint64_t)symbol->nrecords + (uint64_t)v;
/* Set the memory space to the correct size */
if(H5Sset_extent_simple(mem_sid, 2, count, NULL) < 0)
return -1;
/* Get the coordinates to write */
start[1] = symbol->nrecords;
/* Cork the metadata cache, to prevent the object header from being
* flushed before the data has been written */
/*if(H5Fset_mdc_config(fid, &mdc_config_cork) < 0)
return(-1);*/
/* Extend the dataset's dataspace to hold the new record */
symbol->nrecords+= count[1];
dim[1] = symbol->nrecords;
if(H5Dset_extent(symbol->dsid, dim) < 0)
return -1;
/* Get the dataset's dataspace */
if((file_sid = H5Dget_space(symbol->dsid)) < 0)
return -1;
/* Choose the last record in the dataset */
if(H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0)
return -1;
/* Write record to the dataset */
if(H5Dwrite(symbol->dsid, tid, mem_sid, file_sid, H5P_DEFAULT, &buf) < 0)
return -1;
/* Uncork the metadata cache */
/*if(H5Fset_mdc_config(fid, &mdc_config_orig) < 0)
return -1;*/
/* Close the dataset's dataspace */
if(H5Sclose(file_sid) < 0)
return -1;
} /* end if */
else {
/* Shrink the dataset's dataspace */
if(count[1] > symbol->nrecords)
symbol->nrecords = 0;
else
symbol->nrecords -= count[1];
dim[1] = symbol->nrecords;
if(H5Dset_extent(symbol->dsid, dim) < 0)
return -1;
} /* end else */
/* Check for flushing file */
if(flush_count > 0) {
/* Decrement count of records to write before flushing */
op_to_flush--;
/* Check for counter being reached */
if(0 == op_to_flush) {
/* Flush contents of file */
if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0)
return -1;
/* Reset flush counter */
op_to_flush = flush_count;
} /* end if */
} /* end if */
} /* end for */
/* Emit informational message */
if(verbose)
fprintf(stderr, "Closing datasets\n");
/* Close the datasets */
for(u = 0; u < NLEVELS; u++)
for(v = 0; v < symbol_count[u]; v++)
if(H5Dclose(symbol_info[u][v].dsid) < 0)
return -1;
return 0;
}
int
main(void)
{
hid_t src_sid = -1; /* source dataset's dataspace ID */
hid_t src_dcplid = -1; /* source dataset property list ID */
hid_t vds_sid = -1; /* VDS dataspace ID */
hid_t vds_dcplid = -1; /* VDS dataset property list ID */
hid_t fid = -1; /* HDF5 file ID */
hid_t did = -1; /* dataset ID */
hid_t msid = -1; /* memory dataspace ID */
hid_t fsid = -1; /* file dataspace ID */
/* Hyperslab arrays */
hsize_t start[RANK] = {0, 0, 0};
hsize_t count[RANK] = {H5S_UNLIMITED, 1, 1};
int *buffer = NULL; /* data buffer */
int value = -1; /* value written to datasets */
hsize_t n = 0; /* number of elements in a plane */
int i; /* iterator */
int j; /* iterator */
int k; /* iterator */
/************************************
* Create source files and datasets *
************************************/
/* Create source dataspace ID */
if((src_sid = H5Screate_simple(RANK, UC_4_SOURCE_DIMS,
UC_4_SOURCE_MAX_DIMS)) < 0)
UC_ERROR
if(H5Sselect_hyperslab(src_sid, H5S_SELECT_SET, start, NULL,
UC_4_SOURCE_MAX_DIMS, NULL) < 0)
UC_ERROR
/* Create source files and datasets */
for(i = 0; i < UC_4_N_SOURCES; i++) {
/* source dataset dcpl */
if((src_dcplid = H5Pcreate(H5P_DATASET_CREATE)) < 0)
UC_ERROR
if(H5Pset_chunk(src_dcplid, RANK, UC_4_PLANE) < 0)
UC_ERROR
if(H5Pset_fill_value(src_dcplid, UC_4_SOURCE_DATATYPE,
&UC_4_FILL_VALUES[i]) < 0)
UC_ERROR
if(H5Pset_deflate(src_dcplid, COMPRESSION_LEVEL) < 0)
UC_ERROR
/* Create source file and dataset */
if((fid = H5Fcreate(UC_4_FILE_NAMES[i], H5F_ACC_TRUNC,
H5P_DEFAULT, H5P_DEFAULT)) < 0)
UC_ERROR
if((did = H5Dcreate2(fid, UC_4_SOURCE_DSET_NAME,
UC_4_SOURCE_DATATYPE, src_sid,
H5P_DEFAULT, src_dcplid, H5P_DEFAULT)) < 0)
UC_ERROR
/* Set the dataset's extent */
if(H5Dset_extent(did, UC_4_SOURCE_MAX_DIMS) < 0)
UC_ERROR
/* Create a data buffer that represents a plane */
n = UC_4_PLANE[1] * UC_4_PLANE[2];
if(NULL == (buffer = (int *)malloc(n * sizeof(int))))
UC_ERROR
/* Create the memory dataspace */
if((msid = H5Screate_simple(RANK, UC_4_PLANE, NULL)) < 0)
UC_ERROR
/* Get the file dataspace */
if((fsid = H5Dget_space(did)) < 0)
UC_ERROR
/* Write planes to the dataset */
for(j = 0; j < UC_4_SRC_PLANES; j++) {
value = ((i + 1) * 10) + j;
for(k = 0; k < n; k++)
buffer[k] = value;
start[0] = j;
start[1] = 0;
start[2] = 0;
if(H5Sselect_hyperslab(fsid, H5S_SELECT_SET, start, NULL, UC_4_PLANE, NULL) < 0)
UC_ERROR
if(H5Dwrite(did, H5T_NATIVE_INT, msid, fsid, H5P_DEFAULT, buffer) < 0)
UC_ERROR
} /* end for */
/* close */
if(H5Sclose(msid) < 0)
UC_ERROR
if(H5Sclose(fsid) < 0)
UC_ERROR
if(H5Pclose(src_dcplid) < 0)
UC_ERROR
if(H5Dclose(did) < 0)
UC_ERROR
if(H5Fclose(fid) < 0)
UC_ERROR
free(buffer);
} /* end for */
/*******************
* Create VDS file *
*******************/
/* Create file */
if((fid = H5Fcreate(UC_4_VDS_FILE_NAME, H5F_ACC_TRUNC,
H5P_DEFAULT, H5P_DEFAULT)) < 0)
UC_ERROR
/* Create VDS dcpl */
if((vds_dcplid = H5Pcreate(H5P_DATASET_CREATE)) < 0)
UC_ERROR
if(H5Pset_fill_value(vds_dcplid, UC_4_VDS_DATATYPE,
&UC_4_VDS_FILL_VALUE) < 0)
UC_ERROR
/* Create VDS dataspace */
if((vds_sid = H5Screate_simple(RANK, UC_4_VDS_DIMS,
UC_4_VDS_MAX_DIMS)) < 0)
UC_ERROR
start[0] = 0;
start[1] = 0;
start[2] = 0;
if(H5Sselect_hyperslab(vds_sid, H5S_SELECT_SET, start,
UC_4_SOURCE_MAX_DIMS, count, UC_4_SOURCE_MAX_DIMS) < 0)
UC_ERROR
/* Add VDS mapping - The mapped file name uses a printf-like
* naming scheme that automatically maps new files.
*/
if(H5Pset_virtual(vds_dcplid, vds_sid, UC_4_MAPPING_FILE_NAME,
UC_4_SOURCE_DSET_PATH, src_sid) < 0)
UC_ERROR
/* Create dataset */
if((did = H5Dcreate2(fid, UC_4_VDS_DSET_NAME, UC_4_VDS_DATATYPE, vds_sid,
H5P_DEFAULT, vds_dcplid, H5P_DEFAULT)) < 0)
UC_ERROR
/* close */
if(H5Sclose(src_sid) < 0)
UC_ERROR
if(H5Pclose(vds_dcplid) < 0)
UC_ERROR
if(H5Sclose(vds_sid) < 0)
UC_ERROR
if(H5Dclose(did) < 0)
UC_ERROR
if(H5Fclose(fid) < 0)
UC_ERROR
return EXIT_SUCCESS;
error:
H5E_BEGIN_TRY {
if(src_sid >= 0)
(void)H5Sclose(src_sid);
if(src_dcplid >= 0)
(void)H5Pclose(src_dcplid);
if(vds_sid >= 0)
(void)H5Sclose(vds_sid);
if(vds_dcplid >= 0)
(void)H5Pclose(vds_dcplid);
if(fid >= 0)
(void)H5Fclose(fid);
if(did >= 0)
(void)H5Dclose(did);
if(msid >= 0)
(void)H5Sclose(msid);
if(fsid >= 0)
(void)H5Sclose(fsid);
if(buffer != NULL)
free(buffer);
} H5E_END_TRY
return EXIT_FAILURE;
} /* end main() */
/*+++++++++++++++++++++++++
.IDENTifer PYTABLE_append_array
.PURPOSE append data to HDF5 dataset, extending the dimension ''extdim''
.INPUT/OUTPUT
call as stat = PYTABLE_append_array( locID, dset_name,
extdim, count, buffer );
input:
hid_t locID : HDF5 identifier of file or group
char *dset_name : name of dataset
int extdim : dimension to extend
int count : number of arrays to write
void *buffer : data to write
.RETURNS A negative value is returned on failure.
.COMMENTS none
-------------------------*/
herr_t PYTABLE_append_array( hid_t locID, const char *dset_name,
int extdim, int count, const void *buffer )
{
int rank;
hid_t dataID;
hid_t spaceID = -1;
hid_t mem_spaceID = -1;
hid_t typeID = -1;
hsize_t *dims = NULL;
hsize_t *dims_ext = NULL;
hsize_t *offset = NULL;
herr_t stat;
/* open the dataset. */
if ( (dataID = H5Dopen( locID, dset_name, H5P_DEFAULT )) < 0 ) return -1;
/* get the dataspace handle */
if ( (spaceID = H5Dget_space( dataID )) < 0 ) goto done;
/* get rank */
if ( (rank = H5Sget_simple_extent_ndims( spaceID )) < 0 ) goto done;
/* get dimensions */
dims = (hsize_t *) malloc( rank * sizeof(hsize_t) );
dims_ext = (hsize_t *) malloc( rank * sizeof(hsize_t) );
offset = (hsize_t *) calloc( rank, sizeof(hsize_t) );
if ( H5Sget_simple_extent_dims( spaceID, dims, NULL ) < 0 )
goto done;
offset[extdim] = dims[extdim];
(void) memcpy( dims_ext, dims, rank * sizeof(hsize_t) );
dims_ext[extdim] = count;
dims[extdim] += count;
/* terminate access to the dataspace */
if ( H5Sclose( spaceID ) < 0 ) goto done;
/* extend the dataset */
if ( H5Dset_extent( dataID, dims ) < 0 ) goto done;
/* select a hyperslab */
if ( (spaceID = H5Dget_space( dataID )) < 0 ) goto done;
stat = H5Sselect_hyperslab( spaceID, H5S_SELECT_SET, offset, NULL,
dims_ext, NULL );
free( dims );
free( offset );
if ( stat < 0 ) goto done;
/* define memory space */
if ( (mem_spaceID = H5Screate_simple( rank, dims_ext, NULL )) < 0 )
goto done;
free( dims_ext );
/* get an identifier for the datatype. */
if ( (typeID = H5Dget_type( dataID )) < 0 ) goto done;
/* write the data to the hyperslab */
stat = H5Dwrite( dataID, typeID, mem_spaceID, spaceID, H5P_DEFAULT,
buffer );
if ( stat < 0 ) goto done;
/* end access to the dataset */
if ( H5Dclose( dataID ) ) goto done;
/* terminate access to the datatype */
if ( H5Tclose( typeID ) < 0 ) goto done;
/* terminate access to the dataspace */
if ( H5Sclose( mem_spaceID ) < 0 ) goto done;
if ( H5Sclose( spaceID ) < 0 ) goto done;
return 0;
done:
if ( dims != 0 ) free( dims );
if ( dims_ext != 0 ) free( dims_ext );
if ( offset != 0 ) free( offset );
if ( typeID > 0 ) (void) H5Tclose( typeID );
if ( spaceID > 0 ) (void) H5Sclose( spaceID );
if ( mem_spaceID > 0 ) (void) H5Sclose( mem_spaceID );
if ( dataID > 0 ) (void) H5Dclose( dataID );
return -1;
}
int main(void)
{
hid_t file_id, prop_id, memspace_id, type_id;
hid_t group_id;
hid_t dataset_id, dataspace_id;
herr_t status;
hsize_t dims[1];
hsize_t maxdims[1];
float data[NPOINTS];
float floatval;
unsigned numdataobj = 0;
unsigned i, j;
char name[80];
hsize_t start[1] = {0};
hsize_t stride[1] = {1};
hsize_t count[1] = {1};
/* Create a file */
file_id = H5Fcreate(FILEN, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
/* Create a dataset to hold the number of data objects */
/* Create the data space */
dataspace_id = H5Screate(H5S_SCALAR);
/* Create dataset */
dataset_id = H5Dcreate2(file_id, "/NumDataObj",
H5T_NATIVE_UINT, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
/* Write value to NumDataObj dataset */
status = H5Dwrite(dataset_id, H5T_NATIVE_UINT, H5S_ALL,
H5S_ALL, H5P_DEFAULT, &numdataobj);
/* Close the identifiers */
status = H5Dclose(dataset_id);
status = H5Sclose(dataspace_id);
/* Create extendible arrays */
/* Set up for extendible dataset */
prop_id = H5Pcreate(H5P_DATASET_CREATE);
dims[0] = CHUNK_SIZE;
status = H5Pset_chunk(prop_id, 1, dims);
/* Create dataspace */
dims[0]=1;
maxdims[0]=H5S_UNLIMITED;
dataspace_id = H5Screate_simple(1, dims, maxdims);
for(i=0; i<NEXTARRAYS; i++)
{
/* Create dataset */
sprintf(name, "/ExtArray%06d", i);
dataset_id = H5Dcreate2(file_id, name,
H5T_NATIVE_FLOAT, dataspace_id, H5P_DEFAULT, prop_id, H5P_DEFAULT);
/* Close the identifier */
status = H5Dclose(dataset_id);
}
/* Close the identifiers */
status = H5Sclose(dataspace_id);
status = H5Pclose(prop_id);
/* Create group to hold data object data arrays */
group_id = H5Gcreate2(file_id, "/DataArray", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Gclose(group_id);
for(j=0; j<NDATAOBJECTS; j++)
{
printf("\rWriting Object #%d of %d", j+1, NDATAOBJECTS);
fflush(stdout);
floatval = (float)j;
/* Create group to hold data arrays for this object */
sprintf(name, "/DataArray/%06d", j);
group_id = H5Gcreate2(file_id, name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if(group_id < 0) {
fprintf(stderr, "Failed to create DataArray group.\n");
status = H5Fclose(file_id);
return -1;
}
/* Loop over data arrays */
for(i=0; i<NDATAARRAYS; i++)
{
/* Create dataspace */
dims[0]=NPOINTS;
maxdims[0]=NPOINTS;
dataspace_id = H5Screate_simple(1 ,dims, maxdims);
/* Create dataset */
sprintf(name, "DataArray%06d", i);
dataset_id = H5Dcreate2(group_id, name,
H5T_NATIVE_FLOAT, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if(dataset_id < 0) {
fprintf(stderr, "Failed to create DataArray dataset.\n");
status = H5Fclose(file_id);
return -1;
}
/* Write the data array data */
status = H5Dwrite(dataset_id, H5T_NATIVE_FLOAT, H5S_ALL,
H5S_ALL, H5P_DEFAULT, data);
if(status < 0) {
fprintf(stderr, "Failed to write DataArray dataset.\n");
status = H5Fclose(file_id);
return -1;
}
/* Close the identifiers */
status = H5Dclose(dataset_id);
status = H5Sclose(dataspace_id);
}
/* Open NumDataObj dataset */
dataset_id = H5Dopen2(file_id, "/NumDataObj", H5P_DEFAULT);
if(dataset_id < 0) {
fprintf(stderr, "Failed to open NumDataObj dataset.\n");
status = H5Fclose(file_id);
return -1;
}
/* Write value to NumDataObj dataset */
numdataobj = j + 1;
status = H5Dwrite(dataset_id, H5T_NATIVE_UINT, H5S_ALL,
H5S_ALL, H5P_DEFAULT, &numdataobj);
if(status < 0) {
fprintf(stderr, "Failed to write NumDataObj dataset.\n");
status = H5Fclose(file_id);
return -1;
}
/* Close identifiers */
status = H5Dclose(dataset_id);
status = H5Gclose(group_id);
/* Extend attribute arrays */
for(i = 0; i < NEXTARRAYS; i++) {
/* Open extendable dataset */
sprintf(name, "/ExtArray%06d", i);
dataset_id = H5Dopen2(file_id, name, H5P_DEFAULT);
if(dataset_id < 0) {
fprintf(stderr, "Failed to open ExtArray dataset.\n");
status = H5Fclose(file_id);
return -1;
} /* end if */
/* Extend attribute dataset */
dims[0] = (hsize_t)j + 1;
status = H5Dset_extent(dataset_id, dims);
if(status < 0) {
fprintf(stderr, "Failed to extend DataArray dataset.\n");
status = H5Fclose(file_id);
return -1;
} /* end if */
/* Select element and write value to attribute dataset */
dims[0] = 1;
memspace_id = H5Screate_simple(1, dims, dims);
dataspace_id = H5Dget_space(dataset_id);
type_id = H5Dget_type(dataset_id);
start[0] = 0;
status = H5Sselect_hyperslab(memspace_id, H5S_SELECT_SET,
start, stride, count, NULL);
start[0] = (hssize_t)j;
status = H5Sselect_hyperslab(dataspace_id, H5S_SELECT_SET,
start, stride, count, NULL);
status = H5Dwrite(dataset_id, type_id, memspace_id,
dataspace_id, H5P_DEFAULT, &floatval);
if(status < 0)
{
fprintf(stderr, "Failed to write DataArray dataset.\n");
status = H5Fclose(file_id);
return -1;
}
/* Close identifiers */
status = H5Tclose(type_id);
status = H5Sclose(dataspace_id);
status = H5Sclose(memspace_id);
status = H5Dclose(dataset_id);
}
}
/* Close the file */
status = H5Fclose(file_id);
printf("\n");
return 0;
}
void hdfFileSaver::saveNow(void)
{
QDir prnt;
herr_t status;
hsize_t maxdims[1];
hsize_t d_dims[1];
int rank = 1;
hsize_t datasetlength;
size_t memdims[1];
hsize_t count[1]; /* size of subset in the file */
hsize_t offset[1]; /* subset offset in the file */
hsize_t mcount[1];
hsize_t moffset[1];
hid_t prop;
if (events==0)
return;
if (access(fname.toStdString().c_str(), F_OK) ==-1)
{
hfile = H5Fcreate(
fname.toStdString().c_str(),
H5F_ACC_EXCL ,
H5P_DEFAULT,
H5P_DEFAULT ) ;
topgroup = H5Gcreate(
hfile,
"iqdata_raw",
H5P_DEFAULT,
H5P_DEFAULT,
H5P_DEFAULT);
hid_t aid3 = H5Screate(H5S_SCALAR);
hid_t atype = H5Tcopy(H5T_C_S1);
H5Tset_size(atype, 4);
hid_t attr3 = H5Acreate1(topgroup, "type", atype, aid3, H5P_DEFAULT);
status = H5Awrite(attr3, atype,"dict");
H5Aclose(attr3);
H5Tclose(atype);
H5Sclose(aid3);
}
else
{
//printf("File already exists, will append\n");
hfile = H5Fopen(
fname.toStdString().c_str(),
H5F_ACC_RDWR,
H5P_DEFAULT );
topgroup = H5Gopen(hfile, "iqdata_raw", H5P_DEFAULT);
//fprintf(stderr,"Bad Hdf file already existant, cannot open\n");
}
if (hfile <=0 || topgroup <=0)
{
fprintf(stderr,"Bad Hdf file, cannot open\n");
return;
}
if (true)
{
//QHash<int,QHash<QString,QList<float> > > events;
QList<int> keys1=(*events).keys();
for (int i=0;i<keys1.length();i++)
{
int chan = keys1[i];
if ((*events)[chan]["bin"].length() > 0)
{
int bin = (int)((*events)[chan]["bin"][0]);
QString dirname = QString("keyint_%1").arg(chan);
//turn off errors when we query the group, using open
hid_t error_stack = H5Eget_current_stack();
H5E_auto2_t oldfunc;
void *old_client_data;
H5Eget_auto(error_stack, &oldfunc, &old_client_data);
H5Eset_auto(error_stack, NULL, NULL);
channelgroup = H5Gopen(
topgroup,
dirname.toStdString().c_str(),
H5P_DEFAULT);
//turn errors back on.
H5Eset_auto(error_stack, oldfunc, old_client_data);
if (channelgroup<0)
{
channelgroup = H5Gcreate(
topgroup,
dirname.toStdString().c_str(),
H5P_DEFAULT,
H5P_DEFAULT,
H5P_DEFAULT);
hid_t aid3 = H5Screate(H5S_SCALAR);
hid_t atype = H5Tcopy(H5T_C_S1);
H5Tset_size(atype, 4);
hid_t attr3 = H5Acreate1(channelgroup, "type", atype, aid3, H5P_DEFAULT);
status = H5Awrite(attr3, atype,"dict");
H5Aclose(attr3);
H5Tclose(atype);
H5Sclose(aid3);
}
if (channelgroup!=0)
{
QList<QString> keys2=(*events)[keys1[i]].keys();
for (int i2=0;i2<keys2.length();i2++)
{
QString dname = QString("keystr_%1").arg(keys2[i2]);
datasetlength = (*events)[keys1[i]][keys2[i2]].length();
if (datasetlength>0)
{
//Try to open dataset if it exists
//turn off errors when we query the group, using open
hid_t error_stack = H5Eget_current_stack();
H5E_auto2_t oldfunc;
void *old_client_data;
H5Eget_auto(error_stack, &oldfunc, &old_client_data);
H5Eset_auto(error_stack, NULL, NULL);
//query or open dataset
curdataset = H5Dopen(channelgroup,dname.toStdString().c_str(),H5P_DEFAULT);
//turn errors back on.
H5Eset_auto(error_stack, oldfunc, old_client_data);
//if cannot open dataset, create it, and make it chunked.
if (curdataset<=0)
{
//set up size info, chunks etc...
maxdims[0]=H5S_UNLIMITED;
rank = 1;
d_dims[0]=datasetlength;
curdataspace= H5Screate_simple(rank, d_dims,maxdims);
prop = H5Pcreate(H5P_DATASET_CREATE);
status = H5Pset_chunk(prop, rank, d_dims);
if (status) trap();
curdataset = H5Dcreate(
channelgroup,
dname.toStdString().c_str(),
H5T_NATIVE_FLOAT,
curdataspace,
H5P_DEFAULT,
prop,
H5P_DEFAULT);
hid_t aid3 = H5Screate(H5S_SCALAR);
hid_t atype = H5Tcopy(H5T_C_S1);
H5Tset_size(atype, 5);
hid_t attr3 = H5Acreate1(curdataset, "type", atype, aid3, H5P_DEFAULT);
status = H5Awrite(attr3, atype,"array");
H5Aclose(attr3);
H5Tclose(atype);
H5Sclose(aid3);
H5Pclose(prop);
}//if (curdataset<=0)
else
{
//get dataspace from exant dataset
curdataspace = H5Dget_space(curdataset);
H5Sget_simple_extent_dims(curdataspace, d_dims, maxdims);
H5Sclose(curdataspace);
//if dataset already exist, extend its size
d_dims[0] = d_dims[0] + datasetlength;
status = H5Dset_extent(curdataset, d_dims);
if (status) trap();
curdataspace = H5Dget_space(curdataset);
H5Sget_simple_extent_dims(curdataspace, d_dims, maxdims);
}//else
if (curdataset>0)
{
//
// Set up stride, offset, count block in dastasets...
//
// stride[0]=1;//hop size between floats in dataset...
//block[0]=1;//get 1 slab
count[0]=datasetlength;//size of hyperslab or chunk of data
offset[0]=d_dims[0] - datasetlength;//offset from beginning of file.
status = H5Sselect_hyperslab(
curdataspace,
H5S_SELECT_SET,
offset,
NULL,
count,
NULL);
if (status) trap();
rank = 1;
memdims[0]=datasetlength;
memdataspace= H5Screate_simple(rank, (const hsize_t*)memdims,NULL);
mcount[0] = datasetlength;
moffset[0] = 0;
status = H5Sselect_hyperslab(
memdataspace,
H5S_SELECT_SET,
moffset,
NULL,
mcount,
NULL);
if (status) trap();
for (int mmm=0;mmm<datasetlength;mmm++)
this->rawdata[mmm]=(*events)[keys1[i]][keys2[i2]][mmm];
status = H5Dwrite(
curdataset,
H5T_NATIVE_FLOAT,
memdataspace,
curdataspace,
H5P_DEFAULT,
this->rawdata);
if (status) trap();
H5Sclose(memdataspace);
H5Sclose(curdataspace);
H5Dclose(curdataset);
} //if (curdataset>0)
}
}//for (int i2=0;i2<keys2.length();i2++)
status = H5Gclose(channelgroup);
}//if (channelgroup!=0)
}//if ((*events)[chan]["bin"].length() > 0)
}//for (int i=0;i<keys1.length();i++)
}//if (true)
status = H5Gclose(topgroup);
status = H5Fclose(hfile);
}
