DataSet *load_fMRI3D_DS(std::string filename) {
std::cout << "Opening file " << filename << std::endl;
typedef DataSet DBN_DS;
using namespace H5;
using H5::DataSet;
const H5std_string FILE_NAME(filename);
H5File file(FILE_NAME, H5F_ACC_RDONLY);
int dims[4];
DataSet datadims = file.openDataSet("dims");
datadims.read(dims, PredType::NATIVE_INT);
datadims.close();
DataSet data = file.openDataSet("data");
DataSpace dsp = data.getSpace();
hsize_t dims_out[2];
dsp.getSimpleExtentDims(dims_out, NULL);
int volumes = (int)dims_out[0];
int voxels = (int)dims_out[1];
std::cout << "Dataset images are of size " << dims[0] << "x" << dims[1] << "x" << dims[2] << "x" << dims[3] << std::endl;
std::cout << "Training set is of size " << volumes << "x" << voxels << std::endl;
DBN_DS *dataset = new DBN_DS(AOD, volumes, dims[0], dims[1], dims[2], voxels);
dataset->data_path = filename;
std::cout << "Reading dataset" << std::endl;
float *out_buffer = new float[voxels*volumes];
data.read(out_buffer, PredType::NATIVE_FLOAT);
for (int volume = 0; volume < volumes; ++volume) {
for (int voxel = 0; voxel < voxels; ++voxel) {
float val = out_buffer[volumes*voxel + volume];
dataset->data(volume,voxel) = val;
}
}
data.close();
dsp.close();
#if 0
Vector col = transpose(dataset->data)(9);
col.save("/Users/devon/Research/matlab/tocmat2");
exit(1);
#endif
std::cout << "Reading mask" << std::endl;
DataSet mask_data = file.openDataSet("mask");
dsp = mask_data.getSpace();
mask_data.read(dataset->mask.m->data, PredType::NATIVE_FLOAT);
mask_data.close();
std::cout << "Done reading mask" << std::endl;
dataset->applymask = true;
std::cout << "Done loading dataset" << std::endl;
return dataset;
}
HDF5HandlerBase::HDF5HandlerBase(const std::string &fileName, const std::string &datasetName)
: FILE_NAME(H5std_string(fileName))
, DATASETNAME(H5std_string(datasetName))
{
try
{
Exception::dontPrint();
file = H5File(FILE_NAME, H5F_ACC_TRUNC);
hsize_t dims[1] = {0};
hsize_t maxdims[1] = {H5S_UNLIMITED};
hsize_t chunk_dims[1] = {10000};
DataSpace dataspace = DataSpace(1,dims,maxdims);
DSetCreatPropList prop;
prop.setChunk(1, chunk_dims);
dataset = file.createDataSet( DATASETNAME,
PredType::STD_I32BE, dataspace, prop);
prop.close();
dataspace.close();
} catch (Exception &error) {
// Throw FileIException, DataSetIException, DataSpaceIException
throw;
}
}
void HDF5HandlerBase::save(const std::vector<int> &dataPoints) {
// Return if no data to add
if (dataPoints.size() < 1)
return;
// dataset.write needs not const value of data
int *data = const_cast<int*>(&dataPoints[0]);
// Determine value of
hsize_t dimsext[1];
dimsext[0] = dataPoints.size();
hsize_t size[1];
hsize_t offset[1];
try {
DataSpace filespace = dataset.getSpace();
int ndims = filespace.getSimpleExtentNdims();
hsize_t dims[ndims];
filespace.getSimpleExtentDims(dims);
size[0] = dims[0] + dimsext[0];
dataset.extend(size);
offset[0] = dims[0];
filespace = dataset.getSpace();
filespace.selectHyperslab(H5S_SELECT_SET, dimsext, offset);
DataSpace memspace = DataSpace(1, dimsext, NULL);
dataset.write(data, PredType::NATIVE_INT, memspace, filespace);
filespace.close();
memspace.close();
} catch (Exception &error) {
throw;
}
}
template<class T> Matrix<T>
Read (const std::string& uri) const {
T t;
DataSet dataset = m_file.openDataSet(uri);
DataSpace space = dataset.getSpace();
std::vector<hsize_t> dims (space.getSimpleExtentNdims());
size_t ndim = space.getSimpleExtentDims(&dims[0], NULL);
if (this->m_verb) {
printf ("Reading dataset %s ... ", uri.c_str());
fflush(stdout);
}
if (is_complex(t)) {
dims.pop_back();
--ndim;
}
std::vector<size_t> mdims (ndim,1);
for (size_t i = 0; i < ndim; ++i)
mdims[i] = dims[ndim-i-1];
PredType* type = HDF5Traits<T>::PType();
Matrix<T> M (mdims);
dataset.read (&M[0], *type);
if (this->m_verb)
printf ("O(%s) done\n", DimsToCString(M));
space.close();
dataset.close();
return M;
}
static void test_vl_rewrite()
{
// Output message about test being performed
SUBTEST("I/O on VL strings with link/unlink");
try {
// Create the files.
H5File file1(FILENAME, H5F_ACC_TRUNC);
H5File file2(FILENAME2, H5F_ACC_TRUNC);
// Create the VL string datatype.
StrType type(0, H5T_VARIABLE);
// Create dataspace for the attribute.
DataSpace space (H5S_SCALAR);
// Create in file 1.
int i;
char name[256]; // Buffer for names & data
for (i=0; i<REWRITE_NDATASETS; i++) {
sprintf(name, "/set_%d", i);
write_scalar_dset(file1, type, space, name, name);
}
// Effectively copy data from file 1 to 2.
for (i=0; i<REWRITE_NDATASETS; i++) {
sprintf(name, "/set_%d", i);
read_scalar_dset(file1, type, space, name, name);
write_scalar_dset(file2, type, space, name, name);
}
// Read back from file 2.
for (i=0; i<REWRITE_NDATASETS; i++) {
sprintf(name, "/set_%d", i);
read_scalar_dset(file2, type, space, name, name);
}
// Remove from file 2.
for (i=0; i<REWRITE_NDATASETS; i++) {
sprintf(name, "/set_%d", i);
file2.unlink(name);
}
// Effectively copy from file 1 to file 2.
for (i=0; i<REWRITE_NDATASETS; i++) {
sprintf(name, "/set_%d", i);
read_scalar_dset(file1, type, space, name, name);
write_scalar_dset(file2, type, space, name, name);
}
// Close objects and file.
type.close();
space.close();
file1.close();
file2.close();
PASSED();
} // end try
// Catch all exceptions.
catch (Exception E) {
issue_fail_msg("test_vl_rewrite()", __LINE__, __FILE__, E.getCDetailMsg());
}
} // end test_vl_rewrite()
/****************************************************************
**
** test_attr_mult_read(): Test reading multiple attributes.
**
****************************************************************/
static void test_attr_mult_read()
{
int read_data1[ATTR1_DIM1]={0}; // Buffer for reading 1st attribute
int read_data2[ATTR2_DIM1][ATTR2_DIM2]={{0}}; // Buffer for reading 2nd attribute
double read_data3[ATTR3_DIM1][ATTR3_DIM2][ATTR3_DIM3]={{{0}}}; // Buffer for reading 3rd attribute
int i,j,k;
// Output message about test being performed
SUBTEST("Multiple Attribute Reading Functions");
try {
// Open file
H5File fid1(FILENAME, H5F_ACC_RDWR);
// Open the dataset
DataSet dataset = fid1.openDataSet(DSET1_NAME);
// Verify the correct number of attributes
int num_attrs = dataset.getNumAttrs();
verify_val(num_attrs, 3, "H5Object::getNumAttrs", __LINE__, __FILE__);
// Open 1st attribute for the dataset
Attribute attr = dataset.openAttribute((unsigned)0);
/* Verify Dataspace */
// Get the dataspace of the attribute
DataSpace space = attr.getSpace();
// Get the rank of the dataspace and verify it
int rank = space.getSimpleExtentNdims();
verify_val(rank, ATTR1_RANK, "DataSpace::getSimpleExtentNdims", __LINE__, __FILE__);
// Get the dims of the dataspace and verify them
hsize_t dims[ATTR_MAX_DIMS]; // Attribute dimensions
int ndims = space.getSimpleExtentDims(dims);
if(dims[0]!=ATTR1_DIM1)
TestErrPrintf("%d:attribute dimensions different: dims[0]=%d, should be %d\n",__LINE__,(int)dims[0],ATTR1_DIM1);
/* Verify Datatype */
// Get the class of the datatype that is used by attr
H5T_class_t type_class = attr.getTypeClass();
// Verify that the type is of integer datatype
verify_val(type_class, H5T_INTEGER, "Attribute::getTypeClass", __LINE__, __FILE__);
// Get the integer datatype
IntType i_type1 = attr.getIntType();
// Get and verify the order of this type
H5T_order_t order = i_type1.getOrder();
verify_val(order, PredType::NATIVE_INT.getOrder(), "DataType::getOrder", __LINE__, __FILE__);
// Get and verify the size of this type
size_t size = i_type1.getSize();
verify_val(size, PredType::NATIVE_INT.getSize(), "DataType::getSize", __LINE__, __FILE__);
// Read attribute information
attr.read(PredType::NATIVE_INT, read_data1);
// Verify values read in
for(i=0; i<ATTR1_DIM1; i++)
if(attr_data1[i]!=read_data1[i])
TestErrPrintf("%d: attribute data different: attr_data1[%d]=%d,read_data1[%d]=%d\n",__LINE__,i,attr_data1[i],i,read_data1[i]);
// Verify Name
H5std_string attr_name = attr.getName();
verify_val(attr_name, ATTR1_NAME, "DataType::getName", __LINE__, __FILE__);
attr.close();
space.close();
// Open 2nd attribute for the dataset
attr = dataset.openAttribute((unsigned)1);
/* Verify Dataspace */
// Get the dataspace of the attribute
space = attr.getSpace();
// Get the rank of the dataspace and verify it
rank = space.getSimpleExtentNdims();
verify_val(rank, ATTR2_RANK, "DataSpace::getSimpleExtentNdims", __LINE__, __FILE__);
// Get the dims of the dataspace and verify them
ndims = space.getSimpleExtentDims(dims);
if(dims[0]!=ATTR2_DIM1)
TestErrPrintf("%d:attribute dimensions different: dims[0]=%d, should be %d\n",__LINE__,(int)dims[0],ATTR2_DIM1);
if(dims[1]!=ATTR2_DIM2)
TestErrPrintf("%d:attribute dimensions different: dims[1]=%d, should be %d\n",__LINE__,(int)dims[1],ATTR2_DIM2);
/* Verify Datatype */
// Get the class of the datatype that is used by attr
type_class = attr.getTypeClass();
// Verify that the type is of integer datatype
verify_val(type_class, H5T_INTEGER, "Attribute::getTypeClass", __LINE__, __FILE__);
// Get the integer datatype
IntType i_type2 = attr.getIntType();
// Get and verify the order of this type
order = i_type2.getOrder();
verify_val(order, PredType::NATIVE_INT.getOrder(), "DataType::getOrder", __LINE__, __FILE__);
// Get and verify the size of this type
size = i_type2.getSize();
verify_val(size, PredType::NATIVE_INT.getSize(), "DataType::getSize", __LINE__, __FILE__);
// Read attribute information
attr.read(PredType::NATIVE_INT, read_data2);
//attr.read(i_type, read_data2);
// Verify values read in
for(i=0; i<ATTR2_DIM1; i++)
for(j=0; j<ATTR2_DIM2; j++)
if(attr_data2[i][j]!=read_data2[i][j])
TestErrPrintf("%d: attribute data different: attr_data2[%d][%d]=%d, read_data2[%d][%d]=%d\n",__LINE__,i,j,attr_data2[i][j],i,j,read_data2[i][j]);
// Verify Name
attr_name = attr.getName();
verify_val(attr_name, ATTR2_NAME, "DataType::getName", __LINE__, __FILE__);
attr.close();
space.close();
// Open 3rd attribute for the dataset
attr = dataset.openAttribute((unsigned)2);
/* Verify Dataspace */
// Get the dataspace of the attribute
space = attr.getSpace();
// Get the rank of the dataspace and verify it
rank = space.getSimpleExtentNdims();
verify_val(rank, ATTR3_RANK, "DataSpace::getSimpleExtentNdims", __LINE__, __FILE__);
// Get the dims of the dataspace and verify them
ndims = space.getSimpleExtentDims(dims);
verify_val((long)dims[0],(long)ATTR3_DIM1,"attribute dimensions",__FILE__,__LINE__);
verify_val((long)dims[1],(long)ATTR3_DIM2,"attribute dimensions",__FILE__,__LINE__);
verify_val((long)dims[2],(long)ATTR3_DIM3,"attribute dimensions",__FILE__,__LINE__);
/* Verify Datatype */
// Get the class of the datatype that is used by attr
type_class = attr.getTypeClass();
// Verify that the type is of compound datatype
verify_val(type_class, H5T_FLOAT, "Attribute::getTypeClass", __LINE__, __FILE__);
// Get the double datatype
FloatType f_type = attr.getFloatType();
// Get and verify the order of this type
order = f_type.getOrder();
verify_val(order, PredType::NATIVE_DOUBLE.getOrder(), "DataType::getOrder", __LINE__, __FILE__);
// Get and verify the size of this type
size = f_type.getSize();
verify_val(size, PredType::NATIVE_DOUBLE.getSize(), "DataType::getSize", __LINE__, __FILE__);
// Read attribute information
attr.read(PredType::NATIVE_DOUBLE, read_data3);
// Verify values read in
for(i=0; i<ATTR3_DIM1; i++)
for(j=0; j<ATTR3_DIM2; j++)
for(k=0; k<ATTR3_DIM3; k++)
if(attr_data3[i][j][k]!=read_data3[i][j][k])
TestErrPrintf("%d: attribute data different: attr_data3[%d][%d][%d]=%f, read_data3[%d][%d][%d]=%f\n",__LINE__,i,j,k,attr_data3[i][j][k],i,j,k,read_data3[i][j][k]);
// Verify Name
attr_name = attr.getName();
verify_val(attr_name, ATTR3_NAME, "DataType::getName", __LINE__, __FILE__);
PASSED();
} // end try block
catch (Exception E) {
issue_fail_msg("test_attr_mult_read()", __LINE__, __FILE__, E.getCDetailMsg());
}
} // test_attr_mult_read()
/****************************************************************
**
** test_attr_dtype_shared(): Test code for using shared datatypes
** in attributes.
**
****************************************************************/
static void test_attr_dtype_shared()
{
int data=8; /* Data to write */
int rdata=0; /* Read read in */
#ifndef H5_NO_DEPRECATED_SYMBOLS
H5G_stat_t statbuf; /* Object's information */
#endif /* H5_NO_DEPRECATED_SYMBOLS */
h5_stat_size_t filesize; /* Size of file after modifications */
// Output message about test being performed
SUBTEST("Shared Datatypes with Attributes");
try {
// Create a file
H5File fid1(FILENAME, H5F_ACC_TRUNC);
// Close file
fid1.close();
// Get size of file
h5_stat_size_t empty_filesize; // Size of empty file
empty_filesize = h5_get_file_size(FILENAME.c_str(), H5P_DEFAULT);
if (empty_filesize < 0)
TestErrPrintf("Line %d: file size wrong!\n", __LINE__);
// Open the file again
fid1.openFile(FILENAME, H5F_ACC_RDWR);
// Enclosing to work around the issue of unused variables and/or
// objects created by copy constructors stay around until end of
// scope, causing incorrect number of ref counts.
{ // First enclosed block
// Create a datatype to commit and use
IntType dtype(PredType::NATIVE_INT);
// Commit datatype to file
dtype.commit(fid1, TYPE1_NAME);
#ifndef H5_NO_DEPRECATED_SYMBOLS
// Check reference count on named datatype
fid1.getObjinfo(TYPE1_NAME, statbuf);
verify_val((int)statbuf.nlink, 1, "DataType::getObjinfo", __LINE__, __FILE__);
#endif /* H5_NO_DEPRECATED_SYMBOLS */
// Create dataspace for dataset
DataSpace dspace;
DataSet dset = fid1.createDataSet(DSET1_NAME, dtype, dspace);
#ifndef H5_NO_DEPRECATED_SYMBOLS
// Check reference count on named datatype
fid1.getObjinfo(TYPE1_NAME, statbuf);
verify_val((int)statbuf.nlink, 2, "H5File::getObjinfo", __LINE__, __FILE__);
#endif /* H5_NO_DEPRECATED_SYMBOLS */
// Create attribute on dataset
Attribute attr = dset.createAttribute(ATTR1_NAME,dtype,dspace);
#ifndef H5_NO_DEPRECATED_SYMBOLS
// Check reference count on named datatype
fid1.getObjinfo(TYPE1_NAME, statbuf);
verify_val((int)statbuf.nlink, 3, "DataSet::getObjinfo", __LINE__, __FILE__);
#endif /* H5_NO_DEPRECATED_SYMBOLS */
// Close attribute
attr.close();
// Delete attribute
dset.removeAttr(ATTR1_NAME);
#ifndef H5_NO_DEPRECATED_SYMBOLS
// Check reference count on named datatype
fid1.getObjinfo(TYPE1_NAME, statbuf);
verify_val((int)statbuf.nlink, 2, "DataSet::getObjinfo after DataSet::removeAttr", __LINE__, __FILE__);
#endif /* H5_NO_DEPRECATED_SYMBOLS */
// Create attribute on dataset
attr = dset.createAttribute(ATTR1_NAME,dtype,dspace);
#ifndef H5_NO_DEPRECATED_SYMBOLS
// Check reference count on named datatype
fid1.getObjinfo(TYPE1_NAME, statbuf);
verify_val((int)statbuf.nlink, 3, "DataSet::createAttribute", __LINE__, __FILE__);
#endif /* H5_NO_DEPRECATED_SYMBOLS */
// Write data into the attribute
attr.write(PredType::NATIVE_INT,&data);
// Close attribute, dataset, dataspace, datatype, and file
attr.close();
dset.close();
dspace.close();
dtype.close();
} // end of first enclosing
fid1.close();
// Open the file again
fid1.openFile(FILENAME, H5F_ACC_RDWR);
{ // Second enclosed block...
// Open dataset
DataSet *dset2 = new DataSet (fid1.openDataSet(DSET1_NAME));
// Open attribute
Attribute *attr2 = new Attribute (dset2->openAttribute(ATTR1_NAME));
// Read data from the attribute
attr2->read(PredType::NATIVE_INT, &rdata);
verify_val(data, rdata, "Attribute::read", __LINE__, __FILE__);
// Close attribute and dataset
delete attr2;
delete dset2;
#ifndef H5_NO_DEPRECATED_SYMBOLS
// Check reference count on named datatype
fid1.getObjinfo(TYPE1_NAME, statbuf);
verify_val((int)statbuf.nlink, 3, "DataSet::openAttribute", __LINE__, __FILE__);
#endif /* H5_NO_DEPRECATED_SYMBOLS */
} // end of second enclosing
// Unlink the dataset
fid1.unlink(DSET1_NAME);
#ifndef H5_NO_DEPRECATED_SYMBOLS
// Check reference count on named datatype
fid1.getObjinfo(TYPE1_NAME, statbuf);
verify_val((int)statbuf.nlink, 1, "H5File::unlink", __LINE__, __FILE__);
#endif /* H5_NO_DEPRECATED_SYMBOLS */
// Unlink the named datatype
fid1.unlink(TYPE1_NAME);
// Close file
fid1.close();
// Check size of file
filesize = h5_get_file_size(FILENAME.c_str(), H5P_DEFAULT);
verify_val((long)filesize, (long)empty_filesize, "Checking file size", __LINE__, __FILE__);
PASSED();
} // end try block
catch (Exception E) {
issue_fail_msg("test_attr_dtype_shared()", __LINE__, __FILE__, E.getCDetailMsg());
}
} // test_attr_dtype_shared()
template<class T> bool
Write (const Matrix<T>& M, const std::string& uri) {
T t;
Group group, *tmp;
std::string path;
boost::tokenizer<> tok(uri);
std::vector<std::string> sv (Split (uri, "/"));
std::string name = sv[sv.size() - 1];
sv.pop_back(); // data name not part of path
if (sv.size() == 0)
path = "/";
else
for (size_t i = 0; i < sv.size(); i++) {
if (sv[i].compare(""))
path += "/";
path += sv[i];
}
if (this->m_verb)
printf ("Creating dataset %s at path (%s)\n", name.c_str(), path.c_str());
try {
group = m_file.openGroup(path);
if (this->m_verb)
printf ("Group %s opened for writing\n", path.c_str()) ;
} catch (const Exception& e) {
for (size_t i = 0, depth = 0; i < sv.size(); i++) {
if (sv[i].compare("")) {
try {
group = (depth) ? (*tmp).openGroup(sv[i]) : m_file.openGroup(sv[i]);
} catch (const Exception& e) {
group = (depth) ? (*tmp).createGroup(sv[i]) : m_file.createGroup(sv[i]);
}
tmp = &group;
depth++;
}
}
}
// One more field for complex numbers
size_t tmpdim = ndims(M);
std::vector<hsize_t> dims (tmpdim);
for (size_t i = 0; i < tmpdim; i++)
dims[i] = M.Dim(tmpdim-1-i);
if (is_complex(t)) {
dims.push_back(2);
tmpdim++;
}
DataSpace space (tmpdim, &dims[0]);
PredType* type = HDF5Traits<T>::PType();
DataSet set = group.createDataSet(name, (*type), space);
set.write (M.Ptr(), (*type));
set.close ();
space.close ();
return true;
}
