int
main (void)
{
hid_t vfile, file, src_space, mem_space, vspace,
vdset, dset; /* Handles */
hid_t dcpl;
herr_t status;
hsize_t vdsdims[3] = {4*DIM0_1, VDSDIM1, VDSDIM2},
vdsdims_max[3] = {VDSDIM0, VDSDIM1, VDSDIM2},
dims[3] = {DIM0_1, DIM1, DIM2},
extdims[3] = {2*DIM0_1, DIM1, DIM2},
chunk_dims[3] = {DIM0_1, DIM1, DIM2},
dims_max[3] = {DIM0, DIM1, DIM2},
vdsdims_out[3],
vdsdims_max_out[3],
start[3], /* Hyperslab parameters */
stride[3],
count[3],
src_count[3],
block[3];
hsize_t start_out[3], /* Hyperslab parameter out */
stride_out[3],
count_out[3],
block_out[3];
int i, j, k;
H5D_layout_t layout; /* Storage layout */
size_t num_map; /* Number of mappings */
ssize_t len; /* Length of the string; also a return value */
char *filename;
char *dsetname;
int wdata[DIM0_1*DIM1*DIM2];
int rdata[80][10][10];
int a_rdata[20][10][10];
/*
* Create source files and datasets. This step is optional.
*/
for (i=0; i < PLANE_STRIDE; i++) {
/*
* Initialize data for i-th source dataset.
*/
for (j = 0; j < DIM0_1*DIM1*DIM2; j++) wdata[j] = i+1;
/*
* Create the source files and datasets. Write data to each dataset and
* close all resources.
*/
file = H5Fcreate (SRC_FILE[i], H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
src_space = H5Screate_simple (RANK, dims, dims_max);
dcpl = H5Pcreate(H5P_DATASET_CREATE);
status = H5Pset_chunk (dcpl, RANK, chunk_dims);
dset = H5Dcreate2 (file, SRC_DATASET[i], H5T_NATIVE_INT, src_space, H5P_DEFAULT,
dcpl, H5P_DEFAULT);
status = H5Dwrite (dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT,
wdata);
status = H5Sclose (src_space);
status = H5Pclose (dcpl);
status = H5Dclose (dset);
status = H5Fclose (file);
}
vfile = H5Fcreate (VFILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
/* Create VDS dataspace. */
vspace = H5Screate_simple (RANK, vdsdims, vdsdims_max);
/* Create dataspaces for the source dataset. */
src_space = H5Screate_simple (RANK, dims, dims_max);
/* Create VDS creation property */
dcpl = H5Pcreate (H5P_DATASET_CREATE);
/* Initialize hyperslab values */
start[0] = 0;
start[1] = 0;
start[2] = 0;
stride[0] = PLANE_STRIDE; /* we will select every fifth plane in VDS */
stride[1] = 1;
stride[2] = 1;
count[0] = H5S_UNLIMITED;
count[1] = 1;
count[2] = 1;
src_count[0] = H5S_UNLIMITED;
src_count[1] = 1;
src_count[2] = 1;
block[0] = 1;
block[1] = DIM1;
block[2] = DIM2;
/*
* Build the mappings
*
*/
status = H5Sselect_hyperslab (src_space, H5S_SELECT_SET, start, NULL, src_count, block);
for (i=0; i < PLANE_STRIDE; i++) {
status = H5Sselect_hyperslab (vspace, H5S_SELECT_SET, start, stride, count, block);
status = H5Pset_virtual (dcpl, vspace, SRC_FILE[i], SRC_DATASET[i], src_space);
start[0]++;
}
H5Sselect_none(vspace);
/* Create a virtual dataset */
vdset = H5Dcreate2 (vfile, DATASET, H5T_NATIVE_INT, vspace, H5P_DEFAULT,
dcpl, H5P_DEFAULT);
status = H5Sclose (vspace);
status = H5Sclose (src_space);
status = H5Pclose (dcpl);
/* Let's get space of the VDS and its dimension; we should get 40x10x10 */
vspace = H5Dget_space (vdset);
H5Sget_simple_extent_dims (vspace, vdsdims_out, vdsdims_max_out);
printf ("VDS dimensions first time \n");
printf (" Current: ");
for (i=0; i<RANK; i++)
printf (" %d ", (int)vdsdims_out[i]);
printf ("\n");
/* Let's add data to the source datasets and check new dimensions for VDS */
for (i=0; i < PLANE_STRIDE; i++) {
/*
* Initialize data for i-th source dataset.
*/
for (j = 0; j < DIM0_1*DIM1*DIM2; j++) wdata[j] = 10*(i+1);
/*
* Create the source files and datasets. Write data to each dataset and
* close all resources.
*/
file = H5Fopen (SRC_FILE[i], H5F_ACC_RDWR, H5P_DEFAULT);
dset = H5Dopen2 (file, SRC_DATASET[i], H5P_DEFAULT);
status = H5Dset_extent (dset, extdims);
src_space = H5Dget_space (dset);
start[0] = DIM0_1;
start[1] = 0;
start[2] = 0;
count[0] = 1;
count[1] = 1;
count[2] = 1;
block[0] = DIM0_1;
block[1] = DIM1;
block[2] = DIM2;
mem_space = H5Screate_simple(RANK, dims, NULL);
status = H5Sselect_hyperslab (src_space, H5S_SELECT_SET, start, NULL, count, block);
status = H5Dwrite (dset, H5T_NATIVE_INT, mem_space, src_space, H5P_DEFAULT,
wdata);
status = H5Sclose (src_space);
status = H5Dclose (dset);
status = H5Fclose (file);
}
status = H5Dclose (vdset);
status = H5Fclose (vfile);
/*
* Now we begin the read section of this example.
*/
/*
* Open file and dataset using the default properties.
*/
vfile = H5Fopen (VFILE, H5F_ACC_RDONLY, H5P_DEFAULT);
vdset = H5Dopen2 (vfile, DATASET, H5P_DEFAULT);
/*
* Get creation property list and mapping properties.
*/
dcpl = H5Dget_create_plist (vdset);
/*
* Get storage layout.
*/
layout = H5Pget_layout (dcpl);
if (H5D_VIRTUAL == layout)
printf(" Dataset has a virtual layout \n");
else
printf(" Wrong layout found \n");
/*
* Find number of mappings.
*/
status = H5Pget_virtual_count (dcpl, &num_map);
printf(" Number of mappings is %lu\n", (unsigned long)num_map);
/*
* Get mapping parameters for each mapping.
*/
for (i = 0; i < (int)num_map; i++) {
printf(" Mapping %d \n", i);
printf(" Selection in the virtual dataset \n");
/* Get selection in the virttual dataset */
vspace = H5Pget_virtual_vspace (dcpl, (size_t)i);
if (H5Sget_select_type(vspace) == H5S_SEL_HYPERSLABS) {
if (H5Sis_regular_hyperslab(vspace)) {
status = H5Sget_regular_hyperslab (vspace, start_out, stride_out, count_out, block_out);
printf(" start = [%llu, %llu, %llu] \n", (unsigned long long)start_out[0], (unsigned long long)start_out[1], (unsigned long long)start_out[2]);
printf(" stride = [%llu, %llu, %llu] \n", (unsigned long long)stride_out[0], (unsigned long long)stride_out[1], (unsigned long long)stride_out[2]);
printf(" count = [%llu, %llu, %llu] \n", (unsigned long long)count_out[0], (unsigned long long)count_out[1], (unsigned long long)count_out[2]);
printf(" block = [%llu, %llu, %llu] \n", (unsigned long long)block_out[0], (unsigned long long)block_out[1], (unsigned long long)block_out[2]);
}
}
/* Get source file name */
len = H5Pget_virtual_filename (dcpl, (size_t)i, NULL, 0);
filename = (char *)malloc((size_t)len*sizeof(char)+1);
H5Pget_virtual_filename (dcpl, (size_t)i, filename, len+1);
printf(" Source filename %s\n", filename);
/* Get source dataset name */
len = H5Pget_virtual_dsetname (dcpl, (size_t)i, NULL, 0);
dsetname = (char *)malloc((size_t)len*sizeof(char)+1);
H5Pget_virtual_dsetname (dcpl, (size_t)i, dsetname, len+1);
printf(" Source dataset name %s\n", dsetname);
/* Get selection in the source dataset */
printf(" Selection in the source dataset \n");
src_space = H5Pget_virtual_srcspace (dcpl, (size_t)i);
if (H5Sget_select_type(src_space) == H5S_SEL_HYPERSLABS) {
if (H5Sis_regular_hyperslab(src_space)) {
status = H5Sget_regular_hyperslab (src_space, start_out, stride_out, count_out, block_out);
printf(" start = [%llu, %llu, %llu] \n", (unsigned long long)start_out[0], (unsigned long long)start_out[1], (unsigned long long)start_out[2]);
printf(" stride = [%llu, %llu, %llu] \n", (unsigned long long)stride_out[0], (unsigned long long)stride_out[1], (unsigned long long)stride_out[2]);
printf(" count = [%llu, %llu, %llu] \n", (unsigned long long)count_out[0], (unsigned long long)count_out[1], (unsigned long long)count_out[2]);
printf(" block = [%llu, %llu, %llu] \n", (unsigned long long)block_out[0], (unsigned long long)block_out[1], (unsigned long long)block_out[2]);
}
}
H5Sclose(vspace);
H5Sclose(src_space);
free(filename);
free(dsetname);
}
/*
* Read data from VDS.
*/
vspace = H5Dget_space (vdset);
H5Sget_simple_extent_dims (vspace, vdsdims_out, vdsdims_max_out);
printf ("VDS dimensions second time \n");
printf (" Current: ");
for (i=0; i<RANK; i++)
printf (" %d ", (int)vdsdims_out[i]);
printf ("\n");
/* Read all VDS data */
/* EIP We should be able to do it by using H5S_ALL instead of making selection
* or using H5Sselect_all from vspace.
*/
start[0] = 0;
start[1] = 0;
start[2] = 0;
count[0] = 1;
count[1] = 1;
count[2] = 1;
block[0] = vdsdims_out[0];
block[1] = vdsdims_out[1];
block[2] = vdsdims_out[2];
status = H5Sselect_hyperslab (vspace, H5S_SELECT_SET, start, NULL, count, block);
mem_space = H5Screate_simple(RANK, vdsdims_out, NULL);
status = H5Dread (vdset, H5T_NATIVE_INT, mem_space, vspace, H5P_DEFAULT,
rdata);
printf (" All data: \n");
for (i=0; i < (int)vdsdims_out[0]; i++) {
for (j=0; j < (int)vdsdims_out[1]; j++) {
printf ("(%d, %d, 0)", i, j);
for (k=0; k < (int)vdsdims_out[2]; k++)
printf (" %d ", rdata[i][j][k]);
printf ("\n");
}
}
/* Read VDS, but only data mapeed to dataset a.h5 */
start[0] = 0;
start[1] = 0;
start[2] = 0;
stride[0] = PLANE_STRIDE;
stride[1] = 1;
stride[2] = 1;
count[0] = 2*DIM0_1;
count[1] = 1;
count[2] = 1;
block[0] = 1;
block[1] = vdsdims_out[1];
block[2] = vdsdims_out[2];
dims[0] = 2*DIM0_1;
status = H5Sselect_hyperslab (vspace, H5S_SELECT_SET, start, stride, count, block);
mem_space = H5Screate_simple(RANK, dims, NULL);
status = H5Dread (vdset, H5T_NATIVE_INT, mem_space, vspace, H5P_DEFAULT,
a_rdata);
printf (" All data: \n");
for (i=0; i < 2*DIM0_1; i++) {
for (j=0; j < (int)vdsdims_out[1]; j++) {
printf ("(%d, %d, 0)", i, j);
for (k=0; k < (int)vdsdims_out[2]; k++)
printf (" %d ", a_rdata[i][j][k]);
printf ("\n");
}
}
/*
* Close and release resources.
*/
status = H5Sclose(mem_space);
status = H5Pclose (dcpl);
status = H5Dclose (vdset);
status = H5Fclose (vfile);
return 0;
}
static types::InternalType* import_struct(int dataset)
{
//get struct dims node
int complex = 0;
std::vector<int> pdims;
int size = getDimsNode(dataset, &complex, pdims);
types::Struct* str = new types::Struct(static_cast<int>(pdims.size()), pdims.data());
size = str->getSize();
if (size == 0)
{
//empty struct
closeList6(dataset);
delete str;
return new types::Struct();
}
types::SingleStruct** sstr = str->get();
int fieldCount = 0;
int ret = getListDims6(dataset, &fieldCount);
if (ret < 0)
{
closeList6(dataset);
return str;
}
//get fields name
int dfield = getDataSetIdFromName(dataset, "__fields__");
int dim = 0;
getDatasetInfo(dfield, &complex, &dim, NULL);
std::vector<int> d(dim);
size = getDatasetInfo(dfield, &complex, &dim, d.data());
if (size < 0)
{
closeList6(dataset);
delete str;
return nullptr;
}
//get dims value
std::vector<char*> fields(size);
readStringMatrix(dfield, fields.data());
//open __refs__ node
int refs = getDataSetIdFromName(dataset, "__refs__");
for (const auto & name : fields)
{
wchar_t* field = to_wide_string(name);
str->addField(field);
int dataref = getDataSetIdFromName(dataset, name);
if (dataref < 0)
{
closeList6(dataset);
freeStringMatrix(dfield, fields.data());
FREE(field);
delete str;
return nullptr;
}
int refdim = 0;
getDatasetInfo(dataref, &complex, &refdim, NULL);
std::vector<int> refdims(refdim);
int refcount = getDatasetInfo(dataref, &complex, &refdim, refdims.data());
std::vector<hobj_ref_t> vrefs(refcount);
ret = H5Dread(dataref, H5T_STD_REF_OBJ, H5S_ALL, H5S_ALL, H5P_DEFAULT, vrefs.data());
if (ret < 0)
{
freeStringMatrix(dfield, fields.data());
FREE(field);
delete str;
return nullptr;
}
//import field
for (int j = 0; j < refcount; ++j)
{
int data = H5Rdereference(refs, H5R_OBJECT, &vrefs[j]);
if (data < 0)
{
freeStringMatrix(dfield, fields.data());
FREE(field);
delete str;
return nullptr;
}
types::InternalType* val = import_data(data);
if (val == nullptr)
{
freeStringMatrix(dfield, fields.data());
FREE(field);
delete str;
return nullptr;
}
sstr[j]->set(field, val);
}
FREE(field);
closeDataSet(dataref);
}
freeStringMatrix(dfield, fields.data());
closeList6(refs);
closeList6(dataset);
return str;
}
CPLErr HDF5ImageDataset::CreateProjections()
{
switch(iSubdatasetType)
{
case CSK_PRODUCT:
{
const char *osMissionLevel = NULL;
int productType = PROD_UNKNOWN;
if(GetMetadataItem("Product_Type")!=NULL)
{
//Get the format's level
osMissionLevel = HDF5Dataset::GetMetadataItem("Product_Type");
if(EQUALN(osMissionLevel,"RAW",3))
productType = PROD_CSK_L0;
if(EQUALN(osMissionLevel,"SSC",3))
productType = PROD_CSK_L1A;
if(EQUALN(osMissionLevel,"DGM",3))
productType = PROD_CSK_L1B;
if(EQUALN(osMissionLevel,"GEC",3))
productType = PROD_CSK_L1C;
if(EQUALN(osMissionLevel,"GTC",3))
productType = PROD_CSK_L1D;
}
CaptureCSKGeoTransform(productType);
CaptureCSKGeolocation(productType);
CaptureCSKGCPs(productType);
break;
}
case UNKNOWN_PRODUCT:
{
#define NBGCPLAT 100
#define NBGCPLON 30
hid_t LatitudeDatasetID = -1;
hid_t LongitudeDatasetID = -1;
hid_t LatitudeDataspaceID;
hid_t LongitudeDataspaceID;
float* Latitude;
float* Longitude;
int i,j;
int nDeltaLat;
int nDeltaLon;
nDeltaLat = nRasterYSize / NBGCPLAT;
nDeltaLon = nRasterXSize / NBGCPLON;
if( nDeltaLat == 0 || nDeltaLon == 0 )
return CE_None;
/* -------------------------------------------------------------------- */
/* Create HDF5 Data Hierarchy in a link list */
/* -------------------------------------------------------------------- */
poH5Objects=HDF5FindDatasetObjects( poH5RootGroup, "Latitude" );
if( !poH5Objects ) {
return CE_None;
}
/* -------------------------------------------------------------------- */
/* The Lattitude and Longitude arrays must have a rank of 2 to */
/* retrieve GCPs. */
/* -------------------------------------------------------------------- */
if( poH5Objects->nRank != 2 ) {
return CE_None;
}
/* -------------------------------------------------------------------- */
/* Retrieve HDF5 data information */
/* -------------------------------------------------------------------- */
LatitudeDatasetID = H5Dopen( hHDF5,poH5Objects->pszPath );
LatitudeDataspaceID = H5Dget_space( dataset_id );
poH5Objects=HDF5FindDatasetObjects( poH5RootGroup, "Longitude" );
LongitudeDatasetID = H5Dopen( hHDF5,poH5Objects->pszPath );
LongitudeDataspaceID = H5Dget_space( dataset_id );
if( ( LatitudeDatasetID > 0 ) && ( LongitudeDatasetID > 0) ) {
Latitude = ( float * ) CPLCalloc( nRasterYSize*nRasterXSize,
sizeof( float ) );
Longitude = ( float * ) CPLCalloc( nRasterYSize*nRasterXSize,
sizeof( float ) );
memset( Latitude, 0, nRasterXSize*nRasterYSize*sizeof( float ) );
memset( Longitude, 0, nRasterXSize*nRasterYSize*sizeof( float ) );
H5Dread ( LatitudeDatasetID,
H5T_NATIVE_FLOAT,
H5S_ALL,
H5S_ALL,
H5P_DEFAULT,
Latitude );
H5Dread ( LongitudeDatasetID,
H5T_NATIVE_FLOAT,
H5S_ALL,
H5S_ALL,
H5P_DEFAULT,
Longitude );
oSRS.SetWellKnownGeogCS( "WGS84" );
CPLFree(pszProjection);
CPLFree(pszGCPProjection);
oSRS.exportToWkt( &pszProjection );
oSRS.exportToWkt( &pszGCPProjection );
/* -------------------------------------------------------------------- */
/* Fill the GCPs list. */
/* -------------------------------------------------------------------- */
nGCPCount = nRasterYSize/nDeltaLat * nRasterXSize/nDeltaLon;
pasGCPList = ( GDAL_GCP * )
CPLCalloc( nGCPCount, sizeof( GDAL_GCP ) );
GDALInitGCPs( nGCPCount, pasGCPList );
int k=0;
int nYLimit = ((int)nRasterYSize/nDeltaLat) * nDeltaLat;
int nXLimit = ((int)nRasterXSize/nDeltaLon) * nDeltaLon;
for( j = 0; j < nYLimit; j+=nDeltaLat )
{
for( i = 0; i < nXLimit; i+=nDeltaLon )
{
int iGCP = j * nRasterXSize + i;
pasGCPList[k].dfGCPX = ( double ) Longitude[iGCP]+180.0;
pasGCPList[k].dfGCPY = ( double ) Latitude[iGCP];
pasGCPList[k].dfGCPPixel = i + 0.5;
pasGCPList[k++].dfGCPLine = j + 0.5;
}
}
CPLFree( Latitude );
CPLFree( Longitude );
}
if( LatitudeDatasetID > 0 )
H5Dclose(LatitudeDatasetID);
if( LongitudeDatasetID > 0 )
H5Dclose(LongitudeDatasetID);
break;
}
}
return CE_None;
}
int
main (int argc, char **argv)
{
hid_t file_id, dset_id, grp_id;
hid_t fapl, sid, mem_dataspace;
herr_t ret;
char filename[1024];
int mpi_size, mpi_rank, ndims, i, j;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info info = MPI_INFO_NULL;
hsize_t dims[RANK];
hsize_t start[RANK];
hsize_t count[RANK];
hsize_t stride[RANK];
hsize_t block[RANK];
DATATYPE *data_array = NULL, *dataptr; /* data buffer */
MPI_Init(&argc, &argv);
MPI_Comm_size(comm, &mpi_size);
MPI_Comm_rank(comm, &mpi_rank);
if(MAINPROCESS)
TESTING("proper shutdown of HDF5 library");
/* Set up file access property list with parallel I/O access */
fapl = H5Pcreate(H5P_FILE_ACCESS);
VRFY((fapl >= 0), "H5Pcreate succeeded");
ret = H5Pset_fapl_mpio(fapl, comm, info);
VRFY((ret >= 0), "");
h5_fixname(FILENAME[0], fapl, filename, sizeof filename);
file_id = H5Fopen(filename, H5F_ACC_RDONLY, fapl);
VRFY((file_id >= 0), "H5Fopen succeeded");
grp_id = H5Gopen2(file_id, "Group", H5P_DEFAULT);
VRFY((grp_id >= 0), "H5Gopen succeeded");
dset_id = H5Dopen2(grp_id, "Dataset", H5P_DEFAULT);
VRFY((dset_id >= 0), "H5Dopen succeeded");
sid = H5Dget_space(dset_id);
VRFY((dset_id >= 0), "H5Dget_space succeeded");
ndims = H5Sget_simple_extent_dims(sid, dims, NULL);
VRFY((ndims == 2), "H5Sget_simple_extent_dims succeeded");
VRFY(dims[0] == ROW_FACTOR*mpi_size, "Wrong dataset dimensions");
VRFY(dims[1] == COL_FACTOR*mpi_size, "Wrong dataset dimensions");
/* allocate memory for data buffer */
data_array = (DATATYPE *)HDmalloc(dims[0]*dims[1]*sizeof(DATATYPE));
VRFY((data_array != NULL), "data_array HDmalloc succeeded");
/* Each process takes a slabs of rows. */
block[0] = dims[0]/mpi_size;
block[1] = dims[1];
stride[0] = block[0];
stride[1] = block[1];
count[0] = 1;
count[1] = 1;
start[0] = mpi_rank*block[0];
start[1] = 0;
ret = H5Sselect_hyperslab(sid, H5S_SELECT_SET, start, stride, count, block);
VRFY((ret >= 0), "H5Sset_hyperslab succeeded");
/* create a memory dataspace independently */
mem_dataspace = H5Screate_simple (RANK, block, NULL);
VRFY((mem_dataspace >= 0), "");
/* write data independently */
ret = H5Dread(dset_id, H5T_NATIVE_INT, mem_dataspace, sid,
H5P_DEFAULT, data_array);
VRFY((ret >= 0), "H5Dwrite succeeded");
dataptr = data_array;
for (i=0; i < block[0]; i++){
for (j=0; j < block[1]; j++){
if(*dataptr != mpi_rank+1) {
printf("Dataset Verify failed at [%lu][%lu](row %lu, col %lu): expect %d, got %d\n",
(unsigned long)i, (unsigned long)j,
(unsigned long)(i+start[0]), (unsigned long)(j+start[1]),
mpi_rank+1, *(dataptr));
nerrors ++;
}
dataptr++;
}
}
MPI_Finalize();
HDremove(filename);
/* release data buffers */
if(data_array)
HDfree(data_array);
nerrors += GetTestNumErrs();
if(MAINPROCESS) {
if(0 == nerrors)
PASSED()
else
H5_FAILED()
}
return (nerrors!=0);
}
bool HDFWalkerInput_0_0::put(xmlNodePtr cur)
{
hid_t h_file = H5Fopen(FileName.c_str(),H5F_ACC_RDWR,H5P_DEFAULT);
hid_t h_config = H5Gopen(h_file,"config_collection");
if(h_config<0)
{
app_error() << " HDFWalkerInput_0_0::put config_collection is not found in " << FileName << "." << endl;
H5Fclose(h_file);
return false;
}
int NumSets=0;
hid_t h1=H5Dopen(h_config,"NumOfConfigurations");
if(h1>-1)
{
H5Dread(h1, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT,&(NumSets));
H5Dclose(h1);
}
if(!NumSets)
{
//resolve the integer and long problem with 64bit
hsize_t nset;
H5Gget_num_objs(h_config,&nset);
NumSets=nset;
}
if(!NumSets)
{
app_error() << " HDFWalkerInput_0_0::put " << FileName << " does not contain walkers!" << endl;
H5Gclose(h_config);
H5Gclose(h_file);
return false;
}
//select the last block
int selected=NumSets-1;
typedef MCWalkerConfiguration::PosType PosType;
typedef MCWalkerConfiguration::PropertyContainer_t ProtertyContainer_t;
typedef Matrix<PosType> PosContainer_t;
int nwt = 0;
int npt = 0;
//2D array of PosTypes (x,y,z) indexed by (walker,particle)
PosContainer_t Pos_temp;
//open the group
char GrpName[128];
sprintf(GrpName,"config%04d",selected);
hid_t group_id = H5Gopen(h_config,GrpName);
HDFAttribIO<PosContainer_t> Pos_in(Pos_temp);
//read the dataset
Pos_in.read(group_id,"coord");
//close groups
H5Gclose(group_id);
H5Gclose(h_config);
H5Fclose(h_file);
/*check to see if the number of walkers and particles is consistent with W */
int nptcl = Pos_temp.cols();
nwt = Pos_temp.rows();
int curWalker = targetW.getActiveWalkers();
if(curWalker)
{
app_log() << "HDFWalkerInput_0_0::put Adding " << nwt << " walkers to " << curWalker << endl;
targetW.createWalkers(nwt);
}
else
{
app_log() << "HDFWalkerInput_0_0::put Creating " << nwt << " walkers." << endl;
targetW.resize(nwt,nptcl);
}
//assign configurations to W
int iw=0;
MCWalkerConfiguration::iterator it = targetW.begin()+curWalker;
MCWalkerConfiguration::iterator it_end = targetW.end();
while(it != it_end)
{
std::copy(Pos_temp[iw],Pos_temp[iw+1], (*it)->R.begin());
++it;
++iw;
}
return true;
}
CPLErr BAGRasterBand::IReadBlock( int nBlockXOff, int nBlockYOff,
void *pImage )
{
const int nXOff = nBlockXOff * nBlockXSize;
H5OFFSET_TYPE offset[3] = {
static_cast<H5OFFSET_TYPE>(
std::max(0, nRasterYSize - (nBlockYOff + 1) * nBlockYSize)),
static_cast<H5OFFSET_TYPE>(nXOff),
static_cast<H5OFFSET_TYPE>(0)
};
const int nSizeOfData = static_cast<int>(H5Tget_size(native));
memset(pImage, 0, nBlockXSize * nBlockYSize * nSizeOfData);
// Blocksize may not be a multiple of imagesize.
hsize_t count[3] = {
std::min(static_cast<hsize_t>(nBlockYSize), GetYSize() - offset[0]),
std::min(static_cast<hsize_t>(nBlockXSize), GetXSize() - offset[1]),
static_cast<hsize_t>(0)
};
if( nRasterYSize - (nBlockYOff + 1) * nBlockYSize < 0 )
{
count[0] += (nRasterYSize - (nBlockYOff + 1) * nBlockYSize);
}
// Select block from file space.
{
const herr_t status =
H5Sselect_hyperslab(dataspace, H5S_SELECT_SET,
offset, nullptr, count, nullptr);
if( status < 0 )
return CE_Failure;
}
// Create memory space to receive the data.
hsize_t col_dims[3] = {
static_cast<hsize_t>(nBlockYSize),
static_cast<hsize_t>(nBlockXSize),
static_cast<hsize_t>(0)
};
const int rank = 2;
const hid_t memspace = H5Screate_simple(rank, col_dims, nullptr);
H5OFFSET_TYPE mem_offset[3] = { 0, 0, 0 };
const herr_t status =
H5Sselect_hyperslab(memspace, H5S_SELECT_SET,
mem_offset, nullptr, count, nullptr);
if( status < 0 )
return CE_Failure;
const herr_t status_read =
H5Dread(hDatasetID, native, memspace, dataspace, H5P_DEFAULT, pImage);
H5Sclose(memspace);
// Y flip the data.
const int nLinesToFlip = static_cast<int>(count[0]);
const int nLineSize = nSizeOfData * nBlockXSize;
GByte * const pabyTemp = static_cast<GByte *>(CPLMalloc(nLineSize));
GByte * const pbyImage = static_cast<GByte *>(pImage);
for( int iY = 0; iY < nLinesToFlip / 2; iY++ )
{
memcpy(pabyTemp, pbyImage + iY * nLineSize, nLineSize);
memcpy(pbyImage + iY * nLineSize,
pbyImage + (nLinesToFlip - iY - 1) * nLineSize,
nLineSize);
memcpy(pbyImage + (nLinesToFlip - iY - 1) * nLineSize, pabyTemp,
nLineSize);
}
CPLFree(pabyTemp);
// Return success or failure.
if( status_read < 0 )
{
CPLError(CE_Failure, CPLE_AppDefined, "H5Dread() failed for block.");
return CE_Failure;
}
return CE_None;
}
/*
This should handle properly the cases where PetscInt is 32 or 64 and hsize_t is 32 or 64. These means properly casting with
checks back and forth between the two types of variables.
*/
PetscErrorCode VecLoad_HDF5(Vec xin, PetscViewer viewer)
{
hid_t file_id, group, dset_id, filespace, memspace, plist_id;
hsize_t rdim, dim;
hsize_t dims[4], count[4], offset[4];
herr_t status;
PetscInt n, N, bs = 1, bsInd, lenInd, low, timestep;
PetscScalar *x;
const char *vecname;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscViewerHDF5OpenGroup(viewer, &file_id, &group);CHKERRQ(ierr);
ierr = PetscViewerHDF5GetTimestep(viewer, ×tep);CHKERRQ(ierr);
ierr = VecGetBlockSize(xin,&bs);CHKERRQ(ierr);
/* Create the dataset with default properties and close filespace */
ierr = PetscObjectGetName((PetscObject)xin,&vecname);CHKERRQ(ierr);
#if (H5_VERS_MAJOR * 10000 + H5_VERS_MINOR * 100 + H5_VERS_RELEASE >= 10800)
dset_id = H5Dopen2(group, vecname, H5P_DEFAULT);
#else
dset_id = H5Dopen(group, vecname);
#endif
if (dset_id == -1) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"Could not H5Dopen() with Vec named %s",vecname);
/* Retrieve the dataspace for the dataset */
filespace = H5Dget_space(dset_id);
if (filespace == -1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Could not H5Dget_space()");
dim = 0;
if (timestep >= 0) ++dim;
++dim;
if (bs >= 1) ++dim;
#if defined(PETSC_USE_COMPLEX)
++dim;
#endif
rdim = H5Sget_simple_extent_dims(filespace, dims, NULL);
#if defined(PETSC_USE_COMPLEX)
bsInd = rdim-2;
#else
bsInd = rdim-1;
#endif
lenInd = timestep >= 0 ? 1 : 0;
if (rdim != dim) {
if (rdim == dim+1 && bs == -1) bs = dims[bsInd];
else SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_FILE_UNEXPECTED, "Dimension of array in file %d not %d as expected",rdim,dim);
} else if (bs >= 1 && bs != (PetscInt) dims[bsInd]) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_FILE_UNEXPECTED, "Block size %d specified for vector does not match blocksize in file %d",bs,dims[bsInd]);
/* Set Vec sizes,blocksize,and type if not already set */
if ((xin)->map->n < 0 && (xin)->map->N < 0) {
ierr = VecSetSizes(xin, PETSC_DECIDE, dims[lenInd]*bs);CHKERRQ(ierr);
}
/* If sizes and type already set,check if the vector global size is correct */
ierr = VecGetSize(xin, &N);CHKERRQ(ierr);
if (N/bs != (PetscInt) dims[lenInd]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_FILE_UNEXPECTED, "Vector in file different length (%d) then input vector (%d)", (PetscInt) dims[lenInd], N/bs);
/* Each process defines a dataset and reads it from the hyperslab in the file */
ierr = VecGetLocalSize(xin, &n);CHKERRQ(ierr);
dim = 0;
if (timestep >= 0) {
count[dim] = 1;
++dim;
}
ierr = PetscHDF5IntCast(n/bs,count + dim);CHKERRQ(ierr);
++dim;
if (bs >= 1) {
count[dim] = bs;
++dim;
}
#if defined(PETSC_USE_COMPLEX)
count[dim] = 2;
++dim;
#endif
memspace = H5Screate_simple(dim, count, NULL);
if (memspace == -1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Could not H5Screate_simple()");
/* Select hyperslab in the file */
ierr = VecGetOwnershipRange(xin, &low, NULL);CHKERRQ(ierr);
dim = 0;
if (timestep >= 0) {
offset[dim] = timestep;
++dim;
}
ierr = PetscHDF5IntCast(low/bs,offset + dim);CHKERRQ(ierr);
++dim;
if (bs >= 1) {
offset[dim] = 0;
++dim;
}
#if defined(PETSC_USE_COMPLEX)
offset[dim] = 0;
++dim;
#endif
status = H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, count, NULL);CHKERRQ(status);
/* Create property list for collective dataset read */
plist_id = H5Pcreate(H5P_DATASET_XFER);
if (plist_id == -1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Could not H5Pcreate()");
#if defined(PETSC_HAVE_H5PSET_FAPL_MPIO)
status = H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE);CHKERRQ(status);
#endif
/* To write dataset independently use H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_INDEPENDENT) */
ierr = VecGetArray(xin, &x);CHKERRQ(ierr);
status = H5Dread(dset_id, H5T_NATIVE_DOUBLE, memspace, filespace, plist_id, x);CHKERRQ(status);
ierr = VecRestoreArray(xin, &x);CHKERRQ(ierr);
/* Close/release resources */
if (group != file_id) {
status = H5Gclose(group);CHKERRQ(status);
}
status = H5Pclose(plist_id);CHKERRQ(status);
status = H5Sclose(filespace);CHKERRQ(status);
status = H5Sclose(memspace);CHKERRQ(status);
status = H5Dclose(dset_id);CHKERRQ(status);
ierr = VecAssemblyBegin(xin);CHKERRQ(ierr);
ierr = VecAssemblyEnd(xin);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*------------------------------------------------------------------------- * Function: diff_datasetid * * Purpose: check for comparable datasets and read into a compatible * memory type * * Return: Number of differences found * * Programmer: Pedro Vicente, [email protected] * * Date: May 9, 2003 * * Modifications: * * * October 2006: Read by hyperslabs for big datasets. * * A threshold of H5TOOLS_MALLOCSIZE (128 MB) is the limit upon which I/O hyperslab is done * i.e., if the memory needed to read a dataset is greater than this limit, * then hyperslab I/O is done instead of one operation I/O * For each dataset, the memory needed is calculated according to * * memory needed = number of elements * size of each element * * if the memory needed is lower than H5TOOLS_MALLOCSIZE, then the following operations * are done * * H5Dread( input_dataset1 ) * H5Dread( input_dataset2 ) * * with all elements in the datasets selected. If the memory needed is greater than * H5TOOLS_MALLOCSIZE, then the following operations are done instead: * * a strip mine is defined for each dimension k (a strip mine is defined as a * hyperslab whose size is memory manageable) according to the formula * * (1) strip_mine_size[k ] = MIN(dimension[k ], H5TOOLS_BUFSIZE / size of memory type) * * where H5TOOLS_BUFSIZE is a constant currently defined as 1MB. This formula assures * that for small datasets (small relative to the H5TOOLS_BUFSIZE constant), the strip * mine size k is simply defined as its dimension k, but for larger datasets the * hyperslab size is still memory manageable. * a cycle is done until the number of elements in the dataset is reached. In each * iteration, two parameters are defined for the function H5Sselect_hyperslab, * the start and size of each hyperslab, according to * * (2) hyperslab_size [k] = MIN(dimension[k] - hyperslab_offset[k], strip_mine_size [k]) * * where hyperslab_offset [k] is initially set to zero, and later incremented in * hyperslab_size[k] offsets. The reason for the operation * * dimension[k] - hyperslab_offset[k] * * in (2) is that, when using the strip mine size, it assures that the "remaining" part * of the dataset that does not fill an entire strip mine is processed. * *------------------------------------------------------------------------- */ hsize_t diff_datasetid( hid_t did1, hid_t did2, const char *obj1_name, const char *obj2_name, diff_opt_t *options) { hid_t sid1=-1; hid_t sid2=-1; hid_t f_tid1=-1; hid_t f_tid2=-1; hid_t m_tid1=-1; hid_t m_tid2=-1; size_t m_size1; size_t m_size2; H5T_sign_t sign1; H5T_sign_t sign2; int rank1; int rank2; hsize_t nelmts1; hsize_t nelmts2; hsize_t dims1[H5S_MAX_RANK]; hsize_t dims2[H5S_MAX_RANK]; hsize_t maxdim1[H5S_MAX_RANK]; hsize_t maxdim2[H5S_MAX_RANK]; const char *name1=NULL; /* relative names */ const char *name2=NULL; hsize_t storage_size1; hsize_t storage_size2; hsize_t nfound=0; /* number of differences found */ int can_compare=1; /* do diff or not */ void *buf1=NULL; void *buf2=NULL; void *sm_buf1=NULL; void *sm_buf2=NULL; size_t need; /* bytes needed for malloc */ int i; /* Get the dataspace handle */ if ( (sid1 = H5Dget_space(did1)) < 0 ) goto error; /* Get rank */ if ( (rank1 = H5Sget_simple_extent_ndims(sid1)) < 0 ) goto error; /* Get the dataspace handle */ if ( (sid2 = H5Dget_space(did2)) < 0 ) goto error; /* Get rank */ if ( (rank2 = H5Sget_simple_extent_ndims(sid2)) < 0 ) goto error; /* Get dimensions */ if ( H5Sget_simple_extent_dims(sid1,dims1,maxdim1) < 0 ) goto error; /* Get dimensions */ if ( H5Sget_simple_extent_dims(sid2,dims2,maxdim2) < 0 ) { goto error; } /*------------------------------------------------------------------------- * get the file data type *------------------------------------------------------------------------- */ /* Get the data type */ if ( (f_tid1 = H5Dget_type(did1)) < 0 ) goto error; /* Get the data type */ if ( (f_tid2 = H5Dget_type(did2)) < 0 ) { goto error; } /*------------------------------------------------------------------------- * check for empty datasets *------------------------------------------------------------------------- */ storage_size1=H5Dget_storage_size(did1); storage_size2=H5Dget_storage_size(did2); if (storage_size1==0 || storage_size2==0) { if ( (options->m_verbose||options->m_list_not_cmp) && obj1_name && obj2_name) parallel_print("Not comparable: <%s> or <%s> is an empty dataset\n", obj1_name, obj2_name); can_compare=0; options->not_cmp=1; } /*------------------------------------------------------------------------- * check for comparable TYPE and SPACE *------------------------------------------------------------------------- */ if (diff_can_type(f_tid1, f_tid2, rank1, rank2, dims1, dims2, maxdim1, maxdim2, obj1_name, obj2_name, options, 0)!=1) { can_compare=0; } /*------------------------------------------------------------------------- * memory type and sizes *------------------------------------------------------------------------- */ if ((m_tid1=h5tools_get_native_type(f_tid1)) < 0) goto error; if ((m_tid2=h5tools_get_native_type(f_tid2)) < 0) goto error; m_size1 = H5Tget_size( m_tid1 ); m_size2 = H5Tget_size( m_tid2 ); /*------------------------------------------------------------------------- * check for different signed/unsigned types *------------------------------------------------------------------------- */ sign1=H5Tget_sign(m_tid1); sign2=H5Tget_sign(m_tid2); if ( sign1 != sign2 ) { if ((options->m_verbose||options->m_list_not_cmp) && obj1_name && obj2_name) { parallel_print("Not comparable: <%s> has sign %s ", obj1_name, get_sign(sign1)); parallel_print("and <%s> has sign %s\n", obj2_name, get_sign(sign2)); } can_compare=0; options->not_cmp=1; } /*------------------------------------------------------------------------- * only attempt to compare if possible *------------------------------------------------------------------------- */ if (can_compare ) /* it is possible to compare */ { /*------------------------------------------------------------------------- * get number of elements *------------------------------------------------------------------------- */ nelmts1 = 1; for (i = 0; i < rank1; i++) { nelmts1 *= dims1[i]; } nelmts2 = 1; for (i = 0; i < rank2; i++) { nelmts2 *= dims2[i]; } assert(nelmts1==nelmts2); /*------------------------------------------------------------------------- * "upgrade" the smaller memory size *------------------------------------------------------------------------- */ if ( m_size1 != m_size2 ) { if ( m_size1 < m_size2 ) { H5Tclose(m_tid1); if ((m_tid1=h5tools_get_native_type(f_tid2)) < 0) goto error; m_size1 = H5Tget_size( m_tid1 ); } else { H5Tclose(m_tid2); if ((m_tid2=h5tools_get_native_type(f_tid1)) < 0) goto error; m_size2 = H5Tget_size( m_tid2 ); } } assert(m_size1==m_size2); /* print names */ if (obj1_name) { name1=diff_basename(obj1_name); } if (obj2_name) { name2=diff_basename(obj2_name); } /*------------------------------------------------------------------------- * read/compare *------------------------------------------------------------------------- */ need = (size_t)(nelmts1*m_size1); /* bytes needed */ if ( need < H5TOOLS_MALLOCSIZE) { buf1 = HDmalloc(need); buf2 = HDmalloc(need); } if ( buf1!=NULL && buf2!=NULL) { if ( H5Dread(did1,m_tid1,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf1) < 0 ) goto error; if ( H5Dread(did2,m_tid2,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf2) < 0 ) goto error; /* array diff */ nfound = diff_array(buf1, buf2, nelmts1, (hsize_t)0, rank1, dims1, options, name1, name2, m_tid1, did1, did2); } else /* possibly not enough memory, read/compare by hyperslabs */ { size_t p_type_nbytes = m_size1; /*size of memory type */ hsize_t p_nelmts = nelmts1; /*total selected elmts */ hsize_t elmtno; /*counter */ int carry; /*counter carry value */ unsigned int vl_data = 0; /*contains VL datatypes */ /* stripmine info */ hsize_t sm_size[H5S_MAX_RANK]; /*stripmine size */ hsize_t sm_nbytes; /*bytes per stripmine */ hsize_t sm_nelmts; /*elements per stripmine*/ hid_t sm_space; /*stripmine data space */ /* hyperslab info */ hsize_t hs_offset[H5S_MAX_RANK]; /*starting offset */ hsize_t hs_size[H5S_MAX_RANK]; /*size this pass */ hsize_t hs_nelmts; /*elements in request */ hsize_t zero[8]; /*vector of zeros */ /* check if we have VL data in the dataset's datatype */ if (H5Tdetect_class(m_tid1, H5T_VLEN) == TRUE) vl_data = TRUE; /* * determine the strip mine size and allocate a buffer. The strip mine is * a hyperslab whose size is manageable. */ sm_nbytes = p_type_nbytes; for (i = rank1; i > 0; --i) { hsize_t size = H5TOOLS_BUFSIZE / sm_nbytes; if ( size == 0) /* datum size > H5TOOLS_BUFSIZE */ size = 1; sm_size[i - 1] = MIN(dims1[i - 1], size); sm_nbytes *= sm_size[i - 1]; assert(sm_nbytes > 0); } sm_buf1 = malloc((size_t)sm_nbytes); sm_buf2 = malloc((size_t)sm_nbytes); sm_nelmts = sm_nbytes / p_type_nbytes; sm_space = H5Screate_simple(1, &sm_nelmts, NULL); /* the stripmine loop */ memset(hs_offset, 0, sizeof hs_offset); memset(zero, 0, sizeof zero); for (elmtno = 0; elmtno < p_nelmts; elmtno += hs_nelmts) { /* calculate the hyperslab size */ if (rank1 > 0) { for (i = 0, hs_nelmts = 1; i < rank1; i++) { hs_size[i] = MIN(dims1[i] - hs_offset[i], sm_size[i]); hs_nelmts *= hs_size[i]; } if (H5Sselect_hyperslab(sid1, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL) < 0) goto error; if (H5Sselect_hyperslab(sid2, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL) < 0) goto error; if (H5Sselect_hyperslab(sm_space, H5S_SELECT_SET, zero, NULL, &hs_nelmts, NULL) < 0) goto error; } else { H5Sselect_all(sid1); H5Sselect_all(sid2); H5Sselect_all(sm_space); hs_nelmts = 1; } /* rank */ if ( H5Dread(did1,m_tid1,sm_space,sid1,H5P_DEFAULT,sm_buf1) < 0 ) goto error; if ( H5Dread(did2,m_tid2,sm_space,sid2,H5P_DEFAULT,sm_buf2) < 0 ) goto error; /* get array differences. in the case of hyperslab read, increment the number of differences found in each hyperslab and pass the position at the beggining for printing */ nfound += diff_array(sm_buf1, sm_buf2, hs_nelmts, elmtno, rank1, dims1, options, name1, name2, m_tid1, did1, did2); /* reclaim any VL memory, if necessary */ if(vl_data) { H5Dvlen_reclaim(m_tid1, sm_space, H5P_DEFAULT, sm_buf1); H5Dvlen_reclaim(m_tid1, sm_space, H5P_DEFAULT, sm_buf2); } /* calculate the next hyperslab offset */ for (i = rank1, carry = 1; i > 0 && carry; --i) { hs_offset[i - 1] += hs_size[i - 1]; if (hs_offset[i - 1] == dims1[i - 1]) hs_offset[i - 1] = 0; else carry = 0; } /* i */ } /* elmtno */ H5Sclose(sm_space); /* free */ if (sm_buf1!=NULL) { free(sm_buf1); sm_buf1=NULL; } if (sm_buf2!=NULL) { free(sm_buf2); sm_buf2=NULL; } } /* hyperslab read */ }/*can_compare*/ /*------------------------------------------------------------------------- * compare attributes * the if condition refers to cases when the dataset is a referenced object *------------------------------------------------------------------------- */ if (obj1_name) { nfound += diff_attr(did1,did2,obj1_name,obj2_name,options); } /*------------------------------------------------------------------------- * close *------------------------------------------------------------------------- */ /* free */ if (buf1!=NULL) { free(buf1); buf1=NULL; } if (buf2!=NULL) { free(buf2); buf2=NULL; } if (sm_buf1!=NULL) { free(sm_buf1); sm_buf1=NULL; } if (sm_buf2!=NULL) { free(sm_buf2); sm_buf2=NULL; } H5E_BEGIN_TRY { H5Sclose(sid1); H5Sclose(sid2); H5Tclose(f_tid1); H5Tclose(f_tid2); H5Tclose(m_tid1); H5Tclose(m_tid2); } H5E_END_TRY; return nfound; error: options->err_stat=1; /* free */ if (buf1!=NULL) { free(buf1); buf1=NULL; } if (buf2!=NULL) { free(buf2); buf2=NULL; } if (sm_buf1!=NULL) { free(sm_buf1); sm_buf1=NULL; } if (sm_buf2!=NULL) { free(sm_buf2); sm_buf2=NULL; } /* disable error reporting */ H5E_BEGIN_TRY { H5Sclose(sid1); H5Sclose(sid2); H5Tclose(f_tid1); H5Tclose(f_tid2); H5Tclose(m_tid1); H5Tclose(m_tid2); /* enable error reporting */ } H5E_END_TRY; return nfound; }
int
main (int argc, char **argv)
{
char myh5[STR_LENG] = { 0 };
int rec_start = rec_max/2;
float *qr, *qi;
float k1r, k1i, k2r, k2i, x1, x2;
char name[500];
double chi1[6];
hid_t memspace;
hsize_t count[2];
hsize_t offset[2];
hsize_t col_dims[1];
hid_t file_id, dataset_id, dataspace_id;
hid_t filespace;
herr_t status;
offset[0] = 0;
count[0] = rec_max;
count[1] = 1;
col_dims[0] = rec_max;
qr = (float *) malloc (sizeof (float) * rec_max);
qi = (float *) malloc (sizeof (float) * rec_max);
int pair, rec, k, g;
if (argc == 3)
{
pair=atoi(argv[1]);
offset[1]=atoi(argv[2]);
}
else
{
printf ("Error: usage: dist_chi0 sample_index beta_index\n");
return 0;
}
// for (pair = 0; pair < SAMPLES; pair++) {
sprintf(myh5,"sample_pair_%05d.h5",pair);
file_id = H5Fopen (myh5, H5F_ACC_RDONLY, H5P_DEFAULT);
if(file_id<0){
printf("File do not exist!\n");
exit(0);
}
for(k=0;k<6;k++)
{
x1 = 0.0f;
k1r = 0.0f;
k1i= 0.0f;
sprintf (name, "qk_real_%02d",k+4);
dataset_id = H5Dopen2 (file_id, name, H5P_DEFAULT);
filespace = H5Dget_space (dataset_id);
memspace = H5Screate_simple (RANKC, col_dims, NULL);
status = H5Sselect_hyperslab (filespace, H5S_SELECT_SET, offset, NULL,
count, NULL);
status =
H5Dread (dataset_id, H5T_NATIVE_FLOAT, memspace, filespace,
H5P_DEFAULT, qr);
status = H5Dclose (dataset_id);
sprintf (name, "qk_imag_%02d",k+4);
dataset_id = H5Dopen2 (file_id, name, H5P_DEFAULT);
filespace = H5Dget_space (dataset_id);
memspace = H5Screate_simple (RANKC, col_dims, NULL);
status = H5Sselect_hyperslab (filespace, H5S_SELECT_SET, offset, NULL,
count, NULL);
status =
H5Dread (dataset_id, H5T_NATIVE_FLOAT, memspace, filespace,
H5P_DEFAULT, qi);
status = H5Dclose (dataset_id);
for (rec = rec_start; rec < rec_max; rec++) {
x1 += qr[rec] * qr[rec] + qi[rec]*qi[rec];
k1r += qr[rec];
k1i += qi[rec];
}
double ax1 = x1 / (rec_max - rec_start);
double aq1r = k1r / (rec_max - rec_start);
double aq1i = k1i / (rec_max - rec_start);
ax1 = (ax1 - aq1r * aq1r- aq1i*aq1i)/SZ_CUBE;
chi1[k]=ax1;
}
status = H5Fclose (file_id);
printf ("%7d\t%2d\t%1.5f\n", pair,(int)offset[1], (chi1[0]+chi1[1]+chi1[2]+chi1[3]+chi1[4]+chi1[5])/6.0);
// }
return 0;
}
void
phdf5read2(char *filename, hio_context_t context)
{
hid_t fid1; /* HDF5 file IDs */
hid_t acc_tpl1; /* File access templates */
hid_t file_dataspace; /* File dataspace ID */
hid_t mem_dataspace; /* memory dataspace ID */
hid_t dataset1, dataset2; /* Dataset ID */
DATATYPE data_array1[SPACE1_DIM1][SPACE1_DIM2]; /* data buffer */
DATATYPE data_origin1[SPACE1_DIM1][SPACE1_DIM2]; /* expected data buffer */
hsize_t start[SPACE1_RANK]; /* for hyperslab setting */
hsize_t count[SPACE1_RANK], stride[SPACE1_RANK]; /* for hyperslab setting */
herr_t ret; /* Generic return value */
if (verbose)
printf("Collective read test on file %s\n", filename);
/* -------------------
* OPEN AN HDF5 FILE
* -------------------*/
/* setup file access template with hio IO access. */
acc_tpl1 = H5Pcreate (H5P_FILE_ACCESS);
assert(acc_tpl1 != FAIL);
MESG("H5Pcreate access succeed");
/* set Hio access with communicator */
H5FD_hio_set_read_io(&settings, H5FD_HIO_STRIDED);
ret = H5Pset_fapl_hio(acc_tpl1, &settings);
assert(ret != FAIL);
MESG("H5Pset_fapl_hio succeed");
/* open the file collectively */
fid1=H5Fopen(filename,H5F_ACC_RDONLY, acc_tpl1);
assert(fid1 != FAIL);
MESG("H5Fopen succeed");
/* Release file-access template */
ret=H5Pclose(acc_tpl1);
assert(ret != FAIL);
/* --------------------------
* Open the datasets in it
* ------------------------- */
/* open the dataset1 collectively */
dataset1 = H5Dopen2(fid1, DATASETNAME1, H5P_DEFAULT);
assert(dataset1 != FAIL);
MESG("H5Dopen2 succeed");
/* open another dataset collectively */
dataset2 = H5Dopen2(fid1, DATASETNAME2, H5P_DEFAULT);
assert(dataset2 != FAIL);
MESG("H5Dopen2 2 succeed");
/*
* Set up dimensions of the slab this process accesses.
*/
/* Dataset1: each process takes a block of columns. */
slab_set(start, count, stride, BYROW);
if (verbose)
printf("start[]=(%lu,%lu), count[]=(%lu,%lu), total datapoints=%lu\n",
(unsigned long)start[0], (unsigned long)start[1],
(unsigned long)count[0], (unsigned long)count[1],
(unsigned long)(count[0]*count[1]));
/* create a file dataspace independently */
file_dataspace = H5Dget_space (dataset1);
assert(file_dataspace != FAIL);
MESG("H5Dget_space succeed");
ret=H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride,
count, NULL);
assert(ret != FAIL);
MESG("H5Sset_hyperslab succeed");
/* create a memory dataspace independently */
mem_dataspace = H5Screate_simple (SPACE1_RANK, count, NULL);
assert (mem_dataspace != FAIL);
/* fill dataset with test data */
dataset_fill(start, count, stride, &data_origin1[0][0]);
MESG("data_array initialized");
if (verbose){
MESG("data_array created");
dataset_print(start, count, stride, &data_origin1[0][0]);
}
/* read data collectively */
ret = H5Dread(dataset1, H5T_NATIVE_INT, mem_dataspace, file_dataspace,
H5P_DEFAULT, data_array1);
assert(ret != FAIL);
MESG("H5Dread succeed");
/* verify the read data with original expected data */
ret = dataset_vrfy(start, count, stride, &data_array1[0][0], &data_origin1[0][0]);
assert(ret != FAIL);
/* release all temporary handles. */
/* Could have used them for dataset2 but it is cleaner */
/* to create them again.*/
H5Sclose(file_dataspace);
H5Sclose(mem_dataspace);
/* Dataset2: each process takes a block of rows. */
slab_set(start, count, stride, BYCOL);
if (verbose)
printf("start[]=(%lu,%lu), count[]=(%lu,%lu), total datapoints=%lu\n",
(unsigned long)start[0], (unsigned long)start[1],
(unsigned long)count[0], (unsigned long)count[1],
(unsigned long)(count[0]*count[1]));
/* create a file dataspace independently */
file_dataspace = H5Dget_space (dataset1);
assert(file_dataspace != FAIL);
MESG("H5Dget_space succeed");
ret=H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride,
count, NULL);
assert(ret != FAIL);
MESG("H5Sset_hyperslab succeed");
/* create a memory dataspace independently */
mem_dataspace = H5Screate_simple (SPACE1_RANK, count, NULL);
assert (mem_dataspace != FAIL);
/* fill dataset with test data */
dataset_fill(start, count, stride, &data_origin1[0][0]);
MESG("data_array initialized");
if (verbose){
MESG("data_array created");
dataset_print(start, count, stride, &data_array1[0][0]);
}
/* read data strided non-blocking */
ret = H5Dread(dataset2, H5T_NATIVE_INT, mem_dataspace, file_dataspace,
H5P_DEFAULT, data_array1);
assert(ret != FAIL);
MESG("H5Dread succeed");
/* verify the read data with original expected data */
ret = dataset_vrfy(start, count, stride, &data_array1[0][0], &data_origin1[0][0]);
assert(ret != FAIL);
/* release all temporary handles. */
H5Sclose(file_dataspace);
H5Sclose(mem_dataspace);
/*
* All reads completed. Close datasets collectively
*/
ret=H5Dclose(dataset1);
assert(ret != FAIL);
MESG("H5Dclose1 succeed");
ret=H5Dclose(dataset2);
assert(ret != FAIL);
MESG("H5Dclose2 succeed");
/* close the file collectively */
H5Fclose(fid1);
}
static void* _get_all_samples(hid_t gid_series, char* nam_series, uint32_t type,
int nsamples)
{
void* data = NULL;
hid_t id_data_set, dtyp_memory, g_sample, sz_dest;
herr_t ec;
int smpx ,len;
void *data_prior = NULL, *data_cur = NULL;
char name_sample[MAX_GROUP_NAME+1];
hdf5_api_ops_t* ops;
ops = profile_factory(type);
if (ops == NULL) {
error("Failed to create operations for %s",
acct_gather_profile_type_to_string(type));
return NULL;
}
data = (*(ops->init_job_series))(nsamples);
if (data == NULL) {
xfree(ops);
error("Failed to get memory for combined data");
return NULL;
}
dtyp_memory = (*(ops->create_memory_datatype))();
if (dtyp_memory < 0) {
xfree(ops);
xfree(data);
error("Failed to create %s memory datatype",
acct_gather_profile_type_to_string(type));
return NULL;
}
for (smpx=0; smpx<nsamples; smpx++) {
len = H5Lget_name_by_idx(gid_series, ".", H5_INDEX_NAME,
H5_ITER_INC, smpx, name_sample,
MAX_GROUP_NAME, H5P_DEFAULT);
if (len<1 || len>MAX_GROUP_NAME) {
error("Invalid group name %s", name_sample);
continue;
}
g_sample = H5Gopen(gid_series, name_sample, H5P_DEFAULT);
if (g_sample < 0) {
info("Failed to open %s", name_sample);
}
id_data_set = H5Dopen(g_sample, get_data_set_name(name_sample),
H5P_DEFAULT);
if (id_data_set < 0) {
H5Gclose(g_sample);
error("Failed to open %s dataset",
acct_gather_profile_type_to_string(type));
continue;
}
sz_dest = (*(ops->dataset_size))();
data_cur = xmalloc(sz_dest);
if (data_cur == NULL) {
H5Dclose(id_data_set);
H5Gclose(g_sample);
error("Failed to get memory for prior data");
continue;
}
ec = H5Dread(id_data_set, dtyp_memory, H5S_ALL, H5S_ALL,
H5P_DEFAULT, data_cur);
if (ec < 0) {
xfree(data_cur);
H5Dclose(id_data_set);
H5Gclose(g_sample);
error("Failed to read %s data",
acct_gather_profile_type_to_string(type));
continue;
}
(*(ops->merge_step_series))(g_sample, data_prior, data_cur,
data+(smpx)*sz_dest);
xfree(data_prior);
data_prior = data_cur;
H5Dclose(id_data_set);
H5Gclose(g_sample);
}
xfree(data_cur);
H5Tclose(dtyp_memory);
xfree(ops);
return data;
}
/* Example of using the hio HDF5 library to read a dataset */
void
phdf5read1(char *filename, hio_context_t context)
{
hid_t fid1; /* HDF5 file IDs */
hid_t acc_tpl1; /* File access templates */
hid_t file_dataspace; /* File dataspace ID */
hid_t mem_dataspace; /* memory dataspace ID */
hid_t dataset1, dataset2; /* Dataset ID */
DATATYPE data_array1[SPACE1_DIM1][SPACE1_DIM2]; /* data buffer */
DATATYPE data_origin1[SPACE1_DIM1][SPACE1_DIM2]; /* expected data buffer */
hsize_t start[SPACE1_RANK]; /* for hyperslab setting */
hsize_t count[SPACE1_RANK], stride[SPACE1_RANK]; /* for hyperslab setting */
herr_t ret; /* Generic return value */
if (verbose)
printf("Independent read test on file %s\n", filename);
/* setup file access template */
acc_tpl1 = H5Pcreate (H5P_FILE_ACCESS);
assert(acc_tpl1 != FAIL);
/* set Hio access with communicator */
H5FD_hio_set_read_io(&settings, H5FD_HIO_CONTIGUOUS);
ret = H5Pset_fapl_hio(acc_tpl1, &settings);
assert(ret != FAIL);
/* open the file collectively */
fid1=H5Fopen(filename,H5F_ACC_RDONLY,acc_tpl1);
assert(fid1 != FAIL);
/* Release file-access template */
ret=H5Pclose(acc_tpl1);
assert(ret != FAIL);
/* open the dataset1 collectively */
dataset1 = H5Dopen2(fid1, DATASETNAME1, H5P_DEFAULT);
assert(dataset1 != FAIL);
/* open another dataset collectively */
dataset2 = H5Dopen2(fid1, DATASETNAME1, H5P_DEFAULT);
assert(dataset2 != FAIL);
/* set up dimensions of the slab this process accesses */
start[0] = mpi_rank*SPACE1_DIM1/mpi_size;
start[1] = 0;
count[0] = SPACE1_DIM1/mpi_size;
count[1] = SPACE1_DIM2;
stride[0] = 1;
stride[1] =1;
if (verbose)
printf("start[]=(%lu,%lu), count[]=(%lu,%lu), total datapoints=%lu\n",
(unsigned long)start[0], (unsigned long)start[1],
(unsigned long)count[0], (unsigned long)count[1],
(unsigned long)(count[0]*count[1]));
/* create a file dataspace independently */
file_dataspace = H5Dget_space (dataset1);
assert(file_dataspace != FAIL);
ret=H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride,
count, NULL);
assert(ret != FAIL);
/* create a memory dataspace independently */
mem_dataspace = H5Screate_simple (SPACE1_RANK, count, NULL);
assert (mem_dataspace != FAIL);
/* fill dataset with test data */
dataset_fill(start, count, stride, &data_origin1[0][0]);
/* read data blocking and contiguous */
ret = H5Dread(dataset1, H5T_NATIVE_INT, mem_dataspace, file_dataspace,
H5P_DEFAULT, data_array1);
assert(ret != FAIL);
/* verify the read data with original expected data */
ret = dataset_vrfy(start, count, stride, &data_array1[0][0], &data_origin1[0][0]);
assert(ret != FAIL);
ret = H5Dread(dataset2, H5T_NATIVE_INT, mem_dataspace, file_dataspace,
H5P_DEFAULT, data_array1);
assert(ret != FAIL);
/* verify the read data with original expected data */
ret = dataset_vrfy(start, count, stride, &data_array1[0][0], &data_origin1[0][0]);
assert(ret == 0);
/* close dataset collectively */
ret=H5Dclose(dataset1);
assert(ret != FAIL);
ret=H5Dclose(dataset2);
assert(ret != FAIL);
/* release all IDs created */
H5Sclose(file_dataspace);
/* close the file collectively */
H5Fclose(fid1);
}
bool ReadHDF5file(char* filename, char* fieldname, dtype** outArray, int* dims)
{
#ifdef _HDF5_H
// Open the file
hid_t file_id;
file_id = H5Fopen(filename,H5F_ACC_RDONLY,H5P_DEFAULT);
//? file_id = H5Fopen(filename,H5F_ACC_RDONLY,faplist_id);
if(file_id < 0){
printf("ERROR: Could not open file %s\n",filename);
return false;
}
// Open the dataset
hid_t dataset_id;
hid_t dataspace_id;
dataset_id = H5Dopen1(file_id, fieldname);
if(dataset_id < 0){
printf("ERROR: Could not open the data field %s\n",fieldname);
return false;
}
dataspace_id = H5Dget_space(dataset_id);
// Test if 2D data
int ndims;
ndims = H5Sget_simple_extent_ndims(dataspace_id);
// Get dimensions of data set (nx, ny, nn)
hsize_t* dimsl = new hsize_t[ndims];
H5Sget_simple_extent_dims(dataspace_id,dimsl,NULL);
for(int i = 0;(i<ndims&&i<3);i++)
dims[i] = dimsl[ndims-1-i]; //!!!!!!!! NOT SURE
size_t nn = 1;
for(int i = 0;i<ndims;i++)
nn *= dimsl[i];
// Create space for the new data
dtype* data = *outArray;
if (data!=NULL) delete data;//free(data);
*outArray = new dtype[nn];
data = *outArray;
hid_t datatype_id;
H5T_class_t dataclass;
size_t size;
datatype_id = H5Dget_type(dataset_id);
dataclass = H5Tget_class(datatype_id);
size = H5Tget_size(datatype_id);
int rrr = sizeof(int);
if(dataclass == H5T_FLOAT){
if (size == sizeof(float)) {
float* buffer = (float *) calloc(nn, sizeof(float));
H5Dread(dataset_id, datatype_id, H5S_ALL,H5S_ALL, H5P_DEFAULT, buffer);
for(long i=0; i<nn; i++)
data[i] = buffer[i];
free(buffer);
}
else if (size == sizeof(double)) {
double* buffer = (double *) calloc(nn, sizeof(double));
H5Dread(dataset_id, datatype_id, H5S_ALL,H5S_ALL, H5P_DEFAULT, buffer);
for(long i=0; i<nn; i++)
data[i] = buffer[i];
free(buffer);
}
else {
printf("2dData::readHDF5: unknown floating point type, size=%i\n",(int) size);
return false;
}
}
else if(dataclass == H5T_INTEGER){
if (size == sizeof(short)) {
short* buffer = (short*) calloc(nn, sizeof(short));
H5Dread(dataset_id, datatype_id, H5S_ALL,H5S_ALL, H5P_DEFAULT, buffer);
for(long i=0; i<nn; i++)
data[i] = buffer[i];
free(buffer);
}
else if (size == sizeof(int)) {
int* buffer = (int *) calloc(nn, sizeof(int));
H5Dread(dataset_id, datatype_id, H5S_ALL,H5S_ALL, H5P_DEFAULT, buffer);
for(long i=0; i<nn; i++)
data[i] = buffer[i];
free(buffer);
}
else if (size == sizeof(long)) {
long* buffer = (long *) calloc(nn, sizeof(long));
H5Dread(dataset_id, datatype_id, H5S_ALL,H5S_ALL, H5P_DEFAULT, buffer);
for(long i=0; i<nn; i++)
data[i] = buffer[i];
free(buffer);
}
else {
printf("2dData::readHDF5: unknown integer type, size=%i\n",(int) size);
return false;
}
}
else {
printf("2dData::readHDF5: unknown HDF5 data type\n");
return false;
}
// Close and cleanup
H5Dclose(dataset_id);
// Cleanup stale IDs
hid_t ids[256];
int n_ids = H5Fget_obj_ids(file_id, H5F_OBJ_ALL, 256, ids);
for (long i=0; i<n_ids; i++ ) {
hid_t id;
H5I_type_t type;
id = ids[i];
type = H5Iget_type(id);
if ( type == H5I_GROUP )
H5Gclose(id);
if ( type == H5I_DATASET )
H5Dclose(id);
if ( type == H5I_DATASPACE )
H5Sclose(id);
//if ( type == H5I_DATATYPE )
// H5Dclose(id);
}
H5Fclose(file_id);
return true;
#endif
return false;
}
void cData2d::readHDF5(char* filename, char* fieldname){
// Open the file
hid_t file_id;
file_id = H5Fopen(filename,H5F_ACC_RDONLY,H5P_DEFAULT);
if(file_id < 0){
printf("ERROR: Could not open file %s\n",filename);
return;
}
// Open the dataset
hid_t dataset_id;
hid_t dataspace_id;
dataset_id = H5Dopen1(file_id, fieldname);
dataspace_id = H5Dget_space(dataset_id);
// Test if 2D data
int ndims;
ndims = H5Sget_simple_extent_ndims(dataspace_id);
if(ndims != 2) {
printf("2dData::readHDF5: Not 2D data set, ndims=%i\n",ndims);
exit(0);
}
// Get dimensions of data set (nx, ny, nn)
hsize_t dims[ndims];
H5Sget_simple_extent_dims(dataspace_id,dims,NULL);
ny = dims[0];
nx = dims[1];
nn = 1;
for(int i = 0;i<ndims;i++)
nn *= dims[i];
// Create space for the new data
free(data); data = NULL;
data = (tData2d *) calloc(nn, sizeof(tData2d));
// Read in data after setting up a temporary buffer of the appropriate variable type
// Somehow this works best when split out accordint to different data types
// Fix into general form later
hid_t datatype_id;
H5T_class_t dataclass;
size_t size;
datatype_id = H5Dget_type(dataset_id);
dataclass = H5Tget_class(datatype_id);
size = H5Tget_size(datatype_id);
if(dataclass == H5T_FLOAT){
if (size == sizeof(float)) {
float* buffer = (float *) calloc(nn, sizeof(float));
H5Dread(dataset_id, datatype_id, H5S_ALL,H5S_ALL, H5P_DEFAULT, buffer);
for(long i=0; i<nn; i++)
data[i] = buffer[i];
free(buffer);
}
else if (size == sizeof(double)) {
double* buffer = (double *) calloc(nn, sizeof(double));
H5Dread(dataset_id, datatype_id, H5S_ALL,H5S_ALL, H5P_DEFAULT, buffer);
for(long i=0; i<nn; i++)
data[i] = buffer[i];
free(buffer);
}
else {
printf("2dData::readHDF5: unknown floating point type, size=%i\n",(int) size);
return;
}
}
else if(dataclass == H5T_INTEGER){
if (size == sizeof(char)) {
char* buffer = (char*) calloc(nn, sizeof(char));
H5Dread(dataset_id, datatype_id, H5S_ALL,H5S_ALL, H5P_DEFAULT, buffer);
for(long i=0; i<nn; i++)
data[i] = buffer[i];
free(buffer);
}
else if (size == sizeof(short)) {
short* buffer = (short*) calloc(nn, sizeof(short));
H5Dread(dataset_id, datatype_id, H5S_ALL,H5S_ALL, H5P_DEFAULT, buffer);
for(long i=0; i<nn; i++)
data[i] = buffer[i];
free(buffer);
}
else if (size == sizeof(int)) {
int* buffer = (int *) calloc(nn, sizeof(int));
H5Dread(dataset_id, datatype_id, H5S_ALL,H5S_ALL, H5P_DEFAULT, buffer);
for(long i=0; i<nn; i++)
data[i] = buffer[i];
free(buffer);
}
else if (size == sizeof(long)) {
long* buffer = (long *) calloc(nn, sizeof(long));
H5Dread(dataset_id, datatype_id, H5S_ALL,H5S_ALL, H5P_DEFAULT, buffer);
for(long i=0; i<nn; i++)
data[i] = buffer[i];
free(buffer);
}
else {
printf("2dData::readHDF5: unknown integer type, size=%lu\n",size);
exit(1);
}
}
else {
printf("2dData::readHDF5: unknown HDF5 data type\n");
return;
}
// Close and cleanup
H5Dclose(dataset_id);
// Cleanup stale IDs
hid_t ids[256];
int n_ids = H5Fget_obj_ids(file_id, H5F_OBJ_ALL, 256, ids);
for (long i=0; i<n_ids; i++ ) {
hid_t id;
H5I_type_t type;
id = ids[i];
type = H5Iget_type(id);
if ( type == H5I_GROUP )
H5Gclose(id);
if ( type == H5I_DATASET )
H5Dclose(id);
if ( type == H5I_DATASPACE )
H5Sclose(id);
//if ( type == H5I_DATATYPE )
// H5Dclose(id);
}
H5Fclose(file_id);
}
int main(int argc, char **argv){
int mpi_size, mpi_rank;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info info;
MPI_Init(&argc, &argv);
MPI_Comm_size(comm, &mpi_size);
MPI_Comm_rank(comm, &mpi_rank);
int c;
opterr = 0;
strncpy(output, "/global/cscratch1/sd/jialin/hdf-data/climate/temp1.h5",NAME_MAX);
strncpy(filename, "./fake_xyz_default.h5", NAME_MAX);
strncpy(cb_buffer_size,"16777216", NAME_MAX);
strncpy(cb_nodes, "16", NAME_MAX);
int col=1;//collective read/write
//input args: i: inputfilename,o:outputfilename b: collective_buffersize, n: collective_buffernodes, k:iscollective, v:datasetname
while ((c = getopt (argc, argv, "i:o:b:n:k:v:")) != -1)
switch (c)
{
case 'i':
strncpy(filename, optarg, NAME_MAX);
break;
case 'o':
strncpy(output, optarg, NAME_MAX);
break;
case 'b':
strncpy(cb_buffer_size,optarg, NAME_MAX);
break;
case 'n':
strncpy(cb_nodes, optarg, NAME_MAX);
break;
case 'k':
col = strtol(optarg, NULL, 10);
case 'v':
strncpy(DATASETNAME, optarg, NAME_MAX);
default:
break;
}
MPI_Info_create(&info);
MPI_Info_set(info, "cb_buffer_size", cb_buffer_size);
MPI_Info_set(info, "cb_nodes", cb_nodes);
//Open file/dataset
hid_t fapl,file,dataset;
fapl = H5Pcreate(H5P_FILE_ACCESS);
H5Pset_fapl_mpio(fapl, comm, info);
file= H5Fopen(filename, H5F_ACC_RDONLY, fapl);
H5Pclose(fapl);
dataset= H5Dopen(file, DATASETNAME,H5P_DEFAULT);
hid_t datatype,dataspace;
H5T_class_t class; /* data type class */
H5T_order_t order; /* data order */
size_t size; /* size of data*/
int i, status_n,rank;
dataspace = H5Dget_space(dataset); /* dataspace handle */
rank = H5Sget_simple_extent_ndims(dataspace);
hsize_t dims_out[rank];
status_n = H5Sget_simple_extent_dims(dataspace, dims_out, NULL);
hsize_t offset[rank];
hsize_t count[rank];
//parallelize along x dims
hsize_t rest;
rest = dims_out[0] % mpi_size;
if(mpi_rank != (mpi_size -1)){
count[0] = dims_out[0]/mpi_size;
}else{
count[0] = dims_out[0]/mpi_size + rest;
}
offset[0] = dims_out[0]/mpi_size*mpi_rank;
//select all for other dims
for(i=1; i<rank; i++){
offset[i] = 0;
count[i] = dims_out[i];
}
//specify the selection in the dataspace for each rank
H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, NULL, count, NULL);
hsize_t rankmemsize=1;
hid_t memspace;
for(i=0; i<rank; i++){
rankmemsize*=count[i];
}
memspace = H5Screate_simple(rank,count,NULL);
float totalsizegb=mpi_size * rankmemsize *size / 1024.0 / 1024.0 / 1024.0;
if(mpi_rank==0)
printf("rankmemsize:%d, %dBytes\n",rankmemsize, rankmemsize*sizeof(double));
double * data_t=(double *)malloc( rankmemsize * sizeof(double));
if(data_t == NULL){
printf("Memory allocation fails mpi_rank = %d",mpi_rank);
for (i=0; i< rank; i++){
printf("Dim %d: %d, ",i,count[i]);
}
return -1;
}
MPI_Barrier(comm);
double tr0 = MPI_Wtime();
if(mpi_rank == 0){
if(col==1)
printf("IO: Collective Read\n");
else
printf("IO: Independent Read\n");
}
hid_t plist=H5P_DEFAULT;
if(col==1){
plist = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(plist, H5FD_MPIO_COLLECTIVE);
}
else {
plist = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(plist,H5FD_MPIO_INDEPENDENT);
}
H5Dread(dataset, H5T_NATIVE_DOUBLE, memspace,dataspace, plist, data_t);
printf("rank %d,start0 %lld count0 %lld,start1 %lld count1 %lld\n",mpi_rank,offset[0],count[0],offset[1],count[1]);
H5Pclose(plist);
MPI_Barrier(comm);
double tr1 = MPI_Wtime()-tr0;
if(mpi_rank==0){
printf("\nRank %d, read time %.2fs\n",mpi_rank,tr1);
for(i=0; i<rank; i++){
printf("Start_%d:%d Count_%d:%d\n",i,offset[i],i,count[i]);
}
printf("\n");
printf("Total Loading %f GB with %d mpi processes\n",totalsizegb,mpi_size);
}
//close object handle of file 1
//H5Tclose(datatype);
H5Sclose(dataspace);
H5Sclose(memspace);
H5Dclose(dataset);
//H5Fclose(file);
MPI_Barrier(comm);
if(mpi_rank==0) printf("Closing File %s ok,\nOpening File %s \n",filename,output);
//start to write to new places
//printf("%d:%d,%d\n",mpi_rank,offset[0],count[0]);
hid_t plist_id2, file_id2,dataspace_id2, dataset_id2;
//new file access property
plist_id2 = H5Pcreate(H5P_FILE_ACCESS);
//mpiio
H5Pset_fapl_mpio(plist_id2, comm, info);
//create new file
file_id2 = H5Fcreate(output, H5F_ACC_TRUNC, H5P_DEFAULT, plist_id2);
//file_id2=H5Fcreate(output, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
H5Pclose(plist_id2);
//in mem data space for new file, rank, dims_out, same
dataspace_id2 = H5Screate_simple(rank, dims_out, NULL);
/*if(class == H5T_INTEGER)
dataset_id2 = H5Dcreate(file_id2,DATASETNAME, H5T_NATIVE_INT, dataspace_id2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
else if(class == H5T_FLOAT){
if(size==4) dataset_id2=H5Dcreate(file_id2,DATASETNAME, H5T_NATIVE_FLOAT, dataspace_id2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
else if(size==8) dataset_id2=H5Dcreate(file_id2,DATASETNAME, H5T_NATIVE_DOUBLE, dataspace_id2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
}
*/
//dataset_id2=H5Dcreate(file_id2,DATASETNAME, H5T_NATIVE_DOUBLE, dataspace_id2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Sclose(dataspace_id2);
//dataspace_id2=H5Dget_space(dataset_id2);
//herr_t eit=H5Sselect_hyperslab(dataspace_id2, H5S_SELECT_SET, offset, NULL, count, NULL);
MPI_Barrier(comm);
double tw0 = MPI_Wtime();
if(mpi_rank == 0){
//if(eit<0) printf("hyperslab error in file 2\n");
if(col==1)
printf("IO: Collective Write\n");
else
printf("IO: Independent Write\n");
}
hid_t memspace_id2 = H5Screate_simple(rank, count, NULL);
hid_t plist_id3 = H5P_DEFAULT;
if(col==1){
plist_id3 = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(plist_id3, H5FD_MPIO_COLLECTIVE);
}
/*
if(class == H5T_INTEGER)
H5Dwrite(dataset_id2, H5T_NATIVE_INT, memspace_id2, dataspace_id2, plist_id3, data_t);
else if(class == H5T_FLOAT){
if(size==4) H5Dwrite(dataset_id2, H5T_NATIVE_FLOAT, memspace_id2, dataspace_id2, plist_id3, data_t);
else if(size==8) H5Dwrite(dataset_id2, H5T_NATIVE_DOUBLE, memspace_id2, dataspace_id2, plist_id3, data_t);
}
H5Pclose(plist_id3);
*/
//H5Dwrite(dataset_id2, H5T_NATIVE_DOUBLE, memspace_id2, dataspace_id2, H5P_DEFAULT, data_t);
if(mpi_rank==0||mpi_rank==1){
printf("rank %d:",mpi_rank);
for(i=0;i<10;i++)
//printf("%.2lf ", *(data_t+i));
printf("\n");
}
MPI_Barrier(comm);
double tw1 = MPI_Wtime()-tw0;
if(mpi_rank==0||mpi_rank==mpi_size-1)
{
printf("rank %d,write time %.2fs\n",mpi_rank,tw1);
printf("Total Writing %f GB with %d mpi processes \n",totalsizegb,mpi_size);
}
/*
if(data_t!=NULL){
free(data_t);
if(mpi_rank==0) printf("data free ok\n");
}
*/
//clean up object handle
/*H5Tclose(datatype);
H5Sclose(dataspace);
H5Sclose(memspace);
H5Dclose(dataset);
H5Fclose(file);
*/
/*
H5Sclose(dataspace_id2);
H5Sclose(memspace_id2);
H5Dclose(dataset_id2);
H5Fclose(file_id2);
*/
MPI_Finalize();
}
int
main (void)
{
hid_t file, filetype, memtype, strtype, space, dset;
/* Handles */
herr_t status;
hsize_t dims[1] = {DIM0};
sensor_t wdata[DIM0], /* Write buffer */
*rdata; /* Read buffer */
int ndims,
i;
/*
* Initialize data.
*/
wdata[0].serial_no = 1153;
wdata[0].location = "Exterior (static)";
wdata[0].temperature = 53.23;
wdata[0].pressure = 24.57;
wdata[1].serial_no = 1184;
wdata[1].location = "Intake";
wdata[1].temperature = 55.12;
wdata[1].pressure = 22.95;
wdata[2].serial_no = 1027;
wdata[2].location = "Intake manifold";
wdata[2].temperature = 103.55;
wdata[2].pressure = 31.23;
wdata[3].serial_no = 1313;
wdata[3].location = "Exhaust manifold";
wdata[3].temperature = 1252.89;
wdata[3].pressure = 84.11;
/*
* Create a new file using the default properties.
*/
file = H5Fcreate (FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
/*
* Create variable-length string datatype.
*/
strtype = H5Tcopy (H5T_C_S1);
status = H5Tset_size (strtype, H5T_VARIABLE);
/*
* Create the compound datatype for memory.
*/
memtype = H5Tcreate (H5T_COMPOUND, sizeof (sensor_t));
status = H5Tinsert (memtype, "Serial number",
HOFFSET (sensor_t, serial_no), H5T_NATIVE_INT);
status = H5Tinsert (memtype, "Location", HOFFSET (sensor_t, location),
strtype);
status = H5Tinsert (memtype, "Temperature (F)",
HOFFSET (sensor_t, temperature), H5T_NATIVE_DOUBLE);
status = H5Tinsert (memtype, "Pressure (inHg)",
HOFFSET (sensor_t, pressure), H5T_NATIVE_DOUBLE);
/*
* Create the compound datatype for the file. Because the standard
* types we are using for the file may have different sizes than
* the corresponding native types, we must manually calculate the
* offset of each member.
*/
filetype = H5Tcreate (H5T_COMPOUND, 8 + sizeof (hvl_t) + 8 + 8);
status = H5Tinsert (filetype, "Serial number", 0, H5T_STD_I64BE);
status = H5Tinsert (filetype, "Location", 8, strtype);
status = H5Tinsert (filetype, "Temperature (F)", 8 + sizeof (hvl_t),
H5T_IEEE_F64BE);
status = H5Tinsert (filetype, "Pressure (inHg)", 8 + sizeof (hvl_t) + 8,
H5T_IEEE_F64BE);
/*
* Create dataspace. Setting maximum size to NULL sets the maximum
* size to be the current size.
*/
space = H5Screate_simple (1, dims, NULL);
/*
* Create the dataset and write the compound data to it.
*/
dset = H5Dcreate (file, DATASET, filetype, space, H5P_DEFAULT);
status = H5Dwrite (dset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata);
/*
* Close and release resources.
*/
status = H5Dclose (dset);
status = H5Sclose (space);
status = H5Tclose (filetype);
status = H5Fclose (file);
/*
* Now we begin the read section of this example. Here we assume
* the dataset has the same name and rank, but can have any size.
* Therefore we must allocate a new array to read in data using
* malloc(). For simplicity, we do not rebuild memtype.
*/
/*
* Open file and dataset.
*/
file = H5Fopen (FILE, H5F_ACC_RDONLY, H5P_DEFAULT);
dset = H5Dopen (file, DATASET);
/*
* Get dataspace and allocate memory for read buffer.
*/
space = H5Dget_space (dset);
ndims = H5Sget_simple_extent_dims (space, dims, NULL);
rdata = (sensor_t *) malloc (dims[0] * sizeof (sensor_t));
/*
* Read the data.
*/
status = H5Dread (dset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata);
/*
* Output the data to the screen.
*/
for (i=0; i<dims[0]; i++) {
printf ("%s[%d]:\n", DATASET, i);
printf ("Serial number : %d\n", rdata[i].serial_no);
printf ("Location : %s\n", rdata[i].location);
printf ("Temperature (F) : %f\n", rdata[i].temperature);
printf ("Pressure (inHg) : %f\n\n", rdata[i].pressure);
}
/*
* Close and release resources. H5Dvlen_reclaim will automatically
* traverse the structure and free any vlen data (strings in this
* case).
*/
status = H5Dvlen_reclaim (memtype, space, H5P_DEFAULT, rdata);
free (rdata);
status = H5Dclose (dset);
status = H5Sclose (space);
status = H5Tclose (memtype);
status = H5Tclose (strtype);
status = H5Fclose (file);
return 0;
}
int
main(void)
{
/* First structure and dataset*/
typedef struct s1_t {
int a;
float b;
double c;
} s1_t;
s1_t s1[LENGTH];
hid_t s1_tid; /* File datatype identifier */
/* Second structure (subset of s1_t) and dataset*/
typedef struct s2_t {
double c;
int a;
} s2_t;
s2_t s2[LENGTH];
hid_t s2_tid; /* Memory datatype handle */
/* Third "structure" ( will be used to read float field of s1) */
hid_t s3_tid; /* Memory datatype handle */
float s3[LENGTH];
int i;
hid_t file, dataset, space; /* Handles */
herr_t status;
hsize_t dim[] = {LENGTH}; /* Dataspace dimensions */
/*
* Initialize the data
*/
for (i = 0; i< LENGTH; i++) {
s1[i].a = i;
s1[i].b = i*i;
s1[i].c = 1./(i+1);
}
/*
* Create the data space.
*/
space = H5Screate_simple(RANK, dim, NULL);
/*
* Create the file.
*/
file = H5Fcreate(H5FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
/*
* Create the memory data type.
*/
s1_tid = H5Tcreate (H5T_COMPOUND, sizeof(s1_t));
H5Tinsert(s1_tid, "a_name", HOFFSET(s1_t, a), H5T_NATIVE_INT);
H5Tinsert(s1_tid, "c_name", HOFFSET(s1_t, c), H5T_NATIVE_DOUBLE);
H5Tinsert(s1_tid, "b_name", HOFFSET(s1_t, b), H5T_NATIVE_FLOAT);
/*
* Create the dataset.
*/
dataset = H5Dcreate(file, DATASETNAME, s1_tid, space, H5P_DEFAULT);
/*
* Wtite data to the dataset;
*/
status = H5Dwrite(dataset, s1_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, s1);
/*
* Release resources
*/
H5Tclose(s1_tid);
H5Sclose(space);
H5Dclose(dataset);
H5Fclose(file);
/*
* Open the file and the dataset.
*/
file = H5Fopen(H5FILE_NAME, H5F_ACC_RDONLY, H5P_DEFAULT);
dataset = H5Dopen(file, DATASETNAME);
/*
* Create a data type for s2
*/
s2_tid = H5Tcreate(H5T_COMPOUND, sizeof(s2_t));
H5Tinsert(s2_tid, "c_name", HOFFSET(s2_t, c), H5T_NATIVE_DOUBLE);
H5Tinsert(s2_tid, "a_name", HOFFSET(s2_t, a), H5T_NATIVE_INT);
/*
* Read two fields c and a from s1 dataset. Fields in the file
* are found by their names "c_name" and "a_name".
*/
status = H5Dread(dataset, s2_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, s2);
/*
* Display the fields
*/
printf("\n");
printf("Field c : \n");
for( i = 0; i < LENGTH; i++) printf("%.4f ", s2[i].c);
printf("\n");
printf("\n");
printf("Field a : \n");
for( i = 0; i < LENGTH; i++) printf("%d ", s2[i].a);
printf("\n");
/*
* Create a data type for s3.
*/
s3_tid = H5Tcreate(H5T_COMPOUND, sizeof(float));
status = H5Tinsert(s3_tid, "b_name", 0, H5T_NATIVE_FLOAT);
/*
* Read field b from s1 dataset. Field in the file is found by its name.
*/
status = H5Dread(dataset, s3_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, s3);
/*
* Display the field
*/
printf("\n");
printf("Field b : \n");
for( i = 0; i < LENGTH; i++) printf("%.4f ", s3[i]);
printf("\n");
/*
* Release resources
*/
H5Tclose(s2_tid);
H5Tclose(s3_tid);
H5Dclose(dataset);
H5Fclose(file);
return 0;
}
/*
* This routine verifies the data written in the dataset. It does one of the
* three cases according to the value of parameter `write_pattern'.
* 1. it returns correct fill values though the dataset has not been written;
* 2. it still returns correct fill values though only a small part is written;
* 3. it returns correct values when the whole dataset has been written in an
* interleaved pattern.
*/
static void
verify_data(const char *filename, int chunk_factor, write_type write_pattern, int close, hid_t *file_id, hid_t *dataset)
{
/* HDF5 gubbins */
hid_t dataspace, memspace; /* HDF5 file identifier */
hid_t access_plist; /* HDF5 ID for file access property list */
herr_t hrc; /* HDF5 return code */
hsize_t chunk_dims[1] ={CHUNK_SIZE};
hsize_t count[1];
hsize_t stride[1];
hsize_t block[1];
hsize_t offset[1]; /* Selection offset within dataspace */
/* Variables used in reading data back */
char buffer[CHUNK_SIZE];
int value, i;
int index;
long nchunks;
/* Initialize MPI */
MPI_Comm_size(MPI_COMM_WORLD,&mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank);
nchunks=chunk_factor*mpi_size;
/* Set up MPIO file access property lists */
access_plist = H5Pcreate(H5P_FILE_ACCESS);
VRFY((access_plist >= 0), "");
hrc = H5Pset_fapl_mpio(access_plist, MPI_COMM_WORLD, MPI_INFO_NULL);
VRFY((hrc >= 0), "");
/* Open the file */
if (*file_id<0){
*file_id = H5Fopen(filename, H5F_ACC_RDWR, access_plist);
VRFY((*file_id >= 0), "");
}
/* Open dataset*/
if (*dataset<0){
*dataset = H5Dopen2(*file_id, DSET_NAME, H5P_DEFAULT);
VRFY((*dataset >= 0), "");
}
memspace = H5Screate_simple(1, chunk_dims, NULL);
VRFY((memspace >= 0), "");
dataspace = H5Dget_space(*dataset);
VRFY((dataspace >= 0), "");
/* all processes check all chunks. */
count[0] = 1;
stride[0] = 1;
block[0] = chunk_dims[0];
for (i=0; i<nchunks; i++){
/* reset buffer values */
memset(buffer, -1, CHUNK_SIZE);
offset[0] = i*chunk_dims[0];
hrc = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, offset, stride, count, block);
VRFY((hrc >= 0), "");
/* Read the chunk */
hrc = H5Dread(*dataset, H5T_NATIVE_UCHAR, memspace, dataspace, H5P_DEFAULT, buffer);
VRFY((hrc >= 0), "H5Dread");
/* set expected value according the write pattern */
switch (write_pattern) {
case all:
value = i%mpi_size + 1;
break;
case none:
value = 0;
break;
case sec_last:
if (i==nchunks-2)
value = 100;
else
value = 0;
}
/* verify content of the chunk */
for (index = 0; index < CHUNK_SIZE; index++)
VRFY((buffer[index] == value), "data verification");
}
hrc = H5Sclose (dataspace);
VRFY((hrc >= 0), "");
hrc = H5Sclose (memspace);
VRFY((hrc >= 0), "");
/* Can close some plists */
hrc = H5Pclose(access_plist);
VRFY((hrc >= 0), "");
/* Close up */
if (close){
hrc = H5Dclose(*dataset);
VRFY((hrc >= 0), "");
*dataset = -1;
hrc = H5Fclose(*file_id);
VRFY((hrc >= 0), "");
*file_id = -1;
}
/* Make sure all processes are done before exiting this routine. Otherwise,
* other tests may start and change the test data file before some processes
* of this test are still accessing the file.
*/
MPI_Barrier(MPI_COMM_WORLD);
}
void BAGDataset::LoadMetadata()
{
// Load the metadata from the file.
const hid_t hMDDS = H5Dopen(hHDF5, "/BAG_root/metadata");
const hid_t datatype = H5Dget_type(hMDDS);
const hid_t dataspace = H5Dget_space(hMDDS);
const hid_t native = H5Tget_native_type(datatype, H5T_DIR_ASCEND);
hsize_t dims[3] = {
static_cast<hsize_t>(0),
static_cast<hsize_t>(0),
static_cast<hsize_t>(0)
};
hsize_t maxdims[3] = {
static_cast<hsize_t>(0),
static_cast<hsize_t>(0),
static_cast<hsize_t>(0)
};
H5Sget_simple_extent_dims(dataspace, dims, maxdims);
pszXMLMetadata =
static_cast<char *>(CPLCalloc(static_cast<int>(dims[0] + 1), 1));
H5Dread(hMDDS, native, H5S_ALL, dataspace, H5P_DEFAULT, pszXMLMetadata);
H5Tclose(native);
H5Sclose(dataspace);
H5Tclose(datatype);
H5Dclose(hMDDS);
if( strlen(pszXMLMetadata) == 0 )
return;
// Try to get the geotransform.
CPLXMLNode *psRoot = CPLParseXMLString(pszXMLMetadata);
if( psRoot == nullptr )
return;
CPLStripXMLNamespace(psRoot, nullptr, TRUE);
CPLXMLNode *const psGeo = CPLSearchXMLNode(psRoot, "=MD_Georectified");
if( psGeo != nullptr )
{
char **papszCornerTokens = CSLTokenizeStringComplex(
CPLGetXMLValue(psGeo, "cornerPoints.Point.coordinates", ""), " ,",
FALSE, FALSE);
if( CSLCount(papszCornerTokens) == 4 )
{
const double dfLLX = CPLAtof(papszCornerTokens[0]);
const double dfLLY = CPLAtof(papszCornerTokens[1]);
const double dfURX = CPLAtof(papszCornerTokens[2]);
const double dfURY = CPLAtof(papszCornerTokens[3]);
adfGeoTransform[0] = dfLLX;
adfGeoTransform[1] = (dfURX - dfLLX) / (GetRasterXSize() - 1);
adfGeoTransform[3] = dfURY;
adfGeoTransform[5] = (dfLLY - dfURY) / (GetRasterYSize() - 1);
adfGeoTransform[0] -= adfGeoTransform[1] * 0.5;
adfGeoTransform[3] -= adfGeoTransform[5] * 0.5;
}
CSLDestroy(papszCornerTokens);
}
// Try to get the coordinate system.
OGRSpatialReference oSRS;
if( OGR_SRS_ImportFromISO19115(&oSRS, pszXMLMetadata) == OGRERR_NONE )
{
oSRS.exportToWkt(&pszProjection);
}
else
{
ParseWKTFromXML(pszXMLMetadata);
}
// Fetch acquisition date.
CPLXMLNode *const psDateTime = CPLSearchXMLNode(psRoot, "=dateTime");
if( psDateTime != nullptr )
{
const char *pszDateTimeValue = CPLGetXMLValue(psDateTime, nullptr, "");
if( pszDateTimeValue )
SetMetadataItem("BAG_DATETIME", pszDateTimeValue);
}
CPLDestroyXMLNode(psRoot);
}
int main(int argc, char *argv[]) {
herr_t err = 0;
int n_threads = omp_get_max_threads();
hid_t kernel_file_id = 0;
hid_t levy_basis_file_id = 0;
hid_t levy_basis_dataset_id = 0;
hid_t levy_basis_dataspace_id = 0;
hid_t output_file_id = 0;
hid_t output_dataset_id = 0;
hid_t output_dataspace_id = 0;
hid_t memspace = 0;
hsize_t n_k = 0;
double *tmp = NULL;
double *k_abscissa = NULL;
double *k_ordinate = NULL;
double *x1 = NULL;
double *x2 = NULL;
double *x3 = NULL;
DEBUGPRINT("### Parsing arguments");
struct arguments args;
initialise_arguments(&args);
argp_parse (&argp, argc, argv, 0, 0, &args);
#ifdef DEBUG
print_arguments(&args);
#endif
DEBUGPRINT("### Reading kernel");
kernel_file_id = H5Fopen(args.kernel_file, H5F_ACC_RDONLY, H5P_DEFAULT);
if (kernel_file_id <= 0) {
printf("Error: Could not open \"%s\".\n", args.kernel_file);
err = -1;
goto cleanup;
}
err = H5LTget_dataset_info(kernel_file_id, "/abscissa", &n_k, NULL, NULL);
if (err < 0) {
printf("Error: Could not read dataset info.\n");
goto cleanup;
}
printf("n_k = %i\n", (int)n_k);
k_abscissa = malloc(n_k * sizeof(double));
k_ordinate = malloc(n_k * sizeof(double));
err = H5LTread_dataset_double(kernel_file_id, "/abscissa", k_abscissa);
err = H5LTread_dataset_double(kernel_file_id, "/ordinate", k_ordinate);
DEBUGPRINT("### Reading Levy basis");
hsize_t dims[4];
hsize_t offset[4];
hsize_t count[4];
levy_basis_file_id = H5Fopen(args.levy_basis_file, H5F_ACC_RDONLY, H5P_DEFAULT);
levy_basis_dataset_id = H5Dopen(levy_basis_file_id, "/levy_basis_realization", H5P_DEFAULT);
levy_basis_dataspace_id = H5Dget_space(levy_basis_dataset_id);
err = H5Sget_simple_extent_dims(levy_basis_dataspace_id, dims, NULL);
if (dims[0] != dims[1] || dims[1] != dims[2]) {
printf("Error: The three dimensions must be equal.\n");
err = -1;
goto cleanup;
}
hsize_t dims_pad[3];
dims_pad[0] = dims[0];
dims_pad[1] = dims[1];
dims_pad[2] = 2 * (dims[2] / 2 + 1);
hsize_t n_x = dims_pad[0] * dims_pad[1] * dims_pad[2];
x1 = malloc(n_x * sizeof(double));
x2 = malloc(n_x * sizeof(double));
x3 = malloc(n_x * sizeof(double));
double *x[] = {x1, x2, x3};
if (!x1 || !x2 || !x3) {
printf("Error: Could not allocate memory for the Levy basis.\n");
err = -1;
goto cleanup;
}
#pragma omp parallel for
for (ptrdiff_t i = 0; i < n_x; i++) {
x1[i] = 0.0;
x2[i] = 0.0;
x3[i] = 0.0;
}
/* Define memory dataspace */
memspace = H5Screate_simple(3, dims_pad, NULL);
offset[0] = offset[1] = offset[2] = 0;
count[0] = dims[0];
count[1] = dims[1];
count[2] = dims[2];
/* Define hyperslab in the memory dataspace */
err = H5Sselect_hyperslab(memspace, H5S_SELECT_SET, offset, NULL, count, NULL);
for (int j = 0; j < 3; j++) {
/* Define hyperslap in the file dataspace */
offset[3] = j;
count[3] = 1;
err = H5Sselect_hyperslab(levy_basis_dataspace_id, H5S_SELECT_SET, offset, NULL, count, NULL);
/* Read data from hyperslab */
err = H5Dread(levy_basis_dataset_id, H5T_NATIVE_DOUBLE,
memspace, levy_basis_dataspace_id,
H5P_DEFAULT, x[j]);
if (err < 0) {
printf("Error: Could not read hyperslab.\n");
err = -1;
goto cleanup;
}
}
DEBUGPRINT("### Convolving");
double delta = 2.0 * M_PI / dims[0];
err = ambit_symmetric_odd_isotropic_circular_convolution_inplace(
n_threads, n_k, k_abscissa, k_ordinate, dims[0], delta, x1, x2, x3);
if (err) {
printf("Error in ambit_symmetric_odd_isotropic_circular_convolution_inplace.\n");
goto cleanup;
}
DEBUGPRINT("### Writing output");
output_file_id = H5Fcreate(args.output_file, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
if (output_file_id < 0) {
printf("Error: Could not open \"%s\".\n", args.output_file);
err = -1;
goto cleanup;
}
output_dataspace_id = H5Screate_simple(4, dims, NULL);
output_dataset_id = H5Dcreate(output_file_id, "/simulation", H5T_NATIVE_DOUBLE, output_dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
for (int j = 0; j < 3; j++) {
printf("j = %i\n", j);
/* Define hyperslap in the file dataspace */
offset[3] = j;
count[3] = 1;
err = H5Sselect_hyperslab(output_dataspace_id, H5S_SELECT_SET, offset, NULL, count, NULL);
/* Write data to hyperslab */
err = H5Dwrite(output_dataset_id, H5T_NATIVE_DOUBLE,
memspace, output_dataspace_id,
H5P_DEFAULT, x[j]);
if (err < 0) {
printf("Error: Could not write hyperslab.\n");
err = -1;
goto cleanup;
}
}
cleanup:
if (memspace > 0) H5Sclose(memspace);
if (output_dataspace_id > 0) H5Sclose(output_dataspace_id);
if (output_dataset_id > 0) H5Dclose(output_dataset_id);
if (output_file_id > 0) H5Fclose(output_file_id);
if (levy_basis_dataspace_id > 0) H5Sclose(levy_basis_dataspace_id);
if (levy_basis_dataset_id > 0) H5Dclose(levy_basis_dataset_id);
if (levy_basis_file_id > 0) H5Fclose(levy_basis_file_id);
if (kernel_file_id > 0) H5Fclose(kernel_file_id);
free(tmp);
free(k_abscissa);
free(k_ordinate);
free(x1);
free(x2);
free(x3);
return err;
}
herr_t H5IMget_palette( hid_t loc_id,
const char *image_name,
int pal_number,
unsigned char *pal_data )
{
hid_t did;
int has_pal;
hid_t atid=-1;
hid_t aid;
hid_t asid=-1;
hssize_t n_refs;
hsize_t dim_ref;
hobj_ref_t *refbuf; /* buffer to read references */
hid_t pal_id;
/* check the arguments */
if (image_name == NULL)
return -1;
if (pal_data == NULL)
return -1;
/* Open the dataset. */
if((did = H5Dopen2(loc_id, image_name, H5P_DEFAULT)) < 0)
return -1;
/* Try to find the attribute "PALETTE" on the >>image<< dataset */
has_pal = H5IM_find_palette(did);
if(has_pal == 1 )
{
if((aid = H5Aopen(did, "PALETTE", H5P_DEFAULT)) < 0)
goto out;
if((atid = H5Aget_type(aid)) < 0)
goto out;
if(H5Tget_class(atid) < 0)
goto out;
/* Get the reference(s) */
if((asid = H5Aget_space(aid)) < 0)
goto out;
n_refs = H5Sget_simple_extent_npoints(asid);
dim_ref = (hsize_t)n_refs;
refbuf = (hobj_ref_t*)HDmalloc( sizeof(hobj_ref_t) * (size_t)dim_ref );
if ( H5Aread( aid, atid, refbuf ) < 0)
goto out;
/* Get the palette id */
if ( (pal_id = H5Rdereference( did, H5R_OBJECT, &refbuf[pal_number] )) < 0)
goto out;
/* Read the palette dataset */
if ( H5Dread( pal_id, H5Dget_type(pal_id), H5S_ALL, H5S_ALL, H5P_DEFAULT, pal_data ) < 0)
goto out;
/* close */
if (H5Dclose(pal_id)<0)
goto out;
if ( H5Sclose( asid ) < 0)
goto out;
if ( H5Tclose( atid ) < 0)
goto out;
if ( H5Aclose( aid ) < 0)
goto out;
HDfree( refbuf );
}
/* Close the image dataset. */
if ( H5Dclose( did ) < 0)
return -1;
return 0;
out:
H5Dclose( did );
H5Sclose( asid );
H5Tclose( atid );
H5Aclose( aid );
return -1;
}
int main(int argc, char **argv)
{
int sub_div[3], local_offset[3], count_local[3];
int t = 0, time_count = 0;
sub_div[0] = 0;
sub_div[1] = 0;
sub_div[2] = 0;
local_offset[0] = 0;
local_offset[1] = 0;
local_offset[2] = 0;
count_local[0] = 0;
count_local[1] = 0;
count_local[2] = 0;
#if HDF_IO
hid_t file_id;
hid_t plist_id;
hid_t group_id;
hid_t dataset_id;
hid_t file_dataspace;
hid_t mem_dataspace;
#endif
#if PIDX_IO
PIDX_file file;
PIDX_access access;
PIDX_variable variable;
#endif
const int bits_per_block = 15;
const int blocks_per_file = 512;
int nprocs=1, rank=0, slice;
MPI_Comm comm = MPI_COMM_WORLD;
#if HDF_IO
MPI_Info info = MPI_INFO_NULL;
#endif
MPI_Init(&argc, &argv);
MPI_Comm_size(comm, &nprocs);
MPI_Comm_rank(comm, &rank);
output_file_name = (char*) malloc(sizeof (char) * 1024);
sprintf(output_file_name, "%s%s", "/scratch/project/visus/datasets/flame", ".idx");
if (nprocs == 320) /// 4 x 5 x 16
{
count_local[0] = 256;
count_local[1] = 188;
count_local[2] = 192;
}
else if (nprocs == 640) /// 4 x 5 x 32
{
count_local[0] = 256;
count_local[1] = 188;
count_local[2] = 96;
}
else if (nprocs == 1280) /// 4 x 10 x 32
{
count_local[0] = 256;
count_local[1] = 94;
count_local[2] = 96;
}
else if (nprocs == 2560) /// 8 x 10 x 32
{
count_local[0] = 128;
count_local[1] = 94;
count_local[2] = 96;
}
sub_div[0] = (1024 / count_local[0]);
sub_div[1] = (940 / count_local[1]);
sub_div[2] = (3072 / count_local[2]);
local_offset[2] = (rank / (sub_div[0] * sub_div[1])) * count_local[2];
slice = rank % (sub_div[0] * sub_div[1]);
local_offset[1] = (slice / sub_div[0]) * count_local[1];
local_offset[0] = (slice % sub_div[0]) * count_local[0];
#if PIDX_IO
PIDX_point global_bounding_box, local_offset_point, local_box_count_point;
PIDX_set_point_5D(global_bounding_box, (int64_t)1024, (int64_t)940, (int64_t)3072, 1, 1);
PIDX_set_point_5D(local_offset_point, (int64_t)local_offset[0], (int64_t)local_offset[1], (int64_t)local_offset[2], 0, 0);
PIDX_set_point_5D(local_box_count_point, (int64_t)count_local[0], (int64_t)count_local[1], (int64_t)count_local[2], 1, 1);
#endif
#if HDF_IO
hsize_t count[3];
count[0] = count_local[0];
count[1] = count_local[1];
count[2] = count_local[2];
hsize_t offset[3];
offset[0] = local_offset[0];
offset[1] = local_offset[1];
offset[2] = local_offset[2];
#endif
/// P U V W ZMIX
#if HDF_IO
plist_id = H5Pcreate(H5P_FILE_ACCESS);
H5Pset_fapl_mpio(plist_id, comm, info);
#endif
FILE *fp;
fp = fopen("list", "r");
char file_name[1][1024];
while (!feof(fp))
{
if (fscanf(fp, "%s", file_name[time_count]) != 1)
break;
if(rank == 0)
printf("%s\n", file_name[time_count]);
time_count++;
}
fclose(fp);
#if PIDX_IO
PIDX_create_access(&access);
PIDX_set_mpi_access(access, 1, 1, 1, comm);
PIDX_set_process_extent(access, sub_div[0], sub_div[1], sub_div[2]);
#endif
if (rank == 0)
printf("Number of timesteps = %d\n", time_count);
for (t = 0; t < time_count; t++)
{
#if HDF_IO
file_id = H5Fopen(file_name[t], H5F_ACC_RDONLY, plist_id);
#endif
#if PIDX_IO
PIDX_file_create(output_file_name, PIDX_file_trunc, access, &file);
PIDX_set_dims(file, global_bounding_box);
PIDX_set_current_time_step(file, t);
PIDX_set_block_size(file, bits_per_block);
PIDX_set_aggregation_factor(file, 1);
PIDX_set_block_count(file, blocks_per_file);
PIDX_set_variable_count(file, 1);
PIDX_debug_rst(file, 0);
PIDX_debug_hz(file, 0);
PIDX_dump_agg_info(file, 0);
PIDX_enable_hz(file, 1);
PIDX_enable_agg(file, 1);
PIDX_enable_io(file, 1);
#endif
#if HDF_IO
group_id = H5Gopen(file_id, "/data", H5P_DEFAULT);
dataset_id = H5Dopen2(group_id, "P", H5P_DEFAULT);
mem_dataspace = H5Screate_simple (3, count, NULL);
file_dataspace = H5Dget_space (dataset_id);
H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, offset, NULL, count, NULL);
#endif
buffer = malloc(sizeof(double) * count_local[0] * count_local[1] * count_local[2]);
memset(buffer, 0, sizeof(double) * count_local[0] * count_local[1] * count_local[2]);
#if HDF_IO
H5Dread(dataset_id, H5T_NATIVE_DOUBLE, mem_dataspace, file_dataspace, H5P_DEFAULT, buffer);
H5Sclose(mem_dataspace);
H5Sclose(file_dataspace);
H5Dclose(dataset_id);
H5Gclose(group_id);
H5Fclose(file_id);
if (rank == 0)
printf("Finished reading HDF files\n");
#else
int64_t i1, j1, k1;
for (k1 = 0; k1 < count_local[2]; k1++)
for (j1 = 0; j1 < count_local[1]; j1++)
for (i1 = 0; i1 < count_local[0]; i1++)
{
int64_t index = (int64_t) (count_local[0] * count_local[1] * k1) + (count_local[0] * j1) + i1;
buffer[index] = (256 * rank) / nprocs;
}
#endif
#if PIDX_IO
PIDX_variable_create(file, "P", sizeof(double) * 8, "1*float64", &variable);
PIDX_append_and_write_variable(variable, local_offset_point, local_box_count_point, buffer, PIDX_column_major);
if (rank == 0)
printf("Starting to Write IDX %s Time Step %d\n", output_file_name, t);
PIDX_close(file);
#endif
free(buffer);
buffer = 0;
}
//////////
#if HDF_IO
H5Pclose(plist_id);
#endif
#if PIDX_IO
PIDX_close_access(access);
#endif
free(output_file_name);
output_file_name = 0;
MPI_Finalize();
return 0;
}
int do_copy_refobjs(hid_t fidin,
hid_t fidout,
trav_table_t *travt,
pack_opt_t *options) /* repack options */
{
hid_t grp_in = (-1); /* read group ID */
hid_t grp_out = (-1); /* write group ID */
hid_t dset_in = (-1); /* read dataset ID */
hid_t dset_out = (-1); /* write dataset ID */
hid_t type_in = (-1); /* named type ID */
hid_t dcpl_id = (-1); /* dataset creation property list ID */
hid_t space_id = (-1); /* space ID */
hid_t ftype_id = (-1); /* file data type ID */
hid_t mtype_id = (-1); /* memory data type ID */
size_t msize; /* memory size of memory type */
hsize_t nelmts; /* number of elements in dataset */
int rank; /* rank of dataset */
hsize_t dims[H5S_MAX_RANK]; /* dimensions of dataset */
unsigned int i, j;
int k;
named_dt_t *named_dt_head=NULL; /* Pointer to the stack of named datatypes
copied */
/*-------------------------------------------------------------------------
* browse
*-------------------------------------------------------------------------
*/
for(i = 0; i < travt->nobjs; i++) {
switch(travt->objs[i].type)
{
/*-------------------------------------------------------------------------
* H5TRAV_TYPE_GROUP
*-------------------------------------------------------------------------
*/
case H5TRAV_TYPE_GROUP:
/*-------------------------------------------------------------------------
* copy referenced objects in attributes
*-------------------------------------------------------------------------
*/
if((grp_out = H5Gopen2(fidout, travt->objs[i].name, H5P_DEFAULT)) < 0)
goto error;
if((grp_in = H5Gopen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0)
goto error;
if(copy_refs_attr(grp_in, grp_out, options, travt, fidout) < 0)
goto error;
if(H5Gclose(grp_out) < 0)
goto error;
if(H5Gclose(grp_in) < 0)
goto error;
/*-------------------------------------------------------------------------
* check for hard links
*-------------------------------------------------------------------------
*/
if(travt->objs[i].nlinks)
for(j = 0; j < travt->objs[i].nlinks; j++)
H5Lcreate_hard(fidout, travt->objs[i].name, H5L_SAME_LOC, travt->objs[i].links[j].new_name, H5P_DEFAULT, H5P_DEFAULT);
break;
/*-------------------------------------------------------------------------
* H5TRAV_TYPE_DATASET
*-------------------------------------------------------------------------
*/
case H5TRAV_TYPE_DATASET:
if((dset_in = H5Dopen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0)
goto error;
if((space_id = H5Dget_space(dset_in)) < 0)
goto error;
if((ftype_id = H5Dget_type(dset_in)) < 0)
goto error;
if((dcpl_id = H5Dget_create_plist(dset_in)) < 0)
goto error;
if((rank = H5Sget_simple_extent_ndims(space_id)) < 0)
goto error;
if(H5Sget_simple_extent_dims(space_id, dims, NULL) < 0)
goto error;
nelmts = 1;
for(k = 0; k < rank; k++)
nelmts *= dims[k];
if((mtype_id = h5tools_get_native_type(ftype_id)) < 0)
goto error;
if((msize = H5Tget_size(mtype_id)) == 0)
goto error;
/*-------------------------------------------------------------------------
* check if the dataset creation property list has filters that
* are not registered in the current configuration
* 1) the external filters GZIP and SZIP might not be available
* 2) the internal filters might be turned off
*-------------------------------------------------------------------------
*/
if(h5tools_canreadf(NULL, dcpl_id) == 1) {
/*-------------------------------------------------------------------------
* test for a valid output dataset
*-------------------------------------------------------------------------
*/
dset_out = FAIL;
/*-------------------------------------------------------------------------
* object references are a special case
* we cannot just copy the buffers, but instead we recreate the reference
*-------------------------------------------------------------------------
*/
if(H5Tequal(mtype_id, H5T_STD_REF_OBJ)) {
hid_t refobj_id;
hobj_ref_t *refbuf = NULL; /* buffer for object references */
hobj_ref_t *buf = NULL;
const char* refname;
unsigned u;
/*-------------------------------------------------------------------------
* read to memory
*-------------------------------------------------------------------------
*/
if(nelmts) {
buf = (hobj_ref_t *)HDmalloc((unsigned)(nelmts * msize));
if(buf==NULL) {
printf("cannot read into memory\n" );
goto error;
} /* end if */
if(H5Dread(dset_in, mtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0)
goto error;
refbuf = (hobj_ref_t*) HDcalloc((unsigned)nelmts, msize);
if(refbuf == NULL){
printf("cannot allocate memory\n" );
goto error;
} /* end if */
for(u = 0; u < nelmts; u++) {
H5E_BEGIN_TRY {
if((refobj_id = H5Rdereference(dset_in, H5R_OBJECT, &buf[u])) < 0)
continue;
} H5E_END_TRY;
/* get the name. a valid name could only occur
* in the second traversal of the file
*/
if((refname = MapIdToName(refobj_id, travt)) != NULL) {
/* create the reference, -1 parameter for objects */
if(H5Rcreate(&refbuf[u], fidout, refname, H5R_OBJECT, -1) < 0)
goto error;
if(options->verbose)
{
printf(FORMAT_OBJ,"dset",travt->objs[i].name );
printf("object <%s> object reference created to <%s>\n",
travt->objs[i].name,
refname);
}
} /*refname*/
H5Oclose(refobj_id);
} /* u */
} /*nelmts*/
/*-------------------------------------------------------------------------
* create/write dataset/close
*-------------------------------------------------------------------------
*/
if((dset_out = H5Dcreate2(fidout, travt->objs[i].name, mtype_id, space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT)) < 0)
goto error;
if(nelmts)
if(H5Dwrite(dset_out, mtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, refbuf) < 0)
goto error;
if(buf)
HDfree(buf);
if(refbuf)
HDfree(refbuf);
/*------------------------------------------------------
* copy attrs
*----------------------------------------------------*/
if(copy_attr(dset_in, dset_out, &named_dt_head, travt, options) < 0)
goto error;
} /*H5T_STD_REF_OBJ*/
/*-------------------------------------------------------------------------
* dataset region references
*-------------------------------------------------------------------------
*/
else if(H5Tequal(mtype_id, H5T_STD_REF_DSETREG))
{
hid_t refobj_id;
hdset_reg_ref_t *refbuf = NULL; /* input buffer for region references */
hdset_reg_ref_t *buf = NULL; /* output buffer */
const char* refname;
unsigned u;
/*-------------------------------------------------------------------------
* read input to memory
*-------------------------------------------------------------------------
*/
if(nelmts) {
buf = (hdset_reg_ref_t *)HDmalloc((unsigned)(nelmts * msize));
if(buf == NULL) {
printf("cannot read into memory\n");
goto error;
} /* end if */
if(H5Dread(dset_in, mtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0)
goto error;
/*-------------------------------------------------------------------------
* create output
*-------------------------------------------------------------------------
*/
refbuf = (hdset_reg_ref_t *)HDcalloc(sizeof(hdset_reg_ref_t), (size_t)nelmts); /*init to zero */
if(refbuf == NULL) {
printf("cannot allocate memory\n");
goto error;
} /* end if */
for(u = 0; u < nelmts; u++) {
H5E_BEGIN_TRY {
if((refobj_id = H5Rdereference(dset_in, H5R_DATASET_REGION, &buf[u])) < 0)
continue;
} H5E_END_TRY;
/* get the name. a valid name could only occur
* in the second traversal of the file
*/
if((refname = MapIdToName(refobj_id, travt)) != NULL) {
hid_t region_id; /* region id of the referenced dataset */
if((region_id = H5Rget_region(dset_in, H5R_DATASET_REGION, &buf[u])) < 0)
goto error;
/* create the reference, we need the space_id */
if(H5Rcreate(&refbuf[u], fidout, refname, H5R_DATASET_REGION, region_id) < 0)
goto error;
if(H5Sclose(region_id) < 0)
goto error;
if(options->verbose)
{
printf(FORMAT_OBJ,"dset",travt->objs[i].name );
printf("object <%s> region reference created to <%s>\n",
travt->objs[i].name,
refname);
}
} /*refname*/
H5Oclose(refobj_id);
} /* u */
} /*nelmts*/
/*-------------------------------------------------------------------------
* create/write dataset/close
*-------------------------------------------------------------------------
*/
if((dset_out = H5Dcreate2(fidout, travt->objs[i].name, mtype_id, space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT)) < 0)
goto error;
if(nelmts)
if(H5Dwrite(dset_out, mtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, refbuf) < 0)
goto error;
if(buf)
HDfree(buf);
if(refbuf)
HDfree(refbuf);
/*-----------------------------------------------------
* copy attrs
*----------------------------------------------------*/
if(copy_attr(dset_in, dset_out, &named_dt_head, travt, options) < 0)
goto error;
} /* H5T_STD_REF_DSETREG */
/*-------------------------------------------------------------------------
* not references, open previously created object in 1st traversal
*-------------------------------------------------------------------------
*/
else {
if((dset_out = H5Dopen2(fidout, travt->objs[i].name, H5P_DEFAULT)) < 0)
goto error;
} /* end else */
assert(dset_out != FAIL);
/*-------------------------------------------------------------------------
* copy referenced objects in attributes
*-------------------------------------------------------------------------
*/
if(copy_refs_attr(dset_in, dset_out, options, travt, fidout) < 0)
goto error;
/*-------------------------------------------------------------------------
* check for hard links
*-------------------------------------------------------------------------
*/
if(travt->objs[i].nlinks)
for(j = 0; j < travt->objs[i].nlinks; j++)
H5Lcreate_hard(fidout, travt->objs[i].name, H5L_SAME_LOC, travt->objs[i].links[j].new_name, H5P_DEFAULT, H5P_DEFAULT);
if(H5Dclose(dset_out) < 0)
goto error;
} /*can_read*/
void hdf5dataset::read(void* data)
{
herr_t status = H5Dread(_dataset_id, _type_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, data);
if(status < 0)
throw std::runtime_error("Failed to read dataset (" + _hfile.filename() + ") in file (" + _name + ")");
}
int read_hdf5_buffered(char *file_name, int buffer_size) {
double sum = 0.0;
double *buffer;
long i;
hid_t file_id, dataset_id, dataspace_id, memspace_id;
hsize_t dataspace_dims[1], offset[1], count[0], mem_offset[1],
mem_count[1], remainder;
size_t buffer_bytes = (buffer_size/sizeof(double))*sizeof(double);
if (buffer_size % sizeof(double) != 0) {
warnx("buffer size is not a multiple of double precsion type size");
}
buffer_size = buffer_bytes/sizeof(double);
if ((buffer = (double *) malloc(buffer_bytes)) == NULL) {
errx(EXIT_FAILURE, "can't allocate buffer of size %d",
buffer_bytes);
}
printf("%d, %d\n", buffer_size, buffer_bytes);
file_id = H5Fopen(file_name, H5F_ACC_RDONLY, H5P_DEFAULT);
dataset_id = H5Dopen(file_id, "/values", H5P_DEFAULT);
dataspace_id = H5Dget_space(dataset_id);
H5Sget_simple_extent_dims(dataspace_id, dataspace_dims, NULL);
mem_count[0] = buffer_size;
memspace_id = H5Screate_simple(1, mem_count, NULL);
mem_offset[0] = 0;
H5Sselect_hyperslab(memspace_id, H5S_SELECT_SET, mem_offset, NULL,
mem_count, NULL);
count[0] = buffer_size;
for (offset[0] = 0; offset[0] < dataspace_dims[0] - buffer_size;
offset[0] += buffer_size) {
H5Sselect_hyperslab(dataspace_id, H5S_SELECT_SET, offset, NULL,
count, NULL);
H5Dread(dataset_id, H5T_NATIVE_DOUBLE, memspace_id,
dataspace_id, H5P_DEFAULT, buffer);
for (i = 0; i < buffer_size; i++) {
sum += buffer[i];
}
}
remainder = dataspace_dims[0] % buffer_size;
if (remainder != 0) {
mem_count[0] = remainder;
H5Sselect_hyperslab(memspace_id, H5S_SELECT_SET, mem_offset, NULL,
mem_count, NULL);
offset[0] = dataspace_dims[0] - remainder;
count[0] = remainder;
H5Sselect_hyperslab(dataspace_id, H5S_SELECT_SET, offset, NULL,
count, NULL);
H5Dread(dataset_id, H5T_NATIVE_DOUBLE, memspace_id,
dataspace_id, H5P_DEFAULT, buffer);
for (i = 0; i < remainder; i++) {
sum += buffer[i];
}
}
H5Dclose(dataset_id);
H5Sclose(dataspace_id);
H5Sclose(memspace_id);
H5Fclose(file_id);
fprintf(stderr, "%lu double precision numbers read, sum = %le\n",
(long unsigned) dataspace_dims[0], sum);
free(buffer);
return EXIT_SUCCESS;
}
static int read_data(char *fname)
{
char pathname[1024];
char *srcdir = getenv("srcdir"); /*where the src code is located*/
hid_t file, dataset; /* handles */
hid_t datatype;
hid_t dt;
double data_in[NX][NY]; /* input buffer */
double data_out[NX][NY]; /* output buffer */
int i, j;
unsigned nerrors = 0;
pathname[0] = '\0';
/* Generate correct name for test file by prepending the source path */
if(srcdir && ((strlen(srcdir) + strlen(fname) + 1) < sizeof(pathname))) {
strcpy(pathname, srcdir);
strcat(pathname, "/");
}
strcat(pathname, fname);
/*
* Data and output buffer initialization.
*/
for (j = 0; j < NX; j++) {
for (i = 0; i < NY; i++) {
data_in[j][i] = i + j;
data_out[j][i] = 0;
}
}
/*
* 0 1 2 3 4 5
* 1 2 3 4 5 6
* 2 3 4 5 6 7
* 3 4 5 6 7 8
* 4 5 6 7 8 9
*/
/*
* Open the file and the dataset.
*/
if((file = H5Fopen(pathname, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0)
TEST_ERROR;
if((dataset = H5Dopen2(file, DATASETNAME, H5P_DEFAULT)) < 0)
TEST_ERROR;
/*
* Get datatype and dataspace handles and then query
* dataset class, order, size, rank and dimensions.
*/
if((dt = H5Dget_type(dataset)) < 0) /* datatype handle */
TEST_ERROR;
if((datatype = H5Tget_native_type(dt, H5T_DIR_DEFAULT)) < 0)
TEST_ERROR;
/*
* Read data from hyperslab in the file into the hyperslab in
* memory and display.
*/
if(H5Dread(dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data_out) < 0)
TEST_ERROR;
/* Check results */
for (j=0; j<NX; j++) {
for (i=0; i<NY; i++) {
if (data_out[j][i] != data_in[j][i]) {
if (!nerrors++) {
H5_FAILED();
printf("element [%d][%d] is %g but should have been %g\n",
j, i, data_out[j][i], data_in[j][i]);
}
}
}
}
/*
* Close/release resources.
*/
H5Tclose(dt);
H5Tclose(datatype);
H5Dclose(dataset);
H5Fclose(file);
/* Failure */
if (nerrors) {
printf("total of %d errors out of %d elements\n", nerrors, NX*NY);
return 1;
}
PASSED();
return 0;
error:
H5E_BEGIN_TRY {
H5Fclose(file);
} H5E_END_TRY;
return 1;
}
CPLErr HDF5ImageRasterBand::IReadBlock( int nBlockXOff, int nBlockYOff,
void * pImage )
{
herr_t status;
hsize_t count[3];
H5OFFSET_TYPE offset[3];
int nSizeOfData;
hid_t memspace;
hsize_t col_dims[3];
hsize_t rank;
HDF5ImageDataset *poGDS = ( HDF5ImageDataset * ) poDS;
if( poGDS->eAccess == GA_Update ) {
memset( pImage, 0,
nBlockXSize * nBlockYSize *
GDALGetDataTypeSize( eDataType )/8 );
return CE_None;
}
rank=2;
if( poGDS->ndims == 3 ){
rank=3;
offset[0] = nBand-1;
count[0] = 1;
col_dims[0] = 1;
}
offset[poGDS->ndims - 2] = nBlockYOff*nBlockYSize;
offset[poGDS->ndims - 1] = nBlockXOff*nBlockXSize;
count[poGDS->ndims - 2] = nBlockYSize;
count[poGDS->ndims - 1] = nBlockXSize;
nSizeOfData = H5Tget_size( poGDS->native );
memset( pImage,0,nBlockXSize*nBlockYSize*nSizeOfData );
/* blocksize may not be a multiple of imagesize */
count[poGDS->ndims - 2] = MIN( size_t(nBlockYSize),
poDS->GetRasterYSize() -
offset[poGDS->ndims - 2]);
count[poGDS->ndims - 1] = MIN( size_t(nBlockXSize),
poDS->GetRasterXSize()-
offset[poGDS->ndims - 1]);
/* -------------------------------------------------------------------- */
/* Select block from file space */
/* -------------------------------------------------------------------- */
status = H5Sselect_hyperslab( poGDS->dataspace_id,
H5S_SELECT_SET,
offset, NULL,
count, NULL );
/* -------------------------------------------------------------------- */
/* Create memory space to receive the data */
/* -------------------------------------------------------------------- */
col_dims[poGDS->ndims-2]=nBlockYSize;
col_dims[poGDS->ndims-1]=nBlockXSize;
memspace = H5Screate_simple( (int) rank, col_dims, NULL );
H5OFFSET_TYPE mem_offset[3] = {0, 0, 0};
status = H5Sselect_hyperslab(memspace,
H5S_SELECT_SET,
mem_offset, NULL,
count, NULL);
status = H5Dread ( poGDS->dataset_id,
poGDS->native,
memspace,
poGDS->dataspace_id,
H5P_DEFAULT,
pImage );
H5Sclose( memspace );
if( status < 0 )
{
CPLError( CE_Failure, CPLE_AppDefined,
"H5Dread() failed for block." );
return CE_Failure;
}
else
return CE_None;
}
int VsReader::getData( VsDataset* dataSet,
void* data,
// Use these variables for adjusting
// the read memory space.
std::string indexOrder, // Index ordering
int components, // Index for a component
int* srcMins, // start locations
int* srcCounts, // number of entries
int* srcStrides, // stride in memory
// Use these variables for adjusting
// the write memory space.
int mdims, // spatial dims (rank)
int* destSizes, // overall memory size
int* destMins, // start locations
int* destCounts, // number of entries
int* destStrides ) const // stride in memory
{
// components = -2 No component array (i.e. scalar variable)
// components = -1 Component array present but read all values.
// (i.e. vector variable or point coordinates)
// component >= 0 Component array present read single value at that index.
if (dataSet == NULL) {
VsLog::debugLog() << __CLASS__ << __FUNCTION__ << " " << __LINE__ << " "
<< "Requested dataset is null?" << std::endl;
return -1;
}
if (data == NULL) {
VsLog::debugLog() << __CLASS__ << __FUNCTION__ << " " << __LINE__ << " "
<< "Data storage is null?" << std::endl;
return -1;
}
int err = 0;
// No index ordering info so read all data.
if (indexOrder.length() == 0)
{
err = H5Dread(dataSet->getId(), dataSet->getType(),
H5S_ALL, H5S_ALL, H5P_DEFAULT, data);
if (err < 0) {
VsLog::debugLog() << __CLASS__ << __FUNCTION__ << " " << __LINE__ << " "
<< "Error " << err
<< " in reading variable " << dataSet->getFullName()
<< "." << std::endl;
return err;
}
}
// mins, counts, and/or strides so hyperslab.
else
{
hid_t dataspace = H5Dget_space(dataSet->getId());
int ndims = H5Sget_simple_extent_ndims(dataspace);
if (ndims == 0) {
VsLog::debugLog() << __CLASS__ << __FUNCTION__ << " " << __LINE__ << " "
<< "Unable to load dimensions for variable."
<< "Returning -1." << std::endl;
return -1;
}
std::vector<hsize_t> dims(ndims);
int ndim = H5Sget_simple_extent_dims(dataspace, &(dims[0]), NULL);
if( ndim != ndims ) {
VsLog::debugLog() << __CLASS__ << __FUNCTION__ << " " << __LINE__ << " "
<< "Data dimensions not match. " << std::endl;
return -1;
}
VsLog::debugLog() << __CLASS__ << __FUNCTION__ << " " << __LINE__ << " "
<< "about to set up arguments." << std::endl;
std::vector<hsize_t> count(ndims);
std::vector<hsize_t> start(ndims);
std::vector<hsize_t> stride(ndims);
// FORTRAN ordering.
if((indexOrder == VsSchema::compMinorFKey) ||
(indexOrder == VsSchema::compMajorFKey))
{
// No components - [iz][iy][ix]
if( components == -2 )
{
for (unsigned int i = 0; i< (unsigned int)ndims; i++)
{
// Flip the ordering of the input.
int base = ndims - 1;
start[1] = (hsize_t) srcMins[base-i];
count[1] = (hsize_t) srcCounts[base-i];
stride[1] = (hsize_t) srcStrides[base-i];
VsLog::debugLog()
<< __CLASS__ << __FUNCTION__ << " " << __LINE__ << " "
<< "For i = " << i
<< ", start = " << start[i]
<< ", count = " << count[i]
<< ", stride = " << stride[i]
<< std::endl;
}
}
else if(indexOrder == VsSchema::compMajorFKey)
{
// Multiple components - [ic][iz][iy][ix] compMajorF
for (unsigned int i = 0; i< (unsigned int)ndims-1; i++)
{
// Flip the ordering of the input.
int base = ndims - 1;
start[i+1] = (hsize_t) srcMins[base-i];
count[i+1] = (hsize_t) srcCounts[base-i];
stride[i+1] = (hsize_t) srcStrides[base-i];
VsLog::debugLog()
<< __CLASS__ << __FUNCTION__ << " " << __LINE__ << " "
<< "For i = " << i
<< ", start = " << start[i+1]
<< ", count = " << count[i+1]
<< ", stride = " << stride[i+1]
<< std::endl;
}
// Components present but read all
if( components < 0 )
{
start[0] = 0;
count[0] = dims[ndims-1];
stride[0] = 1;
}
else // Components present but read a single value
{
start[0] = components;
count[0] = 1;
stride[0] = 1;
}
}
else //if( indexOrder == VsSchema::compMinorFKey )
{
// Multiple components - [iz][iy][ix][ic] compMinorF
for (unsigned int i = 0; i< (unsigned int)ndims-1; i++)
{
// Flip the ordering of the input.
int base = ndims - 1;
start[i] = (hsize_t) srcMins[base-i];
count[i] = (hsize_t) srcCounts[base-i];
stride[i] = (hsize_t) srcStrides[base-i];
VsLog::debugLog()
<< __CLASS__ << __FUNCTION__ << " " << __LINE__ << " "
<< "For i = " << i
<< ", start = " << start[i]
<< ", count = " << count[i]
<< ", stride = " << stride[i]
<< std::endl;
}
// Components present but read all
if( components < 0 )
{
start[ndims-1] = 0;
count[ndims-1] = dims[ndims-1];
stride[ndims-1] = 1;
}
else // Components present but read a single value
{
start[ndims-1] = components;
count[ndims-1] = 1;
stride[ndims-1] = 1;
}
}
}
// C ordering.
else //if((indexOrder == VsSchema::compMinorCKey) ||
// (indexOrder == VsSchema::compMajorCKey))
{
if( components == -2 )
{
// No components - [ix][iy][iz]
for (unsigned int i = 0; i< (unsigned int)ndims; i++)
{
start[i] = (hsize_t) srcMins[i];
count[i] = (hsize_t) srcCounts[i];
stride[i] = (hsize_t) srcStrides[i];
VsLog::debugLog()
<< __CLASS__ << __FUNCTION__ << " " << __LINE__ << " "
<< "For i = " << i
<< ", start = " << start[i]
<< ", count = " << count[i]
<< ", stride = " << stride[i]
<< std::endl;
}
}
else if(indexOrder == VsSchema::compMajorCKey)
{
// Multiple components - [ic][ix][iy][iz] compMajorC
for (unsigned int i = 0; i< (unsigned int)ndims-1; i++)
{
start[i+1] = (hsize_t) srcMins[i];
count[i+1] = (hsize_t) srcCounts[i];
stride[i+1] = (hsize_t) srcStrides[i];
VsLog::debugLog()
<< __CLASS__ << __FUNCTION__ << " " << __LINE__ << " "
<< "For i = " << i
<< ", start = " << start[i+1]
<< ", count = " << count[i+1]
<< ", stride = " << stride[i+1]
<< std::endl;
}
// Components present but read all
if( components < 0 )
{
start[0] = 0;
count[0] = dims[ndims-1];
stride[0] = 1;
}
else // Components present but read a single value
{
start[0] = components;
count[0] = 1;
stride[0] = 1;
}
}
else if(indexOrder == VsSchema::compMinorCKey)
{
// Multiple components - [ix][iy][iz][ic] compMinorC
for (unsigned int i = 0; i< (unsigned int)ndims-1; i++)
{
start[i] = (hsize_t) srcMins[i];
count[i] = (hsize_t) srcCounts[i];
stride[i] = (hsize_t) srcStrides[i];
VsLog::debugLog()
<< __CLASS__ << __FUNCTION__ << " " << __LINE__ << " "
<< "For i = " << i
<< ", start = " << start[i]
<< ", count = " << count[i]
<< ", stride = " << stride[i]
<< std::endl;
}
// Components present but read all
if( components < 0 )
{
start[ndims-1] = 0;
count[ndims-1] = dims[ndims-1];
stride[ndims-1] = 1;
}
else // Components present but read a single value
{
start[ndims-1] = components;
count[ndims-1] = 1;
stride[ndims-1] = 1;
}
}
}
// Select subset of the data
if( srcMins )
err = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET,
&(start[0]), &(stride[0]), &(count[0]), NULL);
VsLog::debugLog() << __CLASS__ << __FUNCTION__ << " " << __LINE__ << " "
<< "After selecting the hyperslab, err is " << err
<< std::endl;
// Create memory space for the data
hid_t memspace;
std::vector<hsize_t> destSize(ndims);
std::vector<hsize_t> destCount(ndims);
std::vector<hsize_t> destStart(ndims);
std::vector<hsize_t> destStride(ndims);
if( mdims == 0 )
memspace = H5Screate_simple(ndims, &(count[0]), NULL);
else
{
for( int i=0; i<mdims; ++i )
{
destSize[i] = (hsize_t) destSizes[i];
destStart[i] = (hsize_t) destMins[i];
destCount[i] = (hsize_t) destCounts[i];
destStride[i] = (hsize_t) destStrides[i];
}
memspace = H5Screate_simple(mdims, &(destSize[0]), NULL);
H5Sselect_hyperslab(memspace, H5S_SELECT_SET,
&(destStart[0]), &(destStride[0]), &(destCount[0]),
NULL);
}
// Read data
err = H5Dread(dataSet->getId(), dataSet->getType(), memspace, dataspace,
H5P_DEFAULT, data);
if (err < 0) {
VsLog::debugLog() << __CLASS__ << __FUNCTION__ << " " << __LINE__ << " "
<< ": error " << err
<< " in reading dataset." << std::endl;
}
err = H5Sclose(memspace);
err = H5Sclose(dataspace);
}
VsLog::debugLog() << __CLASS__ << __FUNCTION__ << " " << __LINE__ << " "
<< "Returning " << err << "." << std::endl;
return err;
}
/*-------------------------------------------------------------------------
* Function: main
*
* Purpose:
*
* Return: Success:
*
* Failure:
*
* Programmer: Robb Matzke
* Thursday, March 12, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
int
main (void)
{
static hsize_t size[2] = {REQUEST_SIZE_X, REQUEST_SIZE_Y};
static unsigned nread = NREAD_REQUESTS, nwrite = NWRITE_REQUESTS;
unsigned char *the_data = NULL;
hid_t file, dset, file_space = -1;
herr_t status;
#ifdef H5_HAVE_GETRUSAGE
struct rusage r_start, r_stop;
#else
struct timeval r_start, r_stop;
#endif
struct timeval t_start, t_stop;
int fd;
unsigned u;
hssize_t n;
off_t offset;
hsize_t start[2];
hsize_t count[2];
#ifdef H5_HAVE_SYS_TIMEB
struct _timeb *tbstart = malloc(sizeof(struct _timeb));
struct _timeb *tbstop = malloc(sizeof(struct _timeb));
#endif
/*
* The extra cast in the following statement is a bug workaround for the
* Win32 version 5.0 compiler.
* 1998-11-06 ptl
*/
printf ("I/O request size is %1.1fMB\n",
(double)(hssize_t)(size[0]*size[1])/1024.0F*1024);
/* Open the files */
file = H5Fcreate (HDF5_FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
assert (file>=0);
fd = HDopen (RAW_FILE_NAME, O_RDWR|O_CREAT|O_TRUNC, 0666);
assert (fd>=0);
/* Create the dataset */
file_space = H5Screate_simple (2, size, size);
assert(file_space >= 0);
dset = H5Dcreate2(file, "dset", H5T_NATIVE_UCHAR, file_space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
assert(dset >= 0);
the_data = (unsigned char *)malloc((size_t)(size[0] * size[1]));
/* initial fill for lazy malloc */
HDmemset(the_data, 0xAA, (size_t)(size[0] * size[1]));
/* Fill raw */
synchronize ();
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_start);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_start, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstart);
#endif
#endif
fprintf (stderr, HEADING, "fill raw");
for(u = 0; u < nwrite; u++) {
putc (PROGRESS, stderr);
HDfflush(stderr);
HDmemset(the_data, 0xAA, (size_t)(size[0]*size[1]));
}
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_stop);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_stop, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstop);
t_start.tv_sec = tbstart->time;
t_start.tv_usec = tbstart->millitm;
t_stop.tv_sec = tbstop->time;
t_stop.tv_usec = tbstop->millitm;
#endif
#endif
putc ('\n', stderr);
print_stats ("fill raw",
&r_start, &r_stop, &t_start, &t_stop,
(size_t)(nread*size[0]*size[1]));
/* Fill hdf5 */
synchronize ();
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_start);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_start, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstart);
#endif
#endif
fprintf (stderr, HEADING, "fill hdf5");
for(u = 0; u < nread; u++) {
putc (PROGRESS, stderr);
HDfflush(stderr);
status = H5Dread (dset, H5T_NATIVE_UCHAR, file_space, file_space,
H5P_DEFAULT, the_data);
assert (status>=0);
}
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_stop);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_stop, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstop);
t_start.tv_sec = tbstart->time;
t_start.tv_usec = tbstart->millitm;
t_stop.tv_sec = tbstop->time;
t_stop.tv_usec = tbstop->millitm;
#endif
#endif
putc ('\n', stderr);
print_stats ("fill hdf5",
&r_start, &r_stop, &t_start, &t_stop,
(size_t)(nread*size[0]*size[1]));
/* Write the raw dataset */
synchronize ();
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_start);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_start, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstart);
#endif
#endif
fprintf (stderr, HEADING, "out raw");
for(u = 0; u < nwrite; u++) {
putc (PROGRESS, stderr);
HDfflush(stderr);
offset = HDlseek (fd, (off_t)0, SEEK_SET);
assert (0==offset);
n = HDwrite (fd, the_data, (size_t)(size[0]*size[1]));
assert (n>=0 && (size_t)n==size[0]*size[1]);
}
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_stop);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_stop, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstop);
t_start.tv_sec = tbstart->time;
t_start.tv_usec = tbstart->millitm;
t_stop.tv_sec = tbstop->time;
t_stop.tv_usec = tbstop->millitm;
#endif
#endif
putc ('\n', stderr);
print_stats ("out raw",
&r_start, &r_stop, &t_start, &t_stop,
(size_t)(nread*size[0]*size[1]));
/* Write the hdf5 dataset */
synchronize ();
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_start);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_start, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstart);
#endif
#endif
fprintf (stderr, HEADING, "out hdf5");
for(u = 0; u < nwrite; u++) {
putc (PROGRESS, stderr);
HDfflush(stderr);
status = H5Dwrite (dset, H5T_NATIVE_UCHAR, H5S_ALL, H5S_ALL,
H5P_DEFAULT, the_data);
assert (status>=0);
}
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_stop);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_stop, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstop);
t_start.tv_sec = tbstart->time;
t_start.tv_usec = tbstart->millitm;
t_stop.tv_sec = tbstop->time;
t_stop.tv_usec = tbstop->millitm;
#endif
#endif
putc ('\n', stderr);
print_stats ("out hdf5",
&r_start, &r_stop, &t_start, &t_stop,
(size_t)(nread*size[0]*size[1]));
/* Read the raw dataset */
synchronize ();
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_start);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_start, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstart);
#endif
#endif
fprintf (stderr, HEADING, "in raw");
for(u = 0; u < nread; u++) {
putc (PROGRESS, stderr);
HDfflush(stderr);
offset = HDlseek (fd, (off_t)0, SEEK_SET);
assert (0==offset);
n = HDread (fd, the_data, (size_t)(size[0]*size[1]));
assert (n>=0 && (size_t)n==size[0]*size[1]);
}
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_stop);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_stop, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstop);
t_start.tv_sec = tbstart->time;
t_start.tv_usec = tbstart->millitm;
t_stop.tv_sec = tbstop->time;
t_stop.tv_usec = tbstop->millitm;
#endif
#endif
putc ('\n', stderr);
print_stats ("in raw",
&r_start, &r_stop, &t_start, &t_stop,
(size_t)(nread*size[0]*size[1]));
/* Read the hdf5 dataset */
synchronize ();
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_start);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_start, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstart);
#endif
#endif
fprintf (stderr, HEADING, "in hdf5");
for(u = 0; u < nread; u++) {
putc (PROGRESS, stderr);
HDfflush(stderr);
status = H5Dread (dset, H5T_NATIVE_UCHAR, file_space, file_space,
H5P_DEFAULT, the_data);
assert (status>=0);
}
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_stop);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_stop, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstop);
t_start.tv_sec = tbstart->time;
t_start.tv_usec = tbstart->millitm;
t_stop.tv_sec = tbstop->time;
t_stop.tv_usec = tbstop->millitm;
#endif
#endif
putc ('\n', stderr);
print_stats ("in hdf5",
&r_start, &r_stop, &t_start, &t_stop,
(size_t)(nread*size[0]*size[1]));
/* Read hyperslab */
assert (size[0]>20 && size[1]>20);
start[0] = start[1] = 10;
count[0] = count[1] = size[0]-20;
status = H5Sselect_hyperslab (file_space, H5S_SELECT_SET, start, NULL, count, NULL);
assert (status>=0);
synchronize ();
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_start);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_start, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstart);
#endif
#endif
fprintf (stderr, HEADING, "in hdf5 partial");
for(u = 0; u < nread; u++) {
putc (PROGRESS, stderr);
HDfflush(stderr);
status = H5Dread (dset, H5T_NATIVE_UCHAR, file_space, file_space,
H5P_DEFAULT, the_data);
assert (status>=0);
}
#ifdef H5_HAVE_GETRUSAGE
HDgetrusage(RUSAGE_SELF, &r_stop);
#endif
#ifdef H5_HAVE_GETTIMEOFDAY
HDgettimeofday(&t_stop, NULL);
#else
#ifdef H5_HAVE_SYS_TIMEB
_ftime(tbstop);
t_start.tv_sec = tbstart->time;
t_start.tv_usec = tbstart->millitm;
t_stop.tv_sec = tbstop->time;
t_stop.tv_usec = tbstop->millitm;
#endif
#endif
putc('\n', stderr);
print_stats("in hdf5 partial",
&r_start, &r_stop, &t_start, &t_stop,
(size_t)(nread*count[0]*count[1]));
/* Close everything */
HDclose(fd);
H5Dclose(dset);
H5Sclose(file_space);
H5Fclose(file);
free(the_data);
return 0;
}
/*-------------------------------------------------------------------------
* Function: test_core
*
* Purpose: Tests the file handle interface for CORE driver
*
* Return: Success: 0
* Failure: -1
*
* Programmer: Raymond Lu
* Tuesday, Sept 24, 2002
*
*-------------------------------------------------------------------------
*/
static herr_t
test_core(void)
{
hid_t file=(-1), fapl, access_fapl = -1;
char filename[1024];
void *fhandle=NULL;
hsize_t file_size;
int *points = NULL, *check = NULL, *p1, *p2;
hid_t dset1=-1, space1=-1;
hsize_t dims1[2];
int i, j, n;
TESTING("CORE file driver");
/* Set property list and file name for CORE driver */
fapl = h5_fileaccess();
if(H5Pset_fapl_core(fapl, (size_t)CORE_INCREMENT, TRUE) < 0)
TEST_ERROR;
h5_fixname(FILENAME[1], fapl, filename, sizeof filename);
if((file=H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
TEST_ERROR;
/* Retrieve the access property list... */
if ((access_fapl = H5Fget_access_plist(file)) < 0)
TEST_ERROR;
/* Check that the driver is correct */
if(H5FD_CORE != H5Pget_driver(access_fapl))
TEST_ERROR;
/* ...and close the property list */
if (H5Pclose(access_fapl) < 0)
TEST_ERROR;
if(H5Fget_vfd_handle(file, H5P_DEFAULT, &fhandle) < 0)
TEST_ERROR;
if(fhandle==NULL)
{
printf("fhandle==NULL\n");
TEST_ERROR;
}
/* Check file size API */
if(H5Fget_filesize(file, &file_size) < 0)
TEST_ERROR;
/* There is no garantee the size of metadata in file is constant.
* Just try to check if it's reasonable. Why is this 4KB?
*/
if(file_size<2*KB || file_size>6*KB)
TEST_ERROR;
if(H5Fclose(file) < 0)
TEST_ERROR;
/* Open the file with backing store off for read and write.
* Changes won't be saved in file. */
if(H5Pset_fapl_core(fapl, (size_t)CORE_INCREMENT, FALSE) < 0)
TEST_ERROR;
if((file=H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0)
TEST_ERROR;
/* Allocate memory for data set. */
if(NULL == (points = (int *)HDmalloc(DSET1_DIM1 * DSET1_DIM2 * sizeof(int))))
TEST_ERROR;
if(NULL == (check = (int *)HDmalloc(DSET1_DIM1 * DSET1_DIM2 * sizeof(int))))
TEST_ERROR;
/* Initialize the dset1 */
p1 = points;
for(i = n = 0; i < DSET1_DIM1; i++)
for(j = 0; j < DSET1_DIM2; j++)
*p1++ = n++;
/* Create the data space1 */
dims1[0] = DSET1_DIM1;
dims1[1] = DSET1_DIM2;
if((space1 = H5Screate_simple(2, dims1, NULL)) < 0)
TEST_ERROR;
/* Create the dset1 */
if((dset1 = H5Dcreate2(file, DSET1_NAME, H5T_NATIVE_INT, space1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Write the data to the dset1 */
if(H5Dwrite(dset1, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, points) < 0)
TEST_ERROR;
if(H5Dclose(dset1) < 0)
TEST_ERROR;
if((dset1 = H5Dopen2(file, DSET1_NAME, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Read the data back from dset1 */
if(H5Dread(dset1, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, check) < 0)
TEST_ERROR;
/* Check that the values read are the same as the values written */
p1 = points;
p2 = check;
for(i = 0; i < DSET1_DIM1; i++)
for(j = 0; j < DSET1_DIM2; j++)
if(*p1++ != *p2++) {
H5_FAILED();
printf(" Read different values than written in data set 1.\n");
printf(" At index %d,%d\n", i, j);
TEST_ERROR;
} /* end if */
if(H5Dclose(dset1) < 0)
TEST_ERROR;
if(H5Fclose(file) < 0)
TEST_ERROR;
/* Open the file with backing store on for read and write.
* Changes will be saved in file. */
if(H5Pset_fapl_core(fapl, (size_t)CORE_INCREMENT, TRUE) < 0)
TEST_ERROR;
if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0)
TEST_ERROR;
/* Create the dset1 */
if((dset1 = H5Dcreate2(file, DSET1_NAME, H5T_NATIVE_INT, space1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Write the data to the dset1 */
if(H5Dwrite(dset1, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, points) < 0)
TEST_ERROR;
if(H5Dclose(dset1) < 0)
TEST_ERROR;
if((dset1 = H5Dopen2(file, DSET1_NAME, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Reallocate memory for reading buffer. */
HDassert(check);
HDfree(check);
if(NULL == (check = (int *)HDmalloc(DSET1_DIM1 * DSET1_DIM2 * sizeof(int))))
TEST_ERROR;
/* Read the data back from dset1 */
if(H5Dread(dset1, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, check) < 0)
TEST_ERROR;
/* Check that the values read are the same as the values written */
p1 = points;
p2 = check;
for(i = 0; i < DSET1_DIM1; i++)
for(j = 0; j < DSET1_DIM2; j++)
if(*p1++ != *p2++) {
H5_FAILED();
printf(" Read different values than written in data set 1.\n");
printf(" At index %d,%d\n", i, j);
TEST_ERROR;
} /* end if */
/* Check file size API */
if(H5Fget_filesize(file, &file_size) < 0)
TEST_ERROR;
/* There is no garantee the size of metadata in file is constant.
* Just try to check if it's reasonable. */
if(file_size<64*KB || file_size>256*KB)
TEST_ERROR;
if(H5Sclose(space1) < 0)
TEST_ERROR;
if(H5Dclose(dset1) < 0)
TEST_ERROR;
if(H5Fclose(file) < 0)
TEST_ERROR;
HDassert(points);
HDfree(points);
HDassert(check);
HDfree(check);
h5_cleanup(FILENAME, fapl);
PASSED();
return 0;
error:
H5E_BEGIN_TRY {
H5Pclose(fapl);
H5Fclose(file);
} H5E_END_TRY;
if(points)
HDfree(points);
if(check)
HDfree(check);
return -1;
}
