// Author & Date: Ehsan Azar Nov 13, 2012
// Purpose: Create type for BmiSpike16_t and commit
// Note: For performance reasons and because spike length is constant during
// and experiment we use fixed-length array here instead of varible-length
// Inputs:
// loc - where to add the type
// spikeLength - the spike length to use
// Outputs:
// Returns the type id
hid_t CreateSpike16Type(hid_t loc, UINT16 spikeLength)
{
herr_t ret;
hid_t tid = -1;
// e.g. for spike length of 48 type name will be "BmiSpike16_48_t"
char szNum[4] = {'\0'};
sprintf(szNum, "%u", spikeLength);
std::string strLink = "BmiSpike16_";
strLink += szNum;
strLink += "_t";
// If already there return it
if(H5Lexists(loc, strLink.c_str(), H5P_DEFAULT))
{
tid = H5Topen(loc, strLink.c_str(), H5P_DEFAULT);
return tid;
}
hsize_t dims[1] = {spikeLength};
hid_t tid_arr_wave = H5Tarray_create(H5T_NATIVE_INT16, 1, dims);
tid = H5Tcreate(H5T_COMPOUND, offsetof(BmiSpike16_t, wave) + sizeof(INT16) * spikeLength);
ret = H5Tinsert(tid, "TimeStamp", offsetof(BmiSpike16_t, dwTimestamp), H5T_NATIVE_UINT32);
ret = H5Tinsert(tid, "Unit", offsetof(BmiSpike16_t, unit), H5T_NATIVE_UINT8);
ret = H5Tinsert(tid, "Wave", offsetof(BmiSpike16_t, wave), tid_arr_wave);
ret = H5Tcommit(loc, strLink.c_str(), tid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Tclose(tid_arr_wave);
return tid;
}
// Constructor
FileIO::FileIO(Parallel *_parallel, Setup *setup): parallel(_parallel)
{
// Set Initial values
inputFileName = setup->get("DataOutput.InputFileName", "--- None ---");
outputFileName = setup->get("DataOutput.OutputFileName", "default.h5");
info = setup->get("DataOutput.Info", "No information provided");
resumeFile = setup->get("DataOutput.Resume", 0);
overwriteFile = setup->get("DataOutput.Overwrite", 0) || (setup->flags & HELIOS_OVERWRITE);
dataFileFlushTiming = Timing(setup->get("DataOutput.Flush.Step", -1) , setup->get("DataOutput.Flush.Time", 100.));
///////// Define Timeing Datatype
timing_tid = H5Tcreate(H5T_COMPOUND, sizeof(Timing));
H5Tinsert(timing_tid, "Timestep", HOFFSET(Timing, step), H5T_NATIVE_INT );
H5Tinsert(timing_tid, "Time" , HOFFSET(Timing, time), H5T_NATIVE_DOUBLE);
// don't changes r and i name otherwise it will break compatibiltiy with pyTables
complex_tid = H5Tcreate(H5T_COMPOUND, sizeof (Complex));
H5Tinsert(complex_tid, "r", HOFFSET(Complex,r), H5T_NATIVE_DOUBLE);
H5Tinsert(complex_tid, "i", HOFFSET(Complex,i), H5T_NATIVE_DOUBLE);
vector3D_tid = H5Tcreate(H5T_COMPOUND, sizeof (Vector3D));
H5Tinsert(vector3D_tid, "x", HOFFSET(Vector3D,x), H5T_NATIVE_DOUBLE);
H5Tinsert(vector3D_tid, "y", HOFFSET(Vector3D,y), H5T_NATIVE_DOUBLE);
H5Tinsert(vector3D_tid, "z", HOFFSET(Vector3D,z), H5T_NATIVE_DOUBLE);
hsize_t species_dim[] = { setup->get("Grid.Ns", 1)};
species_tid = H5Tarray_create(H5T_NATIVE_DOUBLE, 1, species_dim);
// used for species name
// BUG : Somehow HDF-8 stores up to 8 chars fro 64 possible. Rest are truncated ! Why ?
s256_tid = H5Tcopy(H5T_C_S1); H5Tset_size(s256_tid, 64); H5Tset_strpad(s256_tid, H5T_STR_NULLTERM);
offset0[0] = 0;
offset0[1] = 0;
offset0[2] = 0;
offset0[3] = 0;
offset0[4] = 0;
offset0[5] = 0;
offset0[6] = 0;
offset0[7] = 0;
// Create/Load HDF5 file
if(resumeFile == false || (inputFileName != outputFileName)) create(setup);
}
void SCIA_WR_H5_LADS( struct param_record param, unsigned int nr_lads,
const struct lads_scia *lads )
/*@globals lads_size, lads_offs@*/
{
hid_t ads_id;
hsize_t adim;
hid_t lads_type[NFIELDS];
hid_t type_id, temp_type00, temp_type01;
const hbool_t compress = (param.flag_deflate == PARAM_SET) ? TRUE : FALSE;
const char *lads_names[NFIELDS] = {
"dsr_time", "attach_flag", "corner_grd"
};
/*
* check number of LADS records
*/
if ( nr_lads == 0 ) return;
/*
* create/open group /ADS
*/
ads_id = NADC_OPEN_HDF5_Group( param.hdf_file_id, "/ADS" );
if ( ads_id < 0 ) NADC_RETURN_ERROR( NADC_ERR_HDF_GRP, "/ADS" );
/*
* define user-defined data types of the Table-fields
*/
temp_type00 = H5Topen( param.hdf_file_id, "mjd", H5P_DEFAULT );
adim = NUM_CORNERS;
type_id = H5Topen( param.hdf_file_id, "coord", H5P_DEFAULT );
temp_type01 = H5Tarray_create( type_id, 1, &adim );
lads_type[0] = temp_type00;
lads_type[1] = H5T_NATIVE_UCHAR;
lads_type[2] = temp_type01;
/*
* create table
*/
(void) H5TBmake_table( "lads", ads_id, TBL_NAME, NFIELDS,
nr_lads, lads_size, lads_names, lads_offs,
lads_type, nr_lads, NULL, compress, lads );
/*
* close interface
*/
(void) H5Tclose( type_id );
(void) H5Tclose( temp_type00 );
(void) H5Tclose( temp_type01 );
(void) H5Gclose( ads_id );
}
extern hid_t
dataVar_getHDF5Datatype(const dataVar_t var)
{
assert(var != NULL);
hid_t dt, baseDt;
switch (var->type) {
case DATAVARTYPE_DOUBLE:
baseDt = H5T_NATIVE_DOUBLE;
break;
case DATAVARTYPE_FLOAT:
baseDt = H5T_NATIVE_FLOAT;
break;
case DATAVARTYPE_INT:
case DATAVARTYPE_INT32:
baseDt = H5T_NATIVE_INT;
break;
case DATAVARTYPE_INT64:
baseDt = H5T_NATIVE_INT64;
break;
case DATAVARTYPE_INT8:
baseDt = H5T_NATIVE_CHAR;
break;
case DATAVARTYPE_FPV:
# ifdef ENABLE_DOUBLE
baseDt = H5T_NATIVE_DOUBLE;
# else
baseDt = H5T_NATIVE_FLOAT;
# endif
break;
}
if (var->numComponents == 1) {
dt = H5Tcopy(baseDt);
} else {
hsize_t tmp[1] = { var->numComponents };
dt = H5Tarray_create(baseDt, 1, tmp);
}
return dt;
} /* dataVar_getHDF5Datatype */
static int check_valid_tag( int tag_idx, mhdf_FileHandle file, struct mhdf_FileDesc* desc )
{
long *ids = 0, count, junk;
long *ranges;
int nranges;
hid_t handles[3];
mhdf_Status status;
const struct mhdf_TagDesc* tag = &(desc->tags[tag_idx]);
int i, result = 0;
long srange[2];
const char* name;
struct mhdf_EntDesc* group;
hid_t h5type;
hsize_t size;
if (tag->have_sparse) {
mhdf_openSparseTagData( file, tag->name, &count, &junk, handles, &status );
if (mhdf_isError(&status)) {
fprintf(stderr,"Internal error opening sparse data for tag \"%s\": %s\n", tag->name, mhdf_message(&status));
return 1;
}
ids = malloc( sizeof(long) * count );
mhdf_readSparseTagEntities( handles[0], 0, count, H5T_NATIVE_LONG, ids, &status );
if (mhdf_isError(&status)) {
fprintf(stderr,"Internal error reading sparse entities for tag \"%s\": %s\n", tag->name, mhdf_message(&status));
mhdf_closeData( file, handles[0], &status );
mhdf_closeData( file, handles[1], &status );
free(ids);
return 1;
}
mhdf_closeData( file, handles[0], &status );
ranges = all_id_ranges( desc, 0, &nranges );
if (!ids_contained( ids, count, ranges, nranges )) {
++result;
printf("Sparse data for tag \"%s\" has values for invalid IDs\n", tag->name );
}
else if (contains_duplicates( ids, count )) {
++result;
printf("Sparse data for tag \"%s\" has duplicate values for one or more entities\n", tag->name );
}
free(ranges);
for (i = 0; i < tag->num_dense_indices; ++i) {
if (tag->dense_elem_indices[i] == -2) {
name = mhdf_set_type_handle();
group = &desc->sets;
}
else if (tag->dense_elem_indices[i] == -1) {
name = mhdf_node_type_handle();
group = &desc->nodes;
}
else {
name = desc->elems[ tag->dense_elem_indices[i] ].handle;
group = &desc->elems[ tag->dense_elem_indices[i] ].desc;
}
srange[0] = group->start_id;
srange[1] = group->count;
if (ids_contained( ids, count, srange, 2 )) {
++result;
printf("Tag \"%s\" has both sparse values and dense values for one or more entities in \"%s\"\n", tag->name, name );
}
}
free(ids);
}
if (tag->type != mhdf_ENTITY_ID) {
if (tag->have_sparse)
mhdf_closeData( file, handles[1], &status );
return result;
}
ranges = all_id_ranges( desc, 1, &nranges );
if (tag->default_value && !ids_contained( tag->default_value, tag->size, ranges, nranges )) {
++result;
printf("Handle tag \"%s\" has invalid ID(s) in its default value.\n", tag->name );
}
if (tag->global_value && !ids_contained( tag->global_value, tag->size, ranges, nranges )) {
++result;
printf("Handle tag \"%s\" has invalid ID(s) in its global/mesh value.\n", tag->name );
}
h5type = H5T_NATIVE_LONG;
if (tag->size > 1) {
size = tag->size;
#if defined(H5Tarray_create_vers) && H5Tarray_create_vers > 1
h5type = H5Tarray_create( H5T_NATIVE_LONG, 1, &size );
#else
h5type = H5Tarray_create( H5T_NATIVE_LONG, 1, &size, NULL );
#endif
}
if (tag->have_sparse) {
ids = malloc( tag->size * count * sizeof(long) );
mhdf_readTagValues( handles[1], 0, count, h5type, ids, &status );
if (mhdf_isError(&status)) {
fprintf(stderr,"Internal error reading sparse values for handle tag \"%s\": %s\n", tag->name, mhdf_message(&status));
mhdf_closeData( file, handles[1], &status );
free(ids);
free(ranges);
if (tag->size > 1)
H5Tclose( h5type );
return 1;
}
mhdf_closeData( file, handles[1], &status );
if (!ids_contained( ids, tag->size * count, ranges, nranges )) {
++result;
printf("Sparse data for one or more entities with handle tag \"%s\" has invalid ID(s).\n", tag->name );
}
free(ids);
}
for (i = 0; i < tag->num_dense_indices; ++i) {
if (tag->dense_elem_indices[i] == -2) {
name = mhdf_set_type_handle();
/*group = &desc->sets;*/
}
else if (tag->dense_elem_indices[i] == -1) {
name = mhdf_node_type_handle();
/*group = &desc->nodes;*/
}
else {
name = desc->elems[ tag->dense_elem_indices[i] ].handle;
/*group = &desc->elems[ tag->dense_elem_indices[i] ].desc;*/
}
handles[0] = mhdf_openDenseTagData( file, tag->name, name, &count, &status );
if (mhdf_isError(&status)) {
fprintf(stderr,"Internal dense values for handle tag \"%s\" on \"%s\": %s\n", tag->name, name, mhdf_message(&status));
++result;
continue;
}
ids = malloc( tag->size * count * sizeof(long) );
mhdf_readTagValues( handles[0], 0, count, h5type, ids, &status );
if (mhdf_isError(&status)) {
fprintf(stderr,"Internal error reading dense values for handle tag \"%s\" on \"%s\": %s\n", tag->name, name, mhdf_message(&status));
mhdf_closeData( file, handles[0], &status );
free(ids);
++result;
continue;
}
mhdf_closeData( file, handles[1], &status );
if (!ids_contained( ids, count, ranges, nranges )) {
++result;
printf("Dense data on \"%s\" for handle tag \"%s\" has invalid ID(s) for one or more entities.\n", name, tag->name );
}
free(ids);
}
if (tag->size > 1)
H5Tclose( h5type );
return result;
}
ColTypeFloat3Array()
{
const hsize_t dim[] = {3};
this->type = H5Tarray_create(H5T_FLOAT, 1, dim);
}
/*+++++++++++++++++++++++++ Main Program or Function +++++++++++++++*/
void SCIA_LV1_WR_H5_SRSN( struct param_record param, unsigned int nr_srsn,
const struct srsn_scia *srsn )
{
hid_t ads_id;
hsize_t adim;
herr_t stat;
hid_t srsn_type[NFIELDS];
const hbool_t compress = (param.flag_deflate == PARAM_SET) ? TRUE : FALSE;
const size_t srsn_size = sizeof( struct srsn_scia );
const char *srsn_names[NFIELDS] = {
"mjd", "flag_mds", "flag_neu", "sun_spec_id", "avg_asm", "avg_esm",
"avg_elev_sun", "dopp_shift", "wvlen_sun", "mean_sun",
"precision_sun", "accuracy_sun", "etalon", "pmd_mean", "pmd_out"
};
const size_t srsn_offs[NFIELDS] = {
HOFFSET( struct srsn_scia, mjd ),
HOFFSET( struct srsn_scia, flag_mds ),
HOFFSET( struct srsn_scia, flag_neu ),
HOFFSET( struct srsn_scia, sun_spec_id ),
HOFFSET( struct srsn_scia, avg_asm ),
HOFFSET( struct srsn_scia, avg_esm ),
HOFFSET( struct srsn_scia, avg_elev_sun ),
HOFFSET( struct srsn_scia, dopp_shift ),
HOFFSET( struct srsn_scia, wvlen_sun ),
HOFFSET( struct srsn_scia, mean_sun ),
HOFFSET( struct srsn_scia, precision_sun ),
HOFFSET( struct srsn_scia, accuracy_sun ),
HOFFSET( struct srsn_scia, etalon ),
HOFFSET( struct srsn_scia, pmd_mean ),
HOFFSET( struct srsn_scia, pmd_out )
};
/*
* check number of PMD records
*/
if ( nr_srsn == 0 ) return;
/*
* open/create group /ADS
*/
ads_id = NADC_OPEN_HDF5_Group( param.hdf_file_id, "/ADS" );
if ( ads_id < 0 ) NADC_RETURN_ERROR( NADC_ERR_HDF_GRP, "/ADS" );
/*
* write SRSN data sets
*/
adim = SCIENCE_PIXELS;
srsn_type[0] = H5Topen( param.hdf_file_id, "mjd", H5P_DEFAULT );
srsn_type[1] = H5Tcopy( H5T_NATIVE_UCHAR );
srsn_type[2] = H5Tcopy( H5T_NATIVE_UCHAR );
srsn_type[3] = H5Tcopy( H5T_C_S1 );
(void) H5Tset_size( srsn_type[3], (size_t) 3 );
srsn_type[4] = H5Tcopy( H5T_NATIVE_FLOAT );
srsn_type[5] = H5Tcopy( H5T_NATIVE_FLOAT );
srsn_type[6] = H5Tcopy( H5T_NATIVE_FLOAT );
srsn_type[7] = H5Tcopy( H5T_NATIVE_FLOAT );
adim = SCIENCE_PIXELS;
srsn_type[8] = H5Tarray_create( H5T_NATIVE_FLOAT, 1, &adim );
srsn_type[9] = H5Tarray_create( H5T_NATIVE_FLOAT, 1, &adim );
srsn_type[10] = H5Tarray_create( H5T_NATIVE_FLOAT, 1, &adim );
srsn_type[11] = H5Tarray_create( H5T_NATIVE_FLOAT, 1, &adim );
srsn_type[12] = H5Tarray_create( H5T_NATIVE_FLOAT, 1, &adim );
adim = PMD_NUMBER;
srsn_type[13] = H5Tarray_create( H5T_NATIVE_FLOAT, 1, &adim );
srsn_type[14] = H5Tarray_create( H5T_NATIVE_FLOAT, 1, &adim );
stat = H5TBmake_table( "srsn", ads_id, "NEW_SUN_REFERENCE",
NFIELDS, 1, srsn_size, srsn_names,
srsn_offs, srsn_type, 1,
NULL, compress, srsn );
if ( stat < 0 ) NADC_GOTO_ERROR( NADC_ERR_HDF_DATA, "srsn" );
/*
* close interface
*/
done:
(void) H5Tclose( srsn_type[0] );
(void) H5Tclose( srsn_type[1] );
(void) H5Tclose( srsn_type[2] );
(void) H5Tclose( srsn_type[3] );
(void) H5Tclose( srsn_type[4] );
(void) H5Tclose( srsn_type[5] );
(void) H5Tclose( srsn_type[6] );
(void) H5Tclose( srsn_type[7] );
(void) H5Tclose( srsn_type[8] );
(void) H5Tclose( srsn_type[9] );
(void) H5Tclose( srsn_type[10] );
(void) H5Tclose( srsn_type[11] );
(void) H5Tclose( srsn_type[12] );
(void) H5Tclose( srsn_type[13] );
(void) H5Tclose( srsn_type[14] );
(void) H5Gclose( ads_id );
}
herr_t H5VLARRAYmake( hid_t loc_id,
const char *dset_name,
const char *obversion,
const int rank,
const hsize_t *dims,
hid_t type_id,
hsize_t chunk_size,
void *fill_data,
int compress,
char *complib,
int shuffle,
int fletcher32,
const void *data)
{
hvl_t vldata;
hid_t dataset_id, space_id, datatype, tid1;
hsize_t dataset_dims[1];
hsize_t maxdims[1] = { H5S_UNLIMITED };
hsize_t dims_chunk[1];
hid_t plist_id;
unsigned int cd_values[6];
if (data)
/* if data, one row will be filled initially */
dataset_dims[0] = 1;
else
/* no data, so no rows on dataset initally */
dataset_dims[0] = 0;
dims_chunk[0] = chunk_size;
/* Fill the vldata estructure with the data to write */
/* This is currectly not used */
vldata.p = (void *)data;
vldata.len = 1; /* Only one array type to save */
/* Create a VL datatype */
if (rank == 0) {
datatype = H5Tvlen_create(type_id);
}
else {
tid1 = H5Tarray_create(type_id, rank, dims);
datatype = H5Tvlen_create(tid1);
H5Tclose( tid1 ); /* Release resources */
}
/* The dataspace */
space_id = H5Screate_simple( 1, dataset_dims, maxdims );
/* Modify dataset creation properties, i.e. enable chunking */
plist_id = H5Pcreate (H5P_DATASET_CREATE);
if ( H5Pset_chunk ( plist_id, 1, dims_chunk ) < 0 )
return -1;
/*
Dataset creation property list is modified to use
*/
/* Fletcher must be first */
if (fletcher32) {
if ( H5Pset_fletcher32( plist_id) < 0 )
return -1;
}
/* Then shuffle (blosc shuffles inplace) */
if (shuffle && (strcmp(complib, "blosc") != 0)) {
if ( H5Pset_shuffle( plist_id) < 0 )
return -1;
}
/* Finally compression */
if (compress) {
cd_values[0] = compress;
cd_values[1] = (int)(atof(obversion) * 10);
cd_values[2] = VLArray;
/* The default compressor in HDF5 (zlib) */
if (strcmp(complib, "zlib") == 0) {
if ( H5Pset_deflate( plist_id, compress) < 0 )
return -1;
}
/* The Blosc compressor does accept parameters */
else if (strcmp(complib, "blosc") == 0) {
cd_values[4] = compress;
cd_values[5] = shuffle;
if ( H5Pset_filter( plist_id, FILTER_BLOSC, H5Z_FLAG_OPTIONAL, 6, cd_values) < 0 )
return -1;
}
/* The LZO compressor does accept parameters */
else if (strcmp(complib, "lzo") == 0) {
if ( H5Pset_filter( plist_id, FILTER_LZO, H5Z_FLAG_OPTIONAL, 3, cd_values) < 0 )
return -1;
}
/* The bzip2 compress does accept parameters */
else if (strcmp(complib, "bzip2") == 0) {
if ( H5Pset_filter( plist_id, FILTER_BZIP2, H5Z_FLAG_OPTIONAL, 3, cd_values) < 0 )
return -1;
}
else {
/* Compression library not supported */
fprintf(stderr, "Compression library not supported\n");
return -1;
}
}
/* Create the dataset. */
if ((dataset_id = H5Dcreate(loc_id, dset_name, datatype, space_id,
H5P_DEFAULT, plist_id, H5P_DEFAULT )) < 0 )
goto out;
/* Write the dataset only if there is data to write */
if (data)
if ( H5Dwrite( dataset_id, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &vldata ) < 0 )
goto out;
/* Terminate access to the data space. */
if ( H5Sclose( space_id ) < 0 )
return -1;
/* Release the datatype in the case that it is not an atomic type */
if ( H5Tclose( datatype ) < 0 )
return -1;
/* End access to the property list */
if ( H5Pclose( plist_id ) < 0 )
goto out;
return dataset_id;
out:
return -1;
}
void UPCdata::readHDFfile(char* filename, int count){ // count: 11040 for segmentation and 14640 for classification
#if HDF5_AVAILABLE==2
// based on http://www.hdfgroup.org/HDF5/doc/cpplus_RM/readdata_8cpp-example.html
// maybe first deleting all data? does it do that automatically or is there some possible memory leak?
// I need to know first the data set sizes... so I create one dummy sample... this is only a hack
samples.resize(0);
/*
Sample dummy;
dummy.data.resize(1020,0);
dummy.concentrations[0]=1;dummy.concentrations[1]=1;dummy.concentrations[2]=1;
dummy.samplenr=1;
dummy.set=Sample::Training;
dummy.compoundnr=1;
*/
const int NFIELDS = 5;
hsize_t chunk_size = 10; // ??
typedef struct _samplestruct{
int concentrations[3];
int samplenr;
int set;
int compoundnr;
float data[1020]; // having difficulties here; dynamic allocation?
}samplestruct;
samplestruct *sampledata;
void *sampleptr = malloc(NRECORDS*sizeof(samplestruct));
sampledata = (samplestruct*)sampleptr;
if (sampleptr == NULL) {
// Memory could not be allocated, the program should handle the error here as appropriate.
cout << "memory could not be allocated!";
return;
}
// ...
// free(samplestruct);
/* size_t dst_size = sizeof(Sample);
size_t dst_offset[NFIELDS] = {
HOFFSET(Sample, concentrations),
HOFFSET(Sample, samplenr),
HOFFSET(Sample, set),
HOFFSET(Sample, compoundnr),
HOFFSET(Sample, data)
};
size_t dst_sizes[NFIELDS] = { sizeof( samples[0].data),
sizeof(samples[0].concentrations),
sizeof(samples[0].samplenr),
sizeof(samples[0].set),
sizeof(samples[0].compoundnr)};
*/
size_t dst_size = sizeof(samplestruct);
size_t dst_offset[NFIELDS] = {HOFFSET(samplestruct, concentrations),
HOFFSET(samplestruct, samplenr),
HOFFSET(samplestruct, set),
HOFFSET(samplestruct, compoundnr),
HOFFSET(samplestruct, data),
};
size_t dst_sizes[NFIELDS] = { sizeof( sampledata[0].data),
sizeof(sampledata[0].concentrations),
sizeof(sampledata[0].samplenr),
sizeof(sampledata[0].set),
sizeof(sampledata[0].compoundnr)
};
hid_t file_id;
int *fill_data = NULL;
int compress = 1;
herr_t status;
//const hsize_t datadim[] = {samples[0].data.size()};
const hsize_t datadim[] = {1020*sizeof(float)};
const hsize_t concdim[] = {3};
hid_t floatarray = H5Tarray_create(H5T_NATIVE_FLOAT, 1, datadim);
hid_t concarray = H5Tarray_create(H5T_NATIVE_FLOAT, 1, concdim);
// Initialize field_type
hid_t field_type[NFIELDS];
field_type[0] = floatarray;
field_type[1] = concarray;
field_type[2] = H5T_NATIVE_INT;
field_type[3] = H5T_NATIVE_INT;
field_type[4] = H5T_NATIVE_INT;
// opening file
file_id=H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT);
if ( file_id < 0 ){
std::cout << "UPCdata::readHDFfile(): Error opening " << filename <<": " << file_id << std::endl;
return;
}
//const int NRECORDS=samples.size(); // how to get the data set size?
// is this the way?
hid_t dataset_id=H5Dopen1(file_id, "Dataset");
hsize_t size = H5Dget_storage_size(dataset_id);
//vector<Sample> buf(NRECORDS+1);
//vector<Sample> buf(static_cast<int>(size+1), 0x00);
status=H5TBread_table(file_id, "Dataset", dst_size, dst_offset, dst_sizes, sampledata);
cout << "converting data format" << endl;
for(int i=0;i<NRECORDS;i++){
Sample dummysample;
dummysample.concentrations[0]=sampledata[i].concentrations[0];
dummysample.concentrations[1]=sampledata[i].concentrations[1];
dummysample.concentrations[2]=sampledata[i].concentrations[2];
dummysample.samplenr=sampledata[i].samplenr;
dummysample.set=(Sample::SetType)sampledata[i].set;
dummysample.compoundnr=sampledata[i].compoundnr;
dummysample.data.resize(1020);
for(int j=0;j<1020;j++)
dummysample.data[j]=sampledata[i].data[j];
samples.push_back(dummysample);
}
// close type(s)??
// close the file /
H5Fclose( file_id );
#endif
}
/*+++++++++++++++++++++++++ Main Program or Function +++++++++++++++*/
void SCIA_OL2_WR_H5_CLD( struct param_record param, unsigned int nr_cld,
const struct cld_sci_ol *cld )
{
register unsigned int nr;
hid_t grp_id;
hbool_t compress;
hsize_t adim;
hvl_t *vdata;
hid_t cld_type[NFIELDS];
const char *cld_names[NFIELDS] = {
"dsr_time", "quality_flag", "integr_time", "pmd_read",
"cl_type_flags", "cloud_flags", "flag_output_flags",
"num_aero_param", "pmd_read_cl", "dsr_length",
"surface_pres", "cl_frac", "cl_frac_err", "cl_top_pres",
"cl_top_pres_err", "cl_opt_depth", "cl_opt_depth_err",
"cl_reflectance", "cl_reflectance_err",
"surface_reflectance", "surface_reflectance_err",
"aero_abso_ind", "aero_ind_diag"
};
/*
* check number of CLD records
*/
if ( nr_cld == 0 ) return;
/*
* set HDF5 boolean variable for compression
*/
if ( param.flag_deflate == PARAM_SET )
compress = TRUE;
else
compress = FALSE;
/*
* create group /MDS
*/
grp_id = NADC_OPEN_HDF5_Group( param.hdf_file_id, "/MDS" );
if ( grp_id < 0 ) NADC_RETURN_ERROR( NADC_ERR_HDF_GRP, "/MDS" );
/*
* define user-defined data types of the Table-fields
*/
cld_type[0] = H5Topen( param.hdf_file_id, "mjd", H5P_DEFAULT );
cld_type[1] = H5Tcopy( H5T_NATIVE_CHAR );
cld_type[2] = H5Tcopy( H5T_NATIVE_USHORT );
cld_type[3] = H5Tcopy( H5T_NATIVE_USHORT );
cld_type[4] = H5Tcopy( H5T_NATIVE_USHORT );
cld_type[5] = H5Tcopy( H5T_NATIVE_USHORT );
cld_type[6] = H5Tcopy( H5T_NATIVE_USHORT );
cld_type[7] = H5Tcopy( H5T_NATIVE_USHORT );
adim = 2;
cld_type[8] = H5Tarray_create( H5T_NATIVE_USHORT, 1, &adim );
cld_type[9] = H5Tcopy( H5T_NATIVE_UINT );
cld_type[10] = H5Tcopy( H5T_NATIVE_FLOAT );
cld_type[11] = H5Tcopy( H5T_NATIVE_FLOAT );
cld_type[12] = H5Tcopy( H5T_NATIVE_FLOAT );
cld_type[13] = H5Tcopy( H5T_NATIVE_FLOAT );
cld_type[14] = H5Tcopy( H5T_NATIVE_FLOAT );
cld_type[15] = H5Tcopy( H5T_NATIVE_FLOAT );
cld_type[16] = H5Tcopy( H5T_NATIVE_FLOAT );
cld_type[17] = H5Tcopy( H5T_NATIVE_FLOAT );
cld_type[18] = H5Tcopy( H5T_NATIVE_FLOAT );
cld_type[19] = H5Tcopy( H5T_NATIVE_FLOAT );
cld_type[20] = H5Tcopy( H5T_NATIVE_FLOAT );
cld_type[21] = H5Tcopy( H5T_NATIVE_FLOAT );
cld_type[22] = H5Tcopy( H5T_NATIVE_FLOAT );
/*
* create table
*/
(void) H5TBmake_table( "Cloud end Aerosol MDS", grp_id, "cld", NFIELDS,
nr_cld, cld_size, cld_names, cld_offs,
cld_type, nr_cld, NULL, compress, cld );
/*
* close interface
*/
for ( nr = 0; nr < NFIELDS; nr++ ) (void) H5Tclose( cld_type[nr] );
/*
* +++++ create/write variable part of the CLOUDS_AEROSOL record
*/
adim = (hsize_t) nr_cld;
/*
* Additional aerosol parameters
*/
vdata = (hvl_t *) malloc( nr_cld * sizeof(hvl_t) );
if ( vdata == NULL ) NADC_RETURN_ERROR( NADC_ERR_ALLOC, "vdata" );
nr = 0;
do {
vdata[nr].len = (size_t) cld[nr].numaeropars;
if ( cld[nr].numaeropars > 0 ) {
vdata[nr].p = malloc( vdata[nr].len * sizeof(float) );
if ( vdata[nr].p == NULL ) {
free( vdata );
NADC_RETURN_ERROR( NADC_ERR_ALLOC, "vdata.p" );
}
(void) memcpy( vdata[nr].p , cld[nr].aeropars,
vdata[nr].len * sizeof(float) );
}
} while ( ++nr < nr_cld );
NADC_WR_HDF5_Vlen_Dataset( compress, grp_id, "aeropars",
H5T_NATIVE_FLOAT, 1, &adim, vdata );
/*
* close interface
*/
(void) H5Gclose( grp_id );
}
/*+++++++++++++++++++++++++ Main Program or Function +++++++++++++++*/
void SCIA_LV1_WR_H5_SCPN( struct param_record param, unsigned int nr_scpn,
const struct scpn_scia *scpn )
{
hid_t ads_id;
hsize_t adim;
herr_t stat;
hid_t scpn_type[NFIELDS];
const hbool_t compress = (param.flag_deflate == PARAM_SET) ? TRUE : FALSE;
const size_t scpn_size = sizeof( struct scpn_scia );
const char *scpn_names[NFIELDS] = {
"mjd", "flag_mds", "orbit_phase", "srs_param", "num_lines",
"wv_error_calib", "sol_spec", "line_pos", "coeffs"
};
const size_t scpn_offs[NFIELDS] = {
HOFFSET( struct scpn_scia, mjd ),
HOFFSET( struct scpn_scia, flag_mds ),
HOFFSET( struct scpn_scia, orbit_phase ),
HOFFSET( struct scpn_scia, srs_param ),
HOFFSET( struct scpn_scia, num_lines ),
HOFFSET( struct scpn_scia, wv_error_calib ),
HOFFSET( struct scpn_scia, sol_spec ),
HOFFSET( struct scpn_scia, line_pos ),
HOFFSET( struct scpn_scia, coeffs )
};
/*
* check number of PMD records
*/
if ( nr_scpn == 0 ) return;
/*
* open/create group /ADS
*/
ads_id = NADC_OPEN_HDF5_Group( param.hdf_file_id, "/ADS" );
if ( ads_id < 0 ) NADC_RETURN_ERROR( NADC_ERR_HDF_GRP, "/ADS" );
/*
* write SCPN data sets
*/
adim = SCIENCE_PIXELS;
scpn_type[0] = H5Topen( param.hdf_file_id, "mjd", H5P_DEFAULT );
scpn_type[1] = H5Tcopy( H5T_NATIVE_UCHAR );
scpn_type[2] = H5Tcopy( H5T_NATIVE_FLOAT );
adim = SCIENCE_CHANNELS;
scpn_type[3] = H5Tarray_create( H5T_NATIVE_UCHAR, 1, &adim );
scpn_type[4] = H5Tarray_create( H5T_NATIVE_USHORT, 1, &adim );
scpn_type[5] = H5Tarray_create( H5T_NATIVE_FLOAT, 1, &adim );
adim = SCIENCE_PIXELS;
scpn_type[6] = H5Tarray_create( H5T_NATIVE_FLOAT, 1, &adim );
adim = 3 * SCIENCE_CHANNELS;
scpn_type[7] = H5Tarray_create( H5T_NATIVE_FLOAT, 1, &adim );
adim = NUM_SPEC_COEFFS * SCIENCE_CHANNELS;
scpn_type[8] = H5Tarray_create( H5T_NATIVE_DOUBLE, 1, &adim );
stat = H5TBmake_table( "scpn", ads_id, "NEW_SPECTRAL_CALIBRATION",
NFIELDS, 1, scpn_size, scpn_names,
scpn_offs, scpn_type, 1,
NULL, compress, scpn );
if ( stat < 0 ) NADC_GOTO_ERROR( NADC_ERR_HDF_DATA, "scpn" );
/*
* close interface
*/
done:
(void) H5Tclose( scpn_type[0] );
(void) H5Tclose( scpn_type[1] );
(void) H5Tclose( scpn_type[2] );
(void) H5Tclose( scpn_type[3] );
(void) H5Tclose( scpn_type[4] );
(void) H5Tclose( scpn_type[5] );
(void) H5Tclose( scpn_type[6] );
(void) H5Tclose( scpn_type[7] );
(void) H5Tclose( scpn_type[8] );
(void) H5Gclose( ads_id );
}
int ias_ancillary_build_attitude_table_definition
(
const char *field_names[ATTITUDE_NFIELDS], /* I/O: Names of table
fields */
size_t field_offsets[ATTITUDE_NFIELDS], /* I/O: Field offsets */
hid_t field_types[ATTITUDE_NFIELDS], /* I/O: Field datatypes */
size_t field_sizes[ATTITUDE_NFIELDS], /* I/O: Field sizes */
hid_t fields_to_close[ATTITUDE_NFIELDS] /* I/O: Array of open field
datatypes */
)
{
hsize_t quaternion_dims = QDIMS; /* Number of quaternion
components */
int i; /* Field counter */
IAS_ANC_ATTITUDE_RECORD attitude_record; /* "Template" for record
data structure */
/* Initialize all datatype fields as 'closed' */
for (i = 0; i < ATTITUDE_NFIELDS; i++)
fields_to_close[i] = -1;
/* Dig into the record data structure. Start with the elapsed time
in seconds from the given epoch. */
i = 0;
field_offsets[i] = HOFFSET(IAS_ANC_ATTITUDE_RECORD, seconds_from_epoch);
field_names[i] = "Time From Epoch";
field_types[i] = H5T_NATIVE_DOUBLE;
field_sizes[i] = sizeof(attitude_record.seconds_from_epoch);
field_offsets[++i] = HOFFSET(IAS_ANC_ATTITUDE_RECORD, eci_quaternion);
field_names[i] = "ECI Quaternion";
field_types[i] = H5Tarray_create(H5T_NATIVE_DOUBLE, 1, &quaternion_dims);
if (field_types[i] < 0)
{
IAS_LOG_ERROR("Creating ECI quaternion array type");
ias_ancillary_cleanup_table_definition(fields_to_close,
ATTITUDE_NFIELDS);
return ERROR;
}
fields_to_close[i] = field_types[i];
field_sizes[i] = sizeof(attitude_record.eci_quaternion);
field_offsets[++i] = HOFFSET(IAS_ANC_ATTITUDE_RECORD, ecef_quaternion);
field_names[i] = "ECEF Quaternion";
field_types[i] = H5Tarray_create(H5T_NATIVE_DOUBLE, 1, &quaternion_dims);
if (field_types[i] < 0)
{
IAS_LOG_ERROR("Creating ECEF quaternion array type");
ias_ancillary_cleanup_table_definition(fields_to_close,
ATTITUDE_NFIELDS);
return ERROR;
}
fields_to_close[i] = field_types[i];
field_sizes[i] = sizeof(attitude_record.ecef_quaternion);
field_offsets[++i] = HOFFSET(IAS_ANC_ATTITUDE_RECORD, roll);
field_names[i] = "Roll";
field_types[i] = H5T_NATIVE_DOUBLE;
field_sizes[i] = sizeof(attitude_record.roll);
field_offsets[++i] = HOFFSET(IAS_ANC_ATTITUDE_RECORD, roll_rate);
field_names[i] = "Roll Rate";
field_types[i] = H5T_NATIVE_DOUBLE;
field_sizes[i] = sizeof(attitude_record.roll_rate);
field_offsets[++i] = HOFFSET(IAS_ANC_ATTITUDE_RECORD, pitch);
field_names[i] = "Pitch";
field_types[i] = H5T_NATIVE_DOUBLE;
field_sizes[i] = sizeof(attitude_record.pitch);
field_offsets[++i] = HOFFSET(IAS_ANC_ATTITUDE_RECORD, pitch_rate);
field_names[i] = "Pitch Rate";
field_types[i] = H5T_NATIVE_DOUBLE;
field_sizes[i] = sizeof(attitude_record.pitch_rate);
field_offsets[++i] = HOFFSET(IAS_ANC_ATTITUDE_RECORD, yaw);
field_names[i] = "Yaw";
field_types[i] = H5T_NATIVE_DOUBLE;
field_sizes[i] = sizeof(attitude_record.yaw);
field_offsets[++i] = HOFFSET(IAS_ANC_ATTITUDE_RECORD, yaw_rate);
field_names[i] = "Yaw Rate";
field_types[i] = H5T_NATIVE_DOUBLE;
field_sizes[i] = sizeof(attitude_record.yaw_rate);
/* Final error checks */
if (i < (ATTITUDE_NFIELDS - 1))
{
/* fewer fields were added than expected */
IAS_LOG_ERROR("Defined %d fields, but expected %d", i + 1,
ATTITUDE_NFIELDS);
ias_ancillary_cleanup_table_definition(fields_to_close,
ATTITUDE_NFIELDS);
return ERROR;
}
else if (i >= ATTITUDE_NFIELDS)
{
/* more fields than expected. The stack is probably now corrupt so
just exit since this is an obvious programming error that was just
introduced. */
IAS_LOG_ERROR("Too many fields found - stack probably "
"corrupted - exiting");
exit(EXIT_FAILURE);
}
/* Done */
return SUCCESS;
}
ColTypeDimArray()
{
const hsize_t dim[] = {DSP_DIM_MAX};
this->type = H5Tarray_create(H5T_NATIVE_HSIZE, 1, dim);
}
static datatype value()
{
auto const shape = utility::Shape<T>::as_array();
return H5Tarray_create(datatype_from<base_type>::value(),
shape.size(), shape.data());
}
/*****************************************************************************
This function generates attributes, groups, and datasets of many types.
Parameters:
fname: file_name.
ngrps: number of top level groups.
ndsets: number of datasets.
attrs: number of attributes.
nrow: number of rows in a dataset.
chunk: chunk size (single number).
vlen: max vlen size.
comp: use latest format.
latest: use gzip comnpression.
Return: Non-negative on success/Negative on failure
Programmer: Peter Cao <*****@*****.**>, Jan. 2013
****************************************************************************/
herr_t create_perf_test_file(const char *fname, int ngrps, int ndsets,
int nattrs, hsize_t nrows, hsize_t dim0, hsize_t chunk, int vlen,
int compressed, int latest)
{
int i, j, k;
hid_t fid, sid_null, sid_scalar, sid_1d, sid_2d, did, aid, sid_2, sid_large,
fapl=H5P_DEFAULT, dcpl=H5P_DEFAULT, gid1, gid2, cmp_tid, tid_str,
tid_enum, tid_array_f, tid_vlen_i, tid_vlen_s;
char name[32], tmp_name1[32], tmp_name2[32], tmp_name3[32];
hsize_t dims[1]={dim0}, dims2d[2]={dim0, (dim0/4+1)}, dims_array[1]={FIXED_LEN},
dim1[1]={2};
char *enum_names[4] = {"SOLID", "LIQUID", "GAS", "PLASMA"};
test_comp_t *buf_comp=NULL, *buf_comp_large=NULL;
int *buf_int=NULL;
float (*buf_float_a)[FIXED_LEN]=NULL;
double **buf_double2d=NULL;
hvl_t *buf_vlen_i=NULL;
char (*buf_str)[FIXED_LEN];
char **buf_vlen_s=NULL;
hobj_ref_t buf_ref[2];
hdset_reg_ref_t buf_reg_ref[2];
size_t offset, len;
herr_t status;
char *names[NTYPES] = { "int", "ulong", "float", "double", "fixed string",
"enum", "fixed float array", "vlen int array", "vlen strings"};
hid_t types[NTYPES] = { H5T_NATIVE_INT, H5T_NATIVE_UINT64, H5T_NATIVE_FLOAT,
H5T_NATIVE_DOUBLE, tid_str, tid_enum, tid_array_f, tid_vlen_i, tid_vlen_s};
hsize_t coords[4][2] = { {0, 1}, {3, 5}, {1, 0}, {2, 4}}, start=0, stride=1, count=1;
if (nrows < NROWS) nrows = NROWS;
if (ngrps<NGROUPS) ngrps=NGROUPS;
if (ndsets<NDSETS) ndsets=NDSETS;
if (nattrs<NATTRS) nattrs=NATTRS;
if (dim0<DIM0) dim0=DIM0;
if (chunk>dim0) chunk=dim0/4;
if (chunk<1) chunk = 1;
if (vlen<1) vlen = MAXVLEN;
/* create fixed string datatype */
types[4] = tid_str = H5Tcopy (H5T_C_S1);
H5Tset_size (tid_str, FIXED_LEN);
/* create enum datatype */
types[5] = tid_enum = H5Tenum_create(H5T_NATIVE_INT);
for (i = (int) SOLID; i <= (int) PLASMA; i++) {
phase_t val = (phase_t) i;
status = H5Tenum_insert (tid_enum, enum_names[i], &val);
}
/* create float array datatype */
types[6] = tid_array_f = H5Tarray_create (H5T_NATIVE_FLOAT, 1, dims_array);
/* create variable length integer datatypes */
types[7] = tid_vlen_i = H5Tvlen_create (H5T_NATIVE_INT);
/* create variable length string datatype */
types[8] = tid_vlen_s = H5Tcopy (H5T_C_S1);
H5Tset_size (tid_vlen_s, H5T_VARIABLE);
/* create compound datatypes */
cmp_tid = H5Tcreate (H5T_COMPOUND, sizeof (test_comp_t));
offset = 0;
for (i=0; i<NTYPES-2; i++) {
H5Tinsert(cmp_tid, names[i], offset, types[i]);
offset += H5Tget_size(types[i]);
}
H5Tinsert(cmp_tid, names[7], offset, types[7]);
offset += sizeof (hvl_t);
H5Tinsert(cmp_tid, names[8], offset, types[8]);
/* create dataspace */
sid_1d = H5Screate_simple (1, dims, NULL);
sid_2d = H5Screate_simple (2, dims2d, NULL);
sid_2 = H5Screate_simple (1, dim1, NULL);
sid_large = H5Screate_simple (1, &nrows, NULL);
sid_null = H5Screate (H5S_NULL);
sid_scalar = H5Screate (H5S_SCALAR);
/* create fid access property */
fapl = H5Pcreate (H5P_FILE_ACCESS);
H5Pset_libver_bounds (fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);
/* create dataset creation property */
dcpl = H5Pcreate (H5P_DATASET_CREATE);
/* set dataset chunk */
if (chunk>0) {
H5Pset_chunk (dcpl, 1, &chunk);
}
/* set dataset compression */
if (compressed) {
if (chunk<=0) {
chunk = dim0/10+1;;
H5Pset_chunk (dcpl, 1, &chunk);
}
H5Pset_shuffle (dcpl);
H5Pset_deflate (dcpl, 6);
}
/* allocate buffers */
buf_comp = (test_comp_t *)calloc(dim0, sizeof(test_comp_t));
buf_comp_large = (test_comp_t *)calloc(nrows, sizeof(test_comp_t));
buf_int = (int *)calloc(dim0, sizeof(int));
buf_float_a = malloc(dim0*sizeof(*buf_float_a));
buf_vlen_i = (hvl_t *)calloc(dim0, sizeof (hvl_t));
buf_vlen_s = (char **)calloc(dim0, sizeof(char *));
buf_str = malloc(dim0*sizeof (*buf_str));
/* allocate array of doulbe pointers */
buf_double2d = (double **)calloc(dims2d[0],sizeof(double *));
/* allocate a contigous chunk of memory for the data */
buf_double2d[0] = (double *)calloc( dims2d[0]*dims2d[1],sizeof(double) );
/* assign memory city to pointer array */
for (i=1; i <dims2d[0]; i++) buf_double2d[i] = buf_double2d[0]+i*dims2d[1];
/* fill buffer values */
len = 1;
for (i=0; i<dims[0]; i++) {
buf_comp[i].i = buf_int[i] = i-2147483648;
buf_comp[i].l = 0xffffffffffffffff-i;
buf_comp[i].f = 1.0/(i+1.0);
buf_comp[i].d = 987654321.0*i+1.0/(i+1.0);
buf_comp[i].e = (phase_t) (i % (int) (PLASMA + 1));
for (j=0; j<FIXED_LEN; j++) {
buf_comp[i].f_array[j] = buf_float_a[i][j] = i*100+j;
buf_str[i][j] = 'a' + (i%26);
}
buf_str[i][FIXED_LEN-1] = 0;
strcpy(buf_comp[i].s, buf_str[i]);
len = (1-cos(i/8.0))/2*vlen+1;
if (!i) len = vlen;
buf_vlen_i[i].len = len;
buf_vlen_i[i].p = (int *)calloc(len, sizeof(int));
for (j=0; j<len; j++) ((int*)(buf_vlen_i[i].p))[j] = i*100+j;
buf_comp[i].i_vlen = buf_vlen_i[i];
buf_vlen_s[i] = (char *)calloc(len, sizeof(char));
for (j=0; j<len-1; j++)
buf_vlen_s[i][j] = j%26+'A';
buf_comp[i].s_vlen = buf_vlen_s[i];
for (j=0; j<dims2d[1]; j++)
buf_double2d[i][j] = i+j/10000.0;
}
for (i=0; i<nrows; i++) {
buf_comp_large[i].i = i-2147483648;
buf_comp_large[i].l = 0xffffffffffffffff-i;
buf_comp_large[i].f = 1.0/(i+1.0);
buf_comp_large[i].d = 987654321.0*i+1.0/(i+1.0);
buf_comp_large[i].e = (phase_t) (i % (int) (PLASMA + 1));
for (j=0; j<FIXED_LEN-1; j++) {
buf_comp_large[i].f_array[j] = i*100+j;
buf_comp_large[i].s[j] = 'a' + (i%26);
}
len = i%vlen+1;
buf_comp_large[i].i_vlen.len = len;
buf_comp_large[i].i_vlen.p = (int *)calloc(len, sizeof(int));
for (j=0; j<len; j++) ((int*)(buf_comp_large[i].i_vlen.p))[j] = i*100+j;
buf_comp_large[i].s_vlen = (char *)calloc(i+2, sizeof(char));
for (j=0; j<i+1; j++) (buf_comp_large[i].s_vlen)[j] = j%26+'A';
}
/* create file */
if (latest)
fid = H5Fcreate (fname, H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
else
fid = H5Fcreate (fname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
add_attrs(fid, 0);
sprintf(name, "a cmp ds of %d rows", nrows);
did = H5Dcreate (fid, name, cmp_tid, sid_large, H5P_DEFAULT, dcpl, H5P_DEFAULT);
H5Dwrite (did, cmp_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_comp_large);
add_attrs(did, 0);
H5Dclose(did);
// /* add attributes*/
gid1 = H5Gcreate (fid, "attributes", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (nattrs<1) nattrs = 1;
i=0;
while (i<nattrs) i += add_attrs(gid1, i);
H5Gclose(gid1);
/* add many sub groups to a group*/
gid1 = H5Gcreate (fid, "groups", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
add_attrs(gid1, 0);
for (i=0; i<ngrps; i++) {
/* create sub groups */
sprintf(name, "g%02d", i);
gid2 = H5Gcreate (gid1, name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (i<10) add_attrs(gid2, 0);
H5Gclose(gid2);
}
H5Gclose(gid1);
/* add many datasets to a group */
gid1 = H5Gcreate (fid, "datasets", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
add_attrs(gid1, 0);
for (j=0; j<ndsets; j+=12) {
/* 1 add a null dataset */
sprintf(name, "%05d null dataset", j);
did = H5Dcreate (gid1, name, H5T_STD_I32LE, sid_null, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 2 add scalar int point */
sprintf(name, "%05d scalar int point", j);
did = H5Dcreate (gid1, name, H5T_NATIVE_INT, sid_scalar, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Dwrite (did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &j);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 3 scalar vlen string */
sprintf(name, "%05d scalar vlen string", j);
did = H5Dcreate (gid1, name, tid_vlen_s, sid_scalar, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Dwrite (did, tid_vlen_s, H5S_ALL, H5S_ALL, H5P_DEFAULT, &buf_vlen_s[0]);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 4 add fixed-length float array */
sprintf(name, "%05d fixed-length float array", j);
did = H5Dcreate (gid1, name, tid_array_f, sid_1d, H5P_DEFAULT, dcpl, H5P_DEFAULT);
H5Dwrite (did, tid_array_f, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_float_a);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 5 add fixed-length strings */
sprintf(name, "%05d fixed-length strings", j);
did = H5Dcreate (gid1, name, tid_str, sid_1d, H5P_DEFAULT, dcpl, H5P_DEFAULT);
H5Dwrite (did, tid_str, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_str);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 6 add compound data */
sprintf(name, "%05d compund data", j);
did = H5Dcreate (gid1, name, cmp_tid, sid_1d, H5P_DEFAULT, dcpl, H5P_DEFAULT);
H5Dwrite (did, cmp_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_comp);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 7 add 2D double */
sprintf(name, "%05d 2D double", j);
strcpy (tmp_name1, name);
did = H5Dcreate (gid1, name, H5T_NATIVE_DOUBLE, sid_2d, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Dwrite (did, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_double2d[0]);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 8 add 1D int array */
sprintf(name, "%05d 1D int array", j);
did = H5Dcreate (gid1, name, H5T_NATIVE_INT, sid_1d, H5P_DEFAULT, dcpl, H5P_DEFAULT);
H5Dwrite (did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_int);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 9 add vlen int array */
sprintf(name, "%05d vlen int array", j);
strcpy (tmp_name2, name);
did = H5Dcreate (gid1, name, tid_vlen_i, sid_1d, H5P_DEFAULT, dcpl, H5P_DEFAULT);
H5Dwrite (did, tid_vlen_i, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_vlen_i);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 10 add vlen strings */
sprintf(name, "%05d vlen strings", j);
strcpy (tmp_name3, name);
did = H5Dcreate (gid1, name, tid_vlen_s, sid_1d, H5P_DEFAULT, dcpl, H5P_DEFAULT);
H5Dwrite (did, tid_vlen_s, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_vlen_s);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 11 add object refs */
H5Rcreate(&buf_ref[0],gid1, ".", H5R_OBJECT, -1);
H5Rcreate(&buf_ref[1],gid1, tmp_name3, H5R_OBJECT, -1);
sprintf(name, "%05d obj refs", j);
did = H5Dcreate (gid1, name, H5T_STD_REF_OBJ, sid_2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Dwrite (did, H5T_STD_REF_OBJ, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_ref);
if (!j) add_attrs(did, j);
H5Dclose(did);
/* 12 add region refs */
H5Sselect_elements (sid_2d, H5S_SELECT_SET, 4, coords[0]);
H5Rcreate(&buf_reg_ref[0],gid1, tmp_name1, H5R_DATASET_REGION, sid_2d);
H5Sselect_none(sid_2d);
count = dims[0]/2+1;
H5Sselect_hyperslab (sid_1d, H5S_SELECT_SET, &start, &stride, &count,NULL);
H5Rcreate(&buf_reg_ref[1],gid1, tmp_name2, H5R_DATASET_REGION, sid_1d);
H5Sselect_none(sid_1d);
sprintf(name, "%05d region refs", j);
did = H5Dcreate (gid1, name, H5T_STD_REF_DSETREG, sid_2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Dwrite (did, H5T_STD_REF_DSETREG, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_reg_ref);
if (!j) add_attrs(did, j);
H5Dclose(did);
}
H5Gclose(gid1);
H5Tclose (tid_array_f);
H5Tclose (tid_vlen_i);
H5Tclose (tid_vlen_s);
H5Tclose (tid_enum);
H5Tclose (tid_str);
H5Tclose (cmp_tid);
H5Pclose (dcpl);
H5Pclose (fapl);
H5Sclose (sid_1d);
H5Sclose (sid_2d);
H5Sclose (sid_2);
H5Sclose (sid_large);
H5Sclose (sid_null);
H5Sclose (sid_scalar);
H5Fclose (fid);
for (i=0; i<dims[0]; i++) {
if (buf_vlen_i[i].p) free(buf_vlen_i[i].p);
if (buf_vlen_s[i]) free(buf_vlen_s[i]);
}
for (i=0; i<nrows; i++) {
if (buf_comp_large[i].i_vlen.p) free(buf_comp_large[i].i_vlen.p);
if (buf_comp_large[i].s_vlen) free(buf_comp_large[i].s_vlen);
}
free (buf_comp);
free (buf_comp_large);
free (buf_int);
free (buf_float_a);
free (buf_double2d[0]);
free (buf_double2d);
free (buf_str);
free(buf_vlen_i);
free(buf_vlen_s);
return 0;
}
/*
adds different types of attributes to an object.
returns the number of attributes added to the objects.
*/
int add_attrs(hid_t oid, int idx)
{
char name[32];
int i0, i1, i2, j, nattrs=0;
hid_t aid, tid, tid1, sid;
hvl_t i_vlen[4];
hobj_ref_t ref;
zipcode_t cmp_data[4];
unsigned int i = 0xffffffff;
long long l = -2147483647;
float f = 123456789.987654321;
double d = 987654321.123456789;
char *s[7] = {"Parting", "is such", "sweeter", "sorrow."};
float f_array[4] = {1.0, 2.22, 3.333, 4.444};
char *s_vlen[4] = {"Parting", "is such", "sweet", "sorrow."};
hsize_t dims1[1]={1}, dims2[1]={4}, dims3[2]={3,5};
int int3d[4][3][5];
size_t offset = 0;
for (i0=0; i0<4; i0++) {
i_vlen[i0].len = (i0+1);
i_vlen[i0].p = (int *)calloc(i_vlen[i0].len, sizeof(int));
for (j=0; j<i_vlen[i0].len; j++)
((int *)i_vlen[i0].p)[j] = i0*100+j;
for (i1=0; i1<3; i1++) {
for (i2=0; i2<5; i2++)
int3d[i0][i1][i2] = i0*i1-i1*i2+i0*i2;
}
}
cmp_data[0].zipcode = 01001;
cmp_data[0].city = "Agawam, Massachusetts";
cmp_data[1].zipcode = 99950;
cmp_data[1].city = "Ketchikan, Alaska";
cmp_data[2].zipcode = 00501;
cmp_data[2].city = "Holtsville, New York";
cmp_data[3].zipcode = 61820;
cmp_data[3].city = "Champaign, Illinois";
/* 1 scalar point */
sid = H5Screate (H5S_SCALAR);
sprintf(name, "%05d scalar int", idx);
nattrs += add_attr(oid, name, H5T_NATIVE_UINT, sid, &i);
sprintf(name, "%05d scalar ulong", idx);
nattrs += add_attr(oid, name, H5T_NATIVE_INT64, sid, &l);
sprintf(name, "%05d scalar str", idx);
tid = H5Tcopy (H5T_C_S1);
H5Tset_size (tid, H5T_VARIABLE);
nattrs += add_attr(oid, name, tid, sid, &s[2]);
H5Tclose(tid);
H5Sclose(sid);
/* 4 single point */
sid = H5Screate_simple (1, dims1, NULL);
H5Rcreate(&ref, oid, ".", H5R_OBJECT, -1);
sprintf(name, "%05d single float", idx);
nattrs += add_attr(oid, name, H5T_NATIVE_FLOAT, sid, &f);
sprintf(name, "%05d single double", idx);
nattrs += add_attr(oid, name, H5T_NATIVE_DOUBLE, sid, &d);
sprintf(name, "%05d single obj_ref", idx);
nattrs += add_attr(oid, name, H5T_STD_REF_OBJ, sid, &ref);
H5Sclose(sid);
/* 7 fixed length 1D array */
sid = H5Screate_simple (1, dims1, NULL);
tid = H5Tarray_create (H5T_NATIVE_FLOAT, 1, dims2);
sprintf(name, "%05d array float", idx);
nattrs += add_attr(oid, name, tid, sid, &f_array[0]);
H5Tclose(tid);
tid = H5Tcopy (H5T_C_S1);
H5Tset_size (tid, strlen(s[0])+1);
tid1 = H5Tarray_create (tid, 1, dims2);
sprintf(name, "%05d array str", idx);
nattrs += add_attr(oid, name, tid1, sid, s);
H5Tclose(tid1);
H5Tclose(tid);
H5Sclose(sid);
/* 9 fixed length 2D int arrays */
sid = H5Screate_simple (1, dims2, NULL);
tid = H5Tarray_create (H5T_NATIVE_INT, 2, dims3);
sprintf(name, "%05d array int 2D", idx);
nattrs += add_attr(oid, name, tid, sid, int3d[0][0]);
H5Tclose(tid);
H5Sclose(sid);
/* 10 variable length arrays */
sid = H5Screate_simple (1, dims2, NULL);
tid = H5Tcopy (H5T_C_S1);
H5Tset_size (tid, H5T_VARIABLE);
sprintf(name, "%05d vlen strings", idx);
nattrs += add_attr(oid, name, tid, sid, s_vlen);
H5Tclose(tid);
tid = H5Tvlen_create (H5T_NATIVE_INT);;
sprintf(name, "%05d vlen int array", idx);
nattrs += add_attr(oid, name, tid, sid, i_vlen);
H5Tclose(tid);
H5Sclose(sid);
/* 12 compound data */
sid = H5Screate_simple (1, dims2, NULL);
tid = H5Tcreate (H5T_COMPOUND, sizeof (zipcode_t));
tid1 = H5Tcopy (H5T_C_S1);
H5Tset_size (tid1, H5T_VARIABLE);
H5Tinsert (tid, "zip code", 0, H5T_NATIVE_INT); offset += sizeof(H5T_NATIVE_INT);
H5Tinsert (tid, "City", offset, tid1); offset += sizeof(char *);
sprintf(name, "%05d compound data", idx);
nattrs += add_attr(oid, name, tid, sid, cmp_data);
H5Tclose(tid1);
H5Tclose(tid);
H5Sclose(sid);
for (i0=0; i0<4; i0++)
free(i_vlen[i0].p);
return nattrs;
}
/*+++++++++++++++++++++++++ Main Program or Function +++++++++++++++*/
void SCIA_LV1_WR_H5_VLCP( struct param_record param, unsigned int nr_vlcp,
const struct vlcp_scia *vlcp )
{
hid_t gads_id;
hsize_t adim;
herr_t stat;
hid_t vlcp_type[NFIELDS];
const hbool_t compress = (param.flag_deflate == PARAM_SET) ? TRUE : FALSE;
const char *vlcp_names[NFIELDS] = {
"orbit_phase", "obm_pmd", "var_lc", "var_lc_error", "solar_stray",
"solar_stray_error", "pmd_stray", "pmd_stray_error", "pmd_dark",
"pmd_dark_error"
};
const size_t vlcp_size = sizeof( struct vlcp_scia );
const size_t vlcp_offs[NFIELDS] = {
HOFFSET( struct vlcp_scia, orbit_phase ),
HOFFSET( struct vlcp_scia, obm_pmd ),
HOFFSET( struct vlcp_scia, var_lc ),
HOFFSET( struct vlcp_scia, var_lc_error ),
HOFFSET( struct vlcp_scia, solar_stray ),
HOFFSET( struct vlcp_scia, solar_stray_error ),
HOFFSET( struct vlcp_scia, pmd_stray ),
HOFFSET( struct vlcp_scia, pmd_stray_error ),
HOFFSET( struct vlcp_scia, pmd_dark ),
HOFFSET( struct vlcp_scia, pmd_dark_error )
};
/*
* check number of RSPO records
*/
if ( nr_vlcp == 0 ) return;
/*
* open/create group /GADS
*/
gads_id = NADC_OPEN_HDF5_Group( param.hdf_file_id, "/GADS" );
if ( gads_id < 0 ) NADC_RETURN_ERROR( NADC_ERR_HDF_GRP, "/GADS" );
/*
* write VLCP data sets
*/
vlcp_type[0] = H5Tcopy( H5T_NATIVE_FLOAT );
adim = IR_CHANNELS + PMD_NUMBER;
vlcp_type[1] = H5Tcopy( H5T_NATIVE_FLOAT );
adim = IR_CHANNELS * CHANNEL_SIZE;
vlcp_type[2] = H5Tcopy( H5T_NATIVE_FLOAT );
vlcp_type[3] = H5Tcopy( H5T_NATIVE_FLOAT );
adim = SCIENCE_PIXELS;
vlcp_type[4] = H5Tcopy( H5T_NATIVE_FLOAT );
vlcp_type[5] = H5Tarray_create( H5T_NATIVE_FLOAT, 1, &adim );
adim = PMD_NUMBER;
vlcp_type[6] = H5Tarray_create( H5T_NATIVE_FLOAT, 1, &adim );
vlcp_type[7] = H5Tarray_create( H5T_NATIVE_FLOAT, 1, &adim );
adim = IR_PMD_NUMBER;
vlcp_type[8] = H5Tarray_create( H5T_NATIVE_FLOAT, 1, &adim );
vlcp_type[9] = H5Tarray_create( H5T_NATIVE_FLOAT, 1, &adim );
stat = H5TBmake_table( "vlcp", gads_id, "LEAKAGE_VARIABLE",
NFIELDS, nr_vlcp, vlcp_size, vlcp_names,
vlcp_offs, vlcp_type, 1,
NULL, compress, vlcp );
if ( stat < 0 ) NADC_GOTO_ERROR( NADC_ERR_HDF_DATA, "vlcp" );
/*
* close interface
*/
done:
(void) H5Tclose( vlcp_type[0] );
(void) H5Tclose( vlcp_type[1] );
(void) H5Tclose( vlcp_type[2] );
(void) H5Tclose( vlcp_type[3] );
(void) H5Tclose( vlcp_type[4] );
(void) H5Tclose( vlcp_type[5] );
(void) H5Tclose( vlcp_type[6] );
(void) H5Tclose( vlcp_type[7] );
(void) H5Tclose( vlcp_type[8] );
(void) H5Tclose( vlcp_type[9] );
(void) H5Gclose( gads_id );
}
void UPCdata::writeHDFfile(char* filename){
#if HDF5_AVAILABLE == 1
#define NRECORDS 50
//const int NRECORDS=samples.size();
const int NFIELDS = 5;
hsize_t chunk_size = 10; // ??
typedef struct _samplestruct{
float data[1020]; // having difficulties here; dynamic allocation?
float concentrations[3];
int samplenr;
int set;
int compoundnr;
}samplestruct_t;
cout << "converting data format" << endl;
samplestruct_t* sampledata=NULL;
sampledata=new samplestruct_t[NRECORDS];
for(int i=0;i<NRECORDS;i++){
for(int j=0;j<1020;j++)
sampledata[i].data[j]=samples[i].data[j];
sampledata[i].concentrations[0]=samples[i].concentrations[0];
sampledata[i].concentrations[1]=samples[i].concentrations[1];
sampledata[i].concentrations[2]=samples[i].concentrations[2];
sampledata[i].samplenr=samples[i].samplenr;
sampledata[i].set=samples[i].set;
sampledata[i].compoundnr=samples[i].compoundnr;
}
size_t dst_offset[NFIELDS] = {
HOFFSET(samplestruct_t, data),
HOFFSET(samplestruct_t, concentrations),
HOFFSET(samplestruct_t, samplenr),
HOFFSET(samplestruct_t, set),
HOFFSET(samplestruct_t, compoundnr),
};
size_t dst_sizes[NFIELDS] = {
sizeof( sampledata[0].data),
sizeof(sampledata[0].concentrations),
sizeof(sampledata[0].samplenr),
sizeof(sampledata[0].set),
sizeof(sampledata[0].compoundnr)
};
hsize_t dim[] = {NRECORDS}; /* Dataspace dimensions */
hid_t space = H5Screate_simple(1, dim, NULL); // one dimensional in this case
hid_t file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); // Create a new file using default properties.
const hsize_t datadim[] = {1020};
const hsize_t concdim[] = {3};
hid_t dataarray = H5Tarray_create(H5T_NATIVE_FLOAT, 1, datadim);
hid_t concarray = H5Tarray_create(H5T_NATIVE_FLOAT, 1, concdim);
hid_t s_tid = H5Tcreate (H5T_COMPOUND, sizeof(samplestruct_t));
H5Tinsert(s_tid, "Data", HOFFSET(samplestruct_t, data), dataarray);
H5Tinsert(s_tid, "Concentrations", HOFFSET(samplestruct_t, concentrations), concarray);
H5Tinsert(s_tid, "samplenr", HOFFSET(samplestruct_t, samplenr), H5T_NATIVE_INT);
H5Tinsert(s_tid, "set", HOFFSET(samplestruct_t, set), H5T_NATIVE_INT);
H5Tinsert(s_tid, "compoundnr", HOFFSET(samplestruct_t, compoundnr), H5T_NATIVE_INT);
hid_t dataset = H5Dcreate1(file, "/dataset3", s_tid, space, H5P_DEFAULT);
herr_t status = H5Dwrite(dataset, s_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, sampledata);
delete [] sampledata;
H5Tclose(dataarray);
H5Tclose(concarray);
H5Tclose(s_tid);
H5Sclose(space);
H5Dclose(dataset);
H5Fclose(file);
#endif
}
ColTypeBool()
{
const hsize_t dim[] = {sizeof (bool)};
this->type = H5Tarray_create(H5T_NATIVE_B8, 1, dim);
}
