void DebugOutput::list_range_real( const char* pfx, const Range& range )
{
if (pfx) {
lineBuffer.insert( lineBuffer.end(), pfx, pfx+strlen(pfx) );
lineBuffer.push_back(' ');
}
if (range.empty()) {
print_real("<empty>\n");
return;
}
char numbuf[48]; // unsigned 64 bit integer can't have more than 20 decimal digits
Range::const_pair_iterator i;
EntityType type = MBMAXTYPE;
for (i = range.const_pair_begin(); i != range.const_pair_end(); ++i) {
if (TYPE_FROM_HANDLE(i->first) != type) {
type = TYPE_FROM_HANDLE(i->first);
const char* name = CN::EntityTypeName(type);
lineBuffer.insert( lineBuffer.end(), name, name+strlen(name) );
}
if (i->first == i->second)
sprintf(numbuf, " %lu,", (unsigned long)(ID_FROM_HANDLE(i->first)));
else
print_range(numbuf, ID_FROM_HANDLE(i->first), ID_FROM_HANDLE(i->second) );
lineBuffer.insert( lineBuffer.end(), numbuf, numbuf+strlen(numbuf) );
}
lineBuffer.push_back('\n');
process_line_buffer();
}
void DebugOutput::list_ints_real( const char* pfx, const Range& range )
{
if (range.empty()) {
print_real("<empty>\n");
return;
}
if (pfx) {
lineBuffer.insert( lineBuffer.end(), pfx, pfx+strlen(pfx) );
lineBuffer.push_back(' ');
}
char numbuf[48]; // unsigned 64 bit integer can't have more than 20 decimal digits
Range::const_pair_iterator i;
for (i = range.const_pair_begin(); i != range.const_pair_end(); ++i) {
if (i->first == i->second)
sprintf(numbuf, " %lu,", (unsigned long)(i->first));
else
print_range(numbuf, (unsigned long)(i->first), (unsigned long)(i->second));
lineBuffer.insert( lineBuffer.end(), numbuf, numbuf+strlen(numbuf) );
}
lineBuffer.push_back('\n');
process_line_buffer();
}
ErrorCode DenseTag::clear_data( bool allocate,
SequenceManager* seqman,
Error* error,
const Range& entities,
const void* value_ptr )
{
ErrorCode rval;
unsigned char* array;
size_t avail;
for (Range::const_pair_iterator p = entities.const_pair_begin();
p != entities.const_pair_end(); ++p) {
EntityHandle start = p->first;
while (start <= p->second) {
rval = get_array( seqman, error, start, array, avail, allocate );
if (MB_SUCCESS != rval)
return rval;
const size_t count = std::min<size_t>(p->second - start + 1, avail);
if (array) // array should never be null if allocate == true
SysUtil::setmem( array, value_ptr, get_size(), count );
start += count;
}
}
return MB_SUCCESS;
}
ErrorCode DenseTag::set_data( SequenceManager* seqman,
Error* error,
const Range& entities,
void const* const* pointers,
const int* /*data_lengths*/ )
{
ErrorCode rval;
unsigned char* array;
size_t avail;
for (Range::const_pair_iterator p = entities.const_pair_begin();
p != entities.const_pair_end(); ++p) {
EntityHandle start = p->first;
while (start <= p->second) {
rval = get_array( seqman, error, start, array, avail, true );
if (MB_SUCCESS != rval)
return rval;
const EntityHandle end = std::min<EntityHandle>(p->second + 1, start + avail );
while (start != end) {
memcpy( array, *pointers, get_size() );
++start;
++pointers;
array += get_size();
}
}
}
return MB_SUCCESS;
}
ErrorCode DenseTag::set_data( SequenceManager* seqman,
Error* error,
const Range& entities,
const void* values )
{
ErrorCode rval;
const char* data = reinterpret_cast<const char*>(values);
unsigned char* array;
size_t avail;
for (Range::const_pair_iterator p = entities.const_pair_begin();
p != entities.const_pair_end(); ++p) {
EntityHandle start = p->first;
while (start <= p->second) {
rval = get_array( seqman, error, start, array, avail, true );
if (MB_SUCCESS != rval)
return rval;
const size_t count = std::min<size_t>(p->second - start + 1, avail);
memcpy( array, data, get_size() * count );
data += get_size() * count;
start += count;
}
}
return MB_SUCCESS;
}
ErrorCode DenseTag::get_data( const SequenceManager* seqman,
Error* error,
const Range& entities,
void* values ) const
{
ErrorCode rval;
size_t avail;
const unsigned char* array = NULL; // initialize to get rid of warning
unsigned char* data = reinterpret_cast<unsigned char*>(values);
for (Range::const_pair_iterator p = entities.const_pair_begin();
p != entities.const_pair_end(); ++p) {
EntityHandle start = p->first;
while (start <= p->second) {
rval = get_array( seqman, error, start, array, avail );
if (MB_SUCCESS != rval)
return rval;
const size_t count = std::min<size_t>(p->second - start + 1, avail);
if (array)
memcpy( data, array, get_size() * count );
else if (get_default_value())
SysUtil::setmem( data, get_default_value(), get_size(), count );
else
return not_found( error, get_name(), start );
data += get_size() * count;
start += count;
}
}
return MB_SUCCESS;
}
ErrorCode DenseTag::get_data( const SequenceManager* seqman,
Error* error,
const Range& entities,
const void** pointers,
int* data_lengths ) const
{
ErrorCode rval;
size_t avail;
const unsigned char* array = 0;
if (data_lengths) {
int len = get_size();
SysUtil::setmem( data_lengths, &len, sizeof(int), entities.size() );
}
for (Range::const_pair_iterator p = entities.const_pair_begin();
p != entities.const_pair_end(); ++p) {
EntityHandle start = p->first;
while (start <= p->second) {
rval = get_array( seqman, error, start, array, avail );
if (MB_SUCCESS != rval)
return rval;
const size_t count = std::min<size_t>(p->second - start + 1, avail);
if (array) {
for (EntityHandle end = start + count; start != end; ++start) {
*pointers = array;
array += get_size();
++pointers;
}
}
else if (const void* val = get_default_value()) {
SysUtil::setmem( pointers, &val, sizeof(void*), count );
pointers += count;
start += count;
}
else {
return not_found( error, get_name(), start );;
}
}
}
return MB_SUCCESS;
}
ErrorCode BitTag::clear_data( SequenceManager* seqman,
Error* error,
const Range& handles,
const void* value_ptr,
int value_len )
{
if (value_len)
return MB_INVALID_SIZE;
ErrorCode rval = seqman->check_valid_entities( error, handles );
if (MB_SUCCESS != rval)
return rval;
EntityType type;
EntityID count;
size_t page;
int offset, per_page = ents_per_page();
const unsigned char value = *reinterpret_cast<const unsigned char*>(value_ptr);
Range::const_pair_iterator i;
for (i = handles.const_pair_begin(); i != handles.const_pair_end(); ++i) {
unpack( i->first, type, page, offset );
assert(TYPE_FROM_HANDLE(i->second) == type); // should be true because id of zero is never used
count = i->second - i->first + 1;
while (count) {
if (page >= pageList[type].size())
pageList[type].resize( page+1, 0 );
if (!pageList[type][page])
pageList[type][page] = new BitPage( storedBitsPerEntity, default_val() );
size_t pcount = std::min( (EntityID)(per_page - offset), count );
pageList[type][page]->set_bits( offset, pcount, storedBitsPerEntity, value );
count -= pcount;
offset = 0;
++page;
}
}
return MB_SUCCESS;
}
ErrorCode BitTag::remove_data( SequenceManager*, Error*, const Range& handles )
{
EntityType type;
EntityID count;
size_t page;
int offset, per_page = ents_per_page();
unsigned char val = default_val();
Range::const_pair_iterator i;
for (i = handles.const_pair_begin(); i != handles.const_pair_end(); ++i) {
unpack( i->first, type, page, offset );
assert(TYPE_FROM_HANDLE(i->second) == type); // should be true because id of zero is never used
count = i->second - i->first + 1;
while (count) {
size_t pcount = std::min( (EntityID)(per_page - offset), count );
if (page < pageList[type].size() && pageList[type][page])
pageList[type][page]->set_bits( offset, pcount, storedBitsPerEntity, val );
count -= pcount;
offset = 0;
++page;
}
}
return MB_SUCCESS;
}
ErrorCode BitTag::get_data( const SequenceManager*,
Error*,
const Range& handles,
void* gen_data ) const
{
EntityType type;
EntityID count;
size_t page;
int offset, per_page = ents_per_page();
unsigned char def = default_val();
unsigned char* data = reinterpret_cast<unsigned char*>(gen_data);
Range::const_pair_iterator i;
for (i = handles.const_pair_begin(); i != handles.const_pair_end(); ++i) {
unpack( i->first, type, page, offset );
assert(TYPE_FROM_HANDLE(i->second) == type); // should be true because id of zero is never used
count = i->second - i->first + 1;
if (page >= pageList[type].size()) {
memset( data, def, count );
data += count;
continue;
}
while (count) {
size_t pcount = std::min( (EntityID)(per_page - offset), count );
if (pageList[type][page])
pageList[type][page]->get_bits( offset, pcount, storedBitsPerEntity, data );
else
memset( data, def, pcount );
data += pcount;
count -= pcount;
offset = 0;
++page;
}
}
return MB_SUCCESS;
}
/*
ErrorCode ReadHDF5VarLen::read_offsets( ReadHDF5Dataset& data_set,
const Range& file_ids,
EntityHandle start_file_id,
unsigned num_columns,
const unsigned indices[],
EntityHandle nudge,
Range offsets_out[],
std::vector<unsigned> counts_out[],
Range* ranged_file_ids = 0 )
{
const int local_index = 1;
// sanity check
const unsigned max_cols = ranged_file_ids ? data_set.columns() - 1 : data_set.columns()
for (unsigned i = 0; i < num_columns; ++i) {
assert(indices[i] >= max_cols);
if (indices[i] >= max_cols)
return MB_FAILURE;
}
// Use hints to make sure insertion into ranges is O(1)
std::vector<Range::iterator> hints;
if (ranged_file_ids) {
hints.resize( num_colums + 1 );
hints.back() = ranged_file_ids->begin();
}
else {
hints.resize( num_columns );
}
for (unsigned i = 0; i < num_columns; ++i)
offsets_out[i].clear();
counts_out[i].clear();
counts_out[i].reserve( file_ids.size() );
hints[i] = offsets_out[i].begin();
}
// If we only need one column from a multi-column data set,
// then read only that column.
if (num_columns == 1 && data_set.columns() > 1 && !ranged_file_ids) {
data_set.set_column( indices[0] );
indices = &local_index;
}
else if (ranged_file_ids && data_set.columns() > 1 && 0 == num_columns) {
data_set.set_column( data_set.columns() - 1 );
}
// NOTE: do not move this above the previous block.
// The previous block changes the results of data_set.columns()!
const size_t table_columns = data_set.columns();
// Calculate which rows we need to read from the offsets table
Range rows;
Range::iterator hint = rows.begin();
Range::const_pair_iterator pair = file_ids.const_pair_begin();
// special case if reading first entity in dataset, because
// there is no previous end value.
if (pair != file_ids.const_pair_end() && pair->first == start_file_id)
hint = rows.insert( nudge, pair->second - start_file_id + nudge );
while (pair != file_ids.const_pair_end()) {
hint = rows.insert( hint,
pair->first + nudge - 1 - start_file_id,
pair->second + nudge - start_file_id );
++pair;
}
// set up read of offsets dataset
hsize_t buffer_size = bufferSize / (sizeof(hssize_t) * data_set.columns());
hssize_t* buffer = reinterpret_cast<hssize_t*>(dataBuffer);
data_set.set_file_ids( rows, nudge, buffer_size, H5T_NATIVE_HSSIZE );
std::vector<hssize_t> prev_end;
// If we're reading the first row of the table, then the
// previous end is implicitly -1.
if (!file_ids.empty() && file_ids.front() == start_file_id)
prev_end.resize(num_columns,-1);
// read offset table
size_t count, offset;
Range::const_iterator fiter = file_ids.begin();
while (!data_set.done()) {
try {
data_set.read( buffer, count );
}
catch (ReadHDF5Dataset::Exception e) {
return MB_FAILURE;
}
if (!count) // might have been NULL read for collective IO
continue;
// If the previous end values were read in the previous iteration,
// then they're stored in prev_end.
size_t offset = 0;
if (!prev_end.empty()) {
for (unsigned i = 0; i < num_columns; ++i) {
counts_out[i].push_back( buffer[indices[i]] - prev_end[i] );
hints[i] = offsets_out[i].insert( hints[i],
prev_end[i] + 1 + nudge,
buffer[indices[i]] + nudge );
}
if (ranged_file_ids && (buffer[table_columns-1] & mhdf_SET_RANGE_BIT))
hints.back() = ranged_file_ids->insert( hints.back(), *fiter );
++fiter;
offset = 1;
prev_end.clear();
}
while (offset < count) {
assert(fiter != file_ids.end());
// whenever we get to a gap between blocks we need to
// advance one step because we read an extra end id
// preceding teah block
if (fiter == fiter.start_of_block()) {
if (offset == count-1)
break;
++offset;
}
for (unsigned i = 0; i < num_columns; ++i) {
size_t s = buffer[(offset-1)*table_columns+indices[i]] + 1;
size_t e = buffer[ offset *table_columns+indices[i]];
counts_out.push_back( e - s + 1 );
hints[i] = offsets_out.insert( hints[i], s, e );
}
if (ranged_file_ids && (buffer[offset*table_columns+table_columns-1] & mhdf_SET_RANGE_BIT))
hints.back() = ranged_file_ids->insert( hints.back(), *fiter );
++fiter;
++offset;
}
// If we did not end on the boundary between two blocks,
// then we need to save the end indices for the final entry
// for use in the next iteration. Similarly, if we ended
// with extra values that were read with the express intention
// of getting the previous end values for a block, we need to
// save them. This case only arises if we hit the break in
// the above loop.
if (fiter != fiter.start_of_block() || offset < count) {
assert(prev_end.empty());
if (offset == count) {
--offset;
assert(fiter != fiter.start_of_block());
}
else {
assert(offset+1 == count);
assert(fiter == fiter.start_of_block());
}
for (unsigned i = 0; i < num_columns; ++i)
prev_end.push_back(buffer[offset*table_columns+indices[i]]);
}
}
assert(prev_end.empty());
assert(fiter == file_ids.end());
return MB_SUCCESS;
}
*/
ErrorCode ReadHDF5VarLen::read_offsets( ReadHDF5Dataset& data_set,
const Range& file_ids,
EntityHandle start_file_id,
EntityHandle nudge,
Range& offsets_out,
std::vector<unsigned>& counts_out )
{
// Use hints to make sure insertion into ranges is O(1)
offsets_out.clear();
counts_out.clear();
counts_out.reserve( file_ids.size() );
Range::iterator hint;
// Calculate which rows we need to read from the offsets table
Range rows;
hint = rows.begin();
Range::const_pair_iterator pair = file_ids.const_pair_begin();
// special case if reading first entity in dataset, because
// there is no previous end value.
if (pair != file_ids.const_pair_end() && pair->first == start_file_id) {
hint = rows.insert( nudge, pair->second - start_file_id + nudge );
++pair;
}
while (pair != file_ids.const_pair_end()) {
hint = rows.insert( hint,
pair->first - start_file_id + nudge - 1,
pair->second - start_file_id + nudge );
++pair;
}
// set up read of offsets dataset
hsize_t buffer_size = bufferSize / sizeof(hssize_t);
hssize_t* buffer = reinterpret_cast<hssize_t*>(dataBuffer);
data_set.set_file_ids( rows, nudge, buffer_size, H5T_NATIVE_HSSIZE );
hssize_t prev_end;
bool have_prev_end = false;
// If we're reading the first row of the table, then the
// previous end is implicitly -1.
if (!file_ids.empty() && file_ids.front() == start_file_id) {
prev_end = -1;
have_prev_end = true;
}
dbgOut.printf( 3, "Reading %s in %lu chunks\n", data_set.get_debug_desc(), data_set.get_read_count() );
// read offset table
size_t count, offset;
Range::const_iterator fiter = file_ids.begin();
hint = offsets_out.begin();
int nn = 0;
while (!data_set.done()) {
dbgOut.printf( 3, "Reading chunk %d of %s\n", ++nn, data_set.get_debug_desc() );
try {
data_set.read( buffer, count );
}
catch (ReadHDF5Dataset::Exception& ) {
return MB_FAILURE;
}
if (!count) // might have been NULL read for collective IO
continue;
// If the previous end values were read in the previous iteration,
// then they're stored in prev_end.
offset = 0;
if (have_prev_end) {
counts_out.push_back( buffer[0] - prev_end );
hint = offsets_out.insert( hint,
prev_end + 1 + nudge,
buffer[0] + nudge );
++fiter;
offset = 1;
have_prev_end = false;
}
while (offset < count) {
assert(fiter != file_ids.end());
// whenever we get to a gap between blocks we need to
// advance one step because we read an extra end id
// preceding teah block
if (fiter == fiter.start_of_block()) {
if (offset == count-1)
break;
++offset;
}
size_t s = buffer[offset-1] + 1;
size_t e = buffer[offset];
counts_out.push_back( e - s + 1 );
hint = offsets_out.insert( hint, s + nudge, e + nudge );
++fiter;
++offset;
}
// If we did not end on the boundary between two blocks,
// then we need to save the end indices for the final entry
// for use in the next iteration. Similarly, if we ended
// with extra values that were read with the express intention
// of getting the previous end values for a block, we need to
// save them. This case only arises if we hit the break in
// the above loop.
if (fiter != fiter.start_of_block() || offset < count) {
assert(!have_prev_end);
if (offset == count) {
--offset;
assert(fiter != fiter.start_of_block());
}
else {
assert(offset+1 == count);
assert(fiter == fiter.start_of_block());
}
have_prev_end = true;
prev_end = buffer[offset];
}
}
assert(!have_prev_end);
assert(fiter == file_ids.end());
return MB_SUCCESS;
}