Exemple #1
0
void DebugOutput::list_range_real( const char* pfx, const Range& range )
{
  if (pfx) {
    lineBuffer.insert( lineBuffer.end(), pfx, pfx+strlen(pfx) );
    lineBuffer.push_back(' ');
  }

  if (range.empty()) {
    print_real("<empty>\n");
    return;
  }

  char numbuf[48]; // unsigned 64 bit integer can't have more than 20 decimal digits
  Range::const_pair_iterator i;
  EntityType type = MBMAXTYPE;
  for (i = range.const_pair_begin(); i != range.const_pair_end(); ++i) {
    if (TYPE_FROM_HANDLE(i->first) != type) {
      type = TYPE_FROM_HANDLE(i->first);
      const char* name = CN::EntityTypeName(type);
      lineBuffer.insert( lineBuffer.end(), name, name+strlen(name) );
    }
    if (i->first == i->second) 
      sprintf(numbuf, " %lu,", (unsigned long)(ID_FROM_HANDLE(i->first)));
    else
      print_range(numbuf, ID_FROM_HANDLE(i->first), ID_FROM_HANDLE(i->second) );
    lineBuffer.insert( lineBuffer.end(), numbuf, numbuf+strlen(numbuf) );
  }

  lineBuffer.push_back('\n');
  process_line_buffer();
}
Exemple #2
0
void DebugOutput::list_ints_real( const char* pfx, const Range& range )
{
  if (range.empty()) {
    print_real("<empty>\n");
    return;
  }

  if (pfx) {
    lineBuffer.insert( lineBuffer.end(), pfx, pfx+strlen(pfx) );
    lineBuffer.push_back(' ');
  }

  char numbuf[48]; // unsigned 64 bit integer can't have more than 20 decimal digits
  Range::const_pair_iterator i;
  for (i = range.const_pair_begin(); i != range.const_pair_end(); ++i) {
    if (i->first == i->second) 
      sprintf(numbuf, " %lu,", (unsigned long)(i->first));
    else
      print_range(numbuf, (unsigned long)(i->first), (unsigned long)(i->second));
    lineBuffer.insert( lineBuffer.end(), numbuf, numbuf+strlen(numbuf) );
  }

  lineBuffer.push_back('\n');
  process_line_buffer();
}
Exemple #3
0
ErrorCode DenseTag::clear_data( bool allocate,
                                SequenceManager* seqman,
                                Error* error,
                                const Range& entities,
                                const void* value_ptr )
{
  ErrorCode rval;
  unsigned char* array;
  size_t avail;

  for (Range::const_pair_iterator p = entities.const_pair_begin(); 
       p != entities.const_pair_end(); ++p) {
       
    EntityHandle start = p->first;
    while (start <= p->second) {
      rval = get_array( seqman, error, start, array, avail, allocate );
      if (MB_SUCCESS != rval)
        return rval;
      
      const size_t count = std::min<size_t>(p->second - start + 1, avail);
      if (array) // array should never be null if allocate == true
        SysUtil::setmem( array, value_ptr, get_size(), count );
      start += count;
    }
  }
  
  return MB_SUCCESS;
}
Exemple #4
0
ErrorCode DenseTag::set_data( SequenceManager* seqman,
                              Error* error,
                              const Range& entities,
                              void const* const* pointers,
                              const int* /*data_lengths*/ )
{
  ErrorCode rval;
  unsigned char* array;
  size_t avail;

  for (Range::const_pair_iterator p = entities.const_pair_begin(); 
       p != entities.const_pair_end(); ++p) {
       
    EntityHandle start = p->first;
    while (start <= p->second) {
      rval = get_array( seqman, error, start, array, avail, true );
      if (MB_SUCCESS != rval)
        return rval;
      
      const EntityHandle end = std::min<EntityHandle>(p->second + 1, start + avail );
      while (start != end) {
        memcpy( array, *pointers, get_size() );
        ++start;
        ++pointers;
        array += get_size();
      }
    }
  }
  
  return MB_SUCCESS;
}
Exemple #5
0
ErrorCode DenseTag::set_data( SequenceManager* seqman,
                              Error* error,
                              const Range& entities,
                              const void* values )
{
  ErrorCode rval;
  const char* data = reinterpret_cast<const char*>(values);
  unsigned char* array;
  size_t avail;

  for (Range::const_pair_iterator p = entities.const_pair_begin(); 
       p != entities.const_pair_end(); ++p) {
       
    EntityHandle start = p->first;
    while (start <= p->second) {
      rval = get_array( seqman, error, start, array, avail, true );
      if (MB_SUCCESS != rval)
        return rval;
      
      const size_t count = std::min<size_t>(p->second - start + 1, avail);
      memcpy( array, data, get_size() * count );
      data += get_size() * count;
      start += count;
    }
  }
  
  return MB_SUCCESS;
}
Exemple #6
0
ErrorCode DenseTag::get_data( const SequenceManager* seqman,
                              Error* error,
                              const Range& entities,
                              void* values ) const
{
  ErrorCode rval;
  size_t avail;
  const unsigned char* array = NULL; // initialize to get rid of warning
  unsigned char* data = reinterpret_cast<unsigned char*>(values);

  for (Range::const_pair_iterator p = entities.const_pair_begin(); 
       p != entities.const_pair_end(); ++p) {
       
    EntityHandle start = p->first;
    while (start <= p->second) {
      rval = get_array( seqman, error, start, array, avail );
      if (MB_SUCCESS != rval)
        return rval;
      
      const size_t count = std::min<size_t>(p->second - start + 1, avail);
      if (array) 
        memcpy( data, array, get_size() * count );
      else if (get_default_value())
        SysUtil::setmem( data, get_default_value(), get_size(), count );
      else
        return not_found( error, get_name(), start );
      
      data += get_size() * count;
      start += count;
    }
  }
  
  return MB_SUCCESS;
}
Exemple #7
0
ErrorCode DenseTag::get_data( const SequenceManager* seqman,
                              Error* error,
                              const Range& entities,
                              const void** pointers,
                              int* data_lengths ) const
{
  ErrorCode rval;
  size_t avail;
  const unsigned char* array = 0;

  if (data_lengths) {
    int len = get_size();
    SysUtil::setmem( data_lengths, &len, sizeof(int), entities.size() );
  }
  
  for (Range::const_pair_iterator p = entities.const_pair_begin(); 
       p != entities.const_pair_end(); ++p) {
       
    EntityHandle start = p->first;
    while (start <= p->second) {
      rval = get_array( seqman, error, start, array, avail );
      if (MB_SUCCESS != rval)
        return rval;
      
      const size_t count = std::min<size_t>(p->second - start + 1, avail);
      if (array) {
        for (EntityHandle end = start + count; start != end; ++start) {
          *pointers = array;
          array += get_size();
          ++pointers;
        }
      }
      else if (const void* val = get_default_value()) {
        SysUtil::setmem( pointers, &val, sizeof(void*), count );
        pointers += count;
        start += count;
      }
      else {
        return not_found( error, get_name(), start );;
      }
    }
  }
  
  return MB_SUCCESS;
}
Exemple #8
0
ErrorCode BitTag::clear_data( SequenceManager* seqman,
                              Error* error,
                              const Range& handles, 
                              const void* value_ptr,
                              int value_len )
{
  if (value_len)
    return MB_INVALID_SIZE;

  ErrorCode rval = seqman->check_valid_entities( error, handles );
  if (MB_SUCCESS != rval)
    return rval;
    
  EntityType type;
  EntityID count;
  size_t page;
  int offset, per_page = ents_per_page();
  const unsigned char value = *reinterpret_cast<const unsigned char*>(value_ptr);
  Range::const_pair_iterator i;
  for (i = handles.const_pair_begin(); i != handles.const_pair_end(); ++i) {
    unpack( i->first, type, page, offset );
    assert(TYPE_FROM_HANDLE(i->second) == type); // should be true because id of zero is never used
    count = i->second - i->first + 1;
    
    while (count) {
      if (page >= pageList[type].size())
        pageList[type].resize( page+1, 0 );
      if (!pageList[type][page])
        pageList[type][page] = new BitPage( storedBitsPerEntity, default_val() );

      size_t pcount = std::min( (EntityID)(per_page - offset), count );
      pageList[type][page]->set_bits( offset, pcount, storedBitsPerEntity, value );
      count -= pcount; 
      offset = 0;
      ++page;
    }
  }
  return MB_SUCCESS;
}
Exemple #9
0
ErrorCode BitTag::remove_data( SequenceManager*, Error*, const Range& handles )
{
  EntityType type;
  EntityID count;
  size_t page;
  int offset, per_page = ents_per_page();
  unsigned char val = default_val();
  Range::const_pair_iterator i;
  for (i = handles.const_pair_begin(); i != handles.const_pair_end(); ++i) {
    unpack( i->first, type, page, offset );
    assert(TYPE_FROM_HANDLE(i->second) == type); // should be true because id of zero is never used
    count = i->second - i->first + 1;
    
    while (count) {
      size_t pcount = std::min( (EntityID)(per_page - offset), count );
      if (page < pageList[type].size() && pageList[type][page])
        pageList[type][page]->set_bits( offset, pcount, storedBitsPerEntity, val );
      count -= pcount; 
      offset = 0;
      ++page;
    }
  }
  return MB_SUCCESS;
}
Exemple #10
0
ErrorCode BitTag::get_data( const SequenceManager*,
                            Error*,
                            const Range& handles, 
                            void* gen_data ) const
{
  EntityType type;
  EntityID count;
  size_t page;
  int offset, per_page = ents_per_page();
  unsigned char def = default_val();
  unsigned char* data = reinterpret_cast<unsigned char*>(gen_data);
  Range::const_pair_iterator i;
  for (i = handles.const_pair_begin(); i != handles.const_pair_end(); ++i) {
    unpack( i->first, type, page, offset );
    assert(TYPE_FROM_HANDLE(i->second) == type); // should be true because id of zero is never used
    count = i->second - i->first + 1;
    if (page >= pageList[type].size()) {
      memset( data, def, count );
      data += count;
      continue;
    }
    
    while (count) {
      size_t pcount = std::min( (EntityID)(per_page - offset), count );
      if (pageList[type][page])
        pageList[type][page]->get_bits( offset, pcount, storedBitsPerEntity, data );
      else
        memset( data, def, pcount );
      data += pcount;
      count -= pcount; 
      offset = 0;
      ++page;
    }
  }
  return MB_SUCCESS;
}
Exemple #11
0
/*
ErrorCode ReadHDF5VarLen::read_offsets( ReadHDF5Dataset& data_set,
                                        const Range& file_ids,
                                        EntityHandle start_file_id,
                                        unsigned num_columns,
                                        const unsigned indices[],
                                        EntityHandle nudge,
                                        Range offsets_out[],
                                        std::vector<unsigned> counts_out[],
                                        Range* ranged_file_ids = 0 )
{
  const int local_index = 1;

    // sanity check
  const unsigned max_cols = ranged_file_ids ? data_set.columns() - 1 : data_set.columns()
  for (unsigned i = 0; i < num_columns; ++i) {
    assert(indices[i] >= max_cols);
    if (indices[i] >= max_cols)    
      return MB_FAILURE;
 }
  
    // Use hints to make sure insertion into ranges is O(1)
  std::vector<Range::iterator> hints;
  if (ranged_file_ids) {
    hints.resize( num_colums + 1 );
    hints.back() = ranged_file_ids->begin();
  }
  else {
    hints.resize( num_columns );
  }
  for (unsigned i = 0; i < num_columns; ++i)
    offsets_out[i].clear();
    counts_out[i].clear();
    counts_out[i].reserve( file_ids.size() );
    hints[i] = offsets_out[i].begin();
  }

    // If we only need one column from a multi-column data set,
    // then read only that column.
  if (num_columns == 1 && data_set.columns() > 1 && !ranged_file_ids) {
    data_set.set_column( indices[0] );
    indices = &local_index;
  }
  else if (ranged_file_ids && data_set.columns() > 1 && 0 == num_columns) {
    data_set.set_column( data_set.columns() - 1 );
  }
    // NOTE: do not move this above the previous block.  
    //       The previous block changes the results of data_set.columns()!
  const size_t table_columns = data_set.columns();

    // Calculate which rows we need to read from the offsets table
  Range rows;
  Range::iterator hint = rows.begin();
  Range::const_pair_iterator pair = file_ids.const_pair_begin();
    // special case if reading first entity in dataset, because
    // there is no previous end value.
  if (pair != file_ids.const_pair_end() && pair->first == start_file_id) 
    hint = rows.insert( nudge, pair->second - start_file_id + nudge );
  while (pair != file_ids.const_pair_end()) {
    hint = rows.insert( hint,
                        pair->first + nudge - 1 - start_file_id, 
                        pair->second + nudge - start_file_id );
    ++pair;
  }
    
    // set up read of offsets dataset
  hsize_t buffer_size = bufferSize / (sizeof(hssize_t) * data_set.columns());
  hssize_t* buffer = reinterpret_cast<hssize_t*>(dataBuffer);
  data_set.set_file_ids( rows, nudge, buffer_size, H5T_NATIVE_HSSIZE );
  std::vector<hssize_t> prev_end;
    // If we're reading the first row of the table, then the 
    // previous end is implicitly -1.
  if (!file_ids.empty() && file_ids.front() == start_file_id) 
    prev_end.resize(num_columns,-1);
  
    // read offset table
  size_t count, offset;
  Range::const_iterator fiter = file_ids.begin();
  while (!data_set.done()) {
    try {
      data_set.read( buffer, count );
    }
    catch (ReadHDF5Dataset::Exception e) {
      return MB_FAILURE;
    }
    if (!count) // might have been NULL read for collective IO
      continue;
    
      // If the previous end values were read in the previous iteration,
      // then they're stored in prev_end.  
    size_t offset = 0;
    if (!prev_end.empty()) {
       for (unsigned i = 0; i < num_columns; ++i) {
        counts_out[i].push_back( buffer[indices[i]] - prev_end[i] );
        hints[i] = offsets_out[i].insert( hints[i],
                                          prev_end[i] + 1 + nudge,
                                          buffer[indices[i]] + nudge );
      }
      if (ranged_file_ids && (buffer[table_columns-1] & mhdf_SET_RANGE_BIT))
        hints.back() = ranged_file_ids->insert( hints.back(), *fiter );
      ++fiter;
      offset = 1;
      prev_end.clear();
    }

    while (offset < count) {
      assert(fiter != file_ids.end());
        // whenever we get to a gap between blocks we need to 
        // advance one step because we read an extra end id 
        // preceding teah block
      if (fiter == fiter.start_of_block()) {
        if (offset == count-1) 
          break;
        ++offset;
      }
      
      for (unsigned i = 0; i < num_columns; ++i) {
        size_t s = buffer[(offset-1)*table_columns+indices[i]] + 1;
        size_t e = buffer[ offset   *table_columns+indices[i]];
        counts_out.push_back( e - s + 1 );
        hints[i] = offsets_out.insert( hints[i], s, e );
      }
      if (ranged_file_ids && (buffer[offset*table_columns+table_columns-1] & mhdf_SET_RANGE_BIT))
        hints.back() = ranged_file_ids->insert( hints.back(), *fiter );
      
      ++fiter;
      ++offset;
    }
    
      // If we did not end on the boundary between two blocks,
      // then we need to save the end indices for the final entry
      // for use in the next iteration.  Similarly, if we ended
      // with extra values that were read with the express intention
      // of getting the previous end values for a block, we need to
      // save them.  This case only arises if we hit the break in
      // the above loop.
    if (fiter != fiter.start_of_block() || offset < count) {
      assert(prev_end.empty());
      if (offset == count) {
        --offset;
        assert(fiter != fiter.start_of_block());
      }
      else {
        assert(offset+1 == count);
        assert(fiter == fiter.start_of_block());
      }
      for (unsigned i = 0; i < num_columns; ++i) 
        prev_end.push_back(buffer[offset*table_columns+indices[i]]);
    }
  }
  assert(prev_end.empty());
  assert(fiter == file_ids.end());
  
  return MB_SUCCESS;
}
*/
ErrorCode ReadHDF5VarLen::read_offsets( ReadHDF5Dataset& data_set,
                                        const Range& file_ids,
                                        EntityHandle start_file_id,
                                        EntityHandle nudge,
                                        Range& offsets_out,
                                        std::vector<unsigned>& counts_out )
{
  
    // Use hints to make sure insertion into ranges is O(1)
  offsets_out.clear();
  counts_out.clear();
  counts_out.reserve( file_ids.size() );
  Range::iterator hint;

    // Calculate which rows we need to read from the offsets table
  Range rows;
  hint = rows.begin();
  Range::const_pair_iterator pair = file_ids.const_pair_begin();
    // special case if reading first entity in dataset, because
    // there is no previous end value.
  if (pair != file_ids.const_pair_end() && pair->first == start_file_id) {
    hint = rows.insert( nudge, pair->second - start_file_id + nudge );
    ++pair;
  }
  while (pair != file_ids.const_pair_end()) {
    hint = rows.insert( hint,
                        pair->first  - start_file_id + nudge - 1, 
                        pair->second - start_file_id + nudge );
    ++pair;
  }
    
    // set up read of offsets dataset
  hsize_t buffer_size = bufferSize / sizeof(hssize_t);
  hssize_t* buffer = reinterpret_cast<hssize_t*>(dataBuffer);
  data_set.set_file_ids( rows, nudge, buffer_size, H5T_NATIVE_HSSIZE );
  hssize_t prev_end;
  bool have_prev_end = false;
    // If we're reading the first row of the table, then the 
    // previous end is implicitly -1.
  if (!file_ids.empty() && file_ids.front() == start_file_id)  {
    prev_end = -1;
    have_prev_end = true;
  }
  
  dbgOut.printf( 3, "Reading %s in %lu chunks\n", data_set.get_debug_desc(), data_set.get_read_count() );
  
    // read offset table
  size_t count, offset;
  Range::const_iterator fiter = file_ids.begin();
  hint = offsets_out.begin();
  int nn = 0;
  while (!data_set.done()) {
    dbgOut.printf( 3, "Reading chunk %d of %s\n", ++nn, data_set.get_debug_desc() );
    try {
      data_set.read( buffer, count );
    }
    catch (ReadHDF5Dataset::Exception& ) {
      return MB_FAILURE;
    }
    if (!count) // might have been NULL read for collective IO
      continue;
    
      // If the previous end values were read in the previous iteration,
      // then they're stored in prev_end.  
    offset = 0;
    if (have_prev_end) {
      counts_out.push_back( buffer[0] - prev_end );
      hint = offsets_out.insert( hint,
                                 prev_end + 1 + nudge,
                                 buffer[0] + nudge );
      ++fiter;
      offset = 1;
      have_prev_end = false;
    }

    while (offset < count) {
      assert(fiter != file_ids.end());
        // whenever we get to a gap between blocks we need to 
        // advance one step because we read an extra end id 
        // preceding teah block
      if (fiter == fiter.start_of_block()) {
        if (offset == count-1) 
          break;
        ++offset;
      }
      
      size_t s = buffer[offset-1] + 1;
      size_t e = buffer[offset];
      counts_out.push_back( e - s + 1 );
      hint = offsets_out.insert( hint, s + nudge, e + nudge );
      
      ++fiter;
      ++offset;
    }
    
      // If we did not end on the boundary between two blocks,
      // then we need to save the end indices for the final entry
      // for use in the next iteration.  Similarly, if we ended
      // with extra values that were read with the express intention
      // of getting the previous end values for a block, we need to
      // save them.  This case only arises if we hit the break in
      // the above loop.
    if (fiter != fiter.start_of_block() || offset < count) {
      assert(!have_prev_end);
      if (offset == count) {
        --offset;
        assert(fiter != fiter.start_of_block());
      }
      else {
        assert(offset+1 == count);
        assert(fiter == fiter.start_of_block());
      }
      have_prev_end = true;
      prev_end = buffer[offset];
    }
  }
  assert(!have_prev_end);
  assert(fiter == file_ids.end());
  
  return MB_SUCCESS;
}