bool empty() const { return m_vertices.empty(); }
/// This version of the constructor take one pass throught the data. It
/// constructs a ibis::index::VMap first, then construct the sbiad from the
/// VMap. It uses more computer memory than the two-pass version, but will
/// probably run a little faster.
void ibis::sbiad::construct1(const char* f, const uint32_t nbase) {
VMap bmap; // a map between values and their position
try {
mapValues(f, bmap);
}
catch (...) { // need to clean up bmap
LOGGER(ibis::gVerbose >= 0)
<< "sbiad::construct reclaiming storage "
"allocated to bitvectors (" << bmap.size() << ")";
for (VMap::iterator it = bmap.begin(); it != bmap.end(); ++ it)
delete (*it).second;
bmap.clear();
ibis::fileManager::instance().signalMemoryAvailable();
throw;
}
if (bmap.empty()) return;
nrows = (*(bmap.begin())).second->size();
if (nrows != col->partition()->nRows()) {
for (VMap::iterator it = bmap.begin(); it != bmap.end(); ++ it)
delete (*it).second;
bmap.clear();
ibis::fileManager::instance().signalMemoryAvailable();
LOGGER(ibis::gVerbose >= 0)
<< "Warning -- sbiad::construct1 the bitvectors "
"do not have the expected size(" << col->partition()->nRows()
<< "). stopping..";
throw ibis::bad_alloc("incorrect bitvector sizes");
}
// convert bmap into the current data structure
// fill the arrays vals and cnts
const uint32_t card = bmap.size();
vals.reserve(card);
cnts.reserve(card);
for (VMap::const_iterator it = bmap.begin(); it != bmap.end(); ++it) {
vals.push_back((*it).first);
cnts.push_back((*it).second->cnt());
}
// fill the array bases
setBases(bases, card, nbase);
// count the number of bitvectors to genreate
const uint32_t nb = bases.size();
uint32_t nobs = 0;
uint32_t i;
for (i = 0; i < nb; ++i)
nobs += bases[i];
// allocate enough bitvectors in bits
bits.resize(nobs);
for (i = 0; i < nobs; ++i)
bits[i] = 0;
if (ibis::gVerbose > 5) {
col->logMessage("sbiad::construct", "initialized the array of "
"bitvectors, start converting %lu bitmaps into %lu-"
"component range code (with %lu bitvectors)",
static_cast<long unsigned>(vals.size()),
static_cast<long unsigned>(nb),
static_cast<long unsigned>(nobs));
}
// converting to multi-level equality encoding first
i = 0;
for (VMap::const_iterator it = bmap.begin(); it != bmap.end();
++it, ++i) {
uint32_t offset = 0;
uint32_t ii = i;
for (uint32_t j = 0; j < nb; ++j) {
uint32_t k = ii % bases[j];
if (bits[offset+k]) {
*(bits[offset+k]) |= *((*it).second);
}
else {
bits[offset+k] = new ibis::bitvector();
bits[offset+k]->copy(*((*it).second));
// expected to be operated on more than 64 times
if (vals.size() > 64*bases[j])
bits[offset+k]->decompress();
}
ii /= bases[j];
offset += bases[j];
}
delete (*it).second; // no longer need the bitmap
}
for (i = 0; i < nobs; ++i) {
if (bits[i] == 0) {
bits[i] = new ibis::bitvector();
bits[i]->set(0, nrows);
}
}
#if DEBUG+0 > 0 || _DEBUG+0 > 0
if (ibis::gVerbose > 11) {
LOGGER(ibis::gVerbose >= 0)
<< "DEBUG -- sbiad::construct1 converted"
<< bmap.size() << " bitmaps for each distinct value into "
<< bits.size() << bases.size()
<< "-component equality encoded bitmaps";
}
#endif
// sum up the bitvectors according to the interval-encoding
array_t<bitvector*> beq;
beq.swap(bits);
try { // use a try block to ensure the bitvectors in beq are freed
uint32_t ke = 0;
bits.clear();
for (i = 0; i < nb; ++i) {
if (bases[i] > 2) {
nobs = (bases[i] - 1) / 2;
bits.push_back(new ibis::bitvector);
bits.back()->copy(*(beq[ke]));
if (nobs > 64)
bits.back()->decompress();
for (uint32_t j = ke+1; j <= ke+nobs; ++j)
*(bits.back()) |= *(beq[j]);
bits.back()->compress();
for (uint32_t j = 1; j < bases[i]-nobs; ++j) {
bits.push_back(*(bits.back()) - *(beq[ke+j-1]));
*(bits.back()) |= *(beq[ke+j+nobs]);
bits.back()->compress();
}
for (uint32_t j = ke; j < ke+bases[i]; ++j) {
delete beq[j];
beq[j] = 0;
}
}
else {
bits.push_back(beq[ke]);
if (bases[i] > 1) {
delete beq[ke+1];
beq[ke+1] = 0;
}
}
ke += bases[i];
}
}
catch (...) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- column::[" << col->name()
<< "]::construct1 encountered an exception while converting "
"to inverval encoding, cleaning up ...";
for (uint32_t i = 0; i < beq.size(); ++ i)
delete beq[i];
throw;
}
beq.clear();
#if DEBUG+0 > 0 || _DEBUG+0 > 0
if (ibis::gVerbose > 11) {
LOGGER(ibis::gVerbose >= 0)
<< "DEBUG -- sbiad::construct1 completed "
<< "converting equality encoding to interval encoding";
}
#endif
optionalUnpack(bits, col->indexSpec());
// write out the current content
if (ibis::gVerbose > 8) {
ibis::util::logger lg;
print(lg());
}
} // ibis::sbiad::construct1