Exemplo n.º 1
0
 bool empty() const { return m_vertices.empty(); }
Exemplo n.º 2
0
/// This version of the constructor take one pass throught the data.  It
/// constructs a ibis::index::VMap first, then construct the sbiad from the
/// VMap.  It uses more computer memory than the two-pass version, but will
/// probably run a little faster.
void ibis::sbiad::construct1(const char* f, const uint32_t nbase) {
    VMap bmap; // a map between values and their position
    try {
	mapValues(f, bmap);
    }
    catch (...) { // need to clean up bmap
	LOGGER(ibis::gVerbose >= 0)
	    << "sbiad::construct reclaiming storage "
	    "allocated to bitvectors (" << bmap.size() << ")";

	for (VMap::iterator it = bmap.begin(); it != bmap.end(); ++ it)
	    delete (*it).second;
	bmap.clear();
	ibis::fileManager::instance().signalMemoryAvailable();
	throw;
    }
    if (bmap.empty()) return;
    nrows = (*(bmap.begin())).second->size();
    if (nrows != col->partition()->nRows()) {
	for (VMap::iterator it = bmap.begin(); it != bmap.end(); ++ it)
	    delete (*it).second;
	bmap.clear();
	ibis::fileManager::instance().signalMemoryAvailable();

	LOGGER(ibis::gVerbose >= 0)
	    << "Warning -- sbiad::construct1 the bitvectors "
	    "do not have the expected size(" << col->partition()->nRows()
	    << "). stopping..";
	throw ibis::bad_alloc("incorrect bitvector sizes");
    }

    // convert bmap into the current data structure
    // fill the arrays vals and cnts
    const uint32_t card = bmap.size();
    vals.reserve(card);
    cnts.reserve(card);
    for (VMap::const_iterator it = bmap.begin(); it != bmap.end(); ++it) {
	vals.push_back((*it).first);
	cnts.push_back((*it).second->cnt());
    }
    // fill the array bases
    setBases(bases, card, nbase);
    // count the number of bitvectors to genreate
    const uint32_t nb = bases.size();
    uint32_t nobs = 0;
    uint32_t i;
    for (i = 0; i < nb; ++i)
	nobs += bases[i];
    // allocate enough bitvectors in bits
    bits.resize(nobs);
    for (i = 0; i < nobs; ++i)
	bits[i] = 0;
    if (ibis::gVerbose > 5) {
	col->logMessage("sbiad::construct", "initialized the array of "
			"bitvectors, start converting %lu bitmaps into %lu-"
			"component range code (with %lu bitvectors)",
			static_cast<long unsigned>(vals.size()),
			static_cast<long unsigned>(nb),
			static_cast<long unsigned>(nobs));
    }

    // converting to multi-level equality encoding first
    i = 0;
    for (VMap::const_iterator it = bmap.begin(); it != bmap.end();
	 ++it, ++i) {
	uint32_t offset = 0;
	uint32_t ii = i;
	for (uint32_t j = 0; j < nb; ++j) {
	    uint32_t k = ii % bases[j];
	    if (bits[offset+k]) {
		*(bits[offset+k]) |= *((*it).second);
	    }
	    else {
		bits[offset+k] = new ibis::bitvector();
		bits[offset+k]->copy(*((*it).second));
		// expected to be operated on more than 64 times
		if (vals.size() > 64*bases[j])
		    bits[offset+k]->decompress();
	    }
	    ii /= bases[j];
	    offset += bases[j];
	}

	delete (*it).second; // no longer need the bitmap
    }
    for (i = 0; i < nobs; ++i) {
	if (bits[i] == 0) {
	    bits[i] = new ibis::bitvector();
	    bits[i]->set(0, nrows);
	}
    }
#if DEBUG+0 > 0 || _DEBUG+0 > 0
    if (ibis::gVerbose > 11) {
	LOGGER(ibis::gVerbose >= 0)
	    << "DEBUG -- sbiad::construct1 converted"
	    << bmap.size() << " bitmaps for each distinct value into "
	    << bits.size() << bases.size()
	    << "-component equality encoded bitmaps";
    }
#endif
    // sum up the bitvectors according to the interval-encoding
    array_t<bitvector*> beq;
    beq.swap(bits);
    try { // use a try block to ensure the bitvectors in beq are freed
	uint32_t ke = 0;
	bits.clear();
	for (i = 0; i < nb; ++i) {
	    if (bases[i] > 2) {
		nobs = (bases[i] - 1) / 2;
		bits.push_back(new ibis::bitvector);
		bits.back()->copy(*(beq[ke]));
		if (nobs > 64)
		    bits.back()->decompress();
		for (uint32_t j = ke+1; j <= ke+nobs; ++j)
		    *(bits.back()) |= *(beq[j]);
		bits.back()->compress();
		for (uint32_t j = 1; j < bases[i]-nobs; ++j) {
		    bits.push_back(*(bits.back()) - *(beq[ke+j-1]));
		    *(bits.back()) |= *(beq[ke+j+nobs]);
		    bits.back()->compress();
		}
		for (uint32_t j = ke; j < ke+bases[i]; ++j) {
		    delete beq[j];
		    beq[j] = 0;
		}
	    }
	    else {
		bits.push_back(beq[ke]);
		if (bases[i] > 1) {
		    delete beq[ke+1];
		    beq[ke+1] = 0;
		}
	    }
	    ke += bases[i];
	}
    }
    catch (...) {
	LOGGER(ibis::gVerbose > 1)
	    << "Warning -- column::[" << col->name()
	    << "]::construct1 encountered an exception while converting "
	    "to inverval encoding, cleaning up ...";
	for (uint32_t i = 0; i < beq.size(); ++ i)
	    delete beq[i];
	throw;
    }
    beq.clear();
#if DEBUG+0 > 0 || _DEBUG+0 > 0
    if (ibis::gVerbose > 11) {
	LOGGER(ibis::gVerbose >= 0)
	    << "DEBUG -- sbiad::construct1 completed "
	    << "converting equality encoding to interval encoding";
    }
#endif
    optionalUnpack(bits, col->indexSpec());

    // write out the current content
    if (ibis::gVerbose > 8) {
 	ibis::util::logger lg;
 	print(lg());
    }
} // ibis::sbiad::construct1