bool QDSDataStore::add( const QUniqueId& id, QFile& data, const QMimeType& type ) { // Create info file and copy data file (in an atomic fashion) QDSLockedFile infoFile( infoFileName( id ) ); if ( infoFile.exists() ) { qLog(DataSharing) << "QDSDataStore::add - info file already exists"; return false; } if ( !infoFile.openLocked( QIODevice::WriteOnly ) ) { qLog(DataSharing) << "QDSDataStore::add - " << "info file can't be written to"; return false; } if ( !data.copy( dataFileName( id ) ) ) { qLog(DataSharing) << "QDSDataStore::add - couldn't copy data file"; return false; } QDataStream ds(&infoFile); ds << 1; ds << 0; ds << type.id(); return true; }
int main( int argc, char *argv[] ) { // Put the file paths back to a flatter system because I do not know if // Windows will barf at the forward slashes and I do not have time to make this // code check which OS it's using. // char temp[256]; // string dataFileName(getcwd(temp, 255)); // string colorFileName(getcwd(temp, 255)); // dataFileName.append("Elevation.bin"); // colorFileName.append("rainbow.pal"); // string dataFileName("Elevation.bin"); // string colorFileName("livingston.pal"); string dataFileName("Head.bin"); string colorFileName("rainbow.pal"); cout << "Data file: " << dataFileName << endl; cout << "Color file: " << colorFileName << endl; // cout << "Setting up data reader" << endl; dr = new DataReader(dataFileName, colorFileName); // cout << "Building texture" << endl; texels = buildTexture(); // texels allocated here // cout << "Everything appears to work" << endl; setup(argc, argv); free(texels); free(dr); return 0; }
bool QDSDataStore::add( const QUniqueId& id, const QByteArray& data, const QMimeType& type ) { // Create info and data files (in an atomic fashion) QDSLockedFile infoFile( infoFileName( id ) ); if ( infoFile.exists() ) { qLog(DataSharing) << "QDSDataStore::add - info file already exists"; return false; } if ( !infoFile.openLocked( QIODevice::WriteOnly ) ) { qLog(DataSharing) << "QDSDataStore::add - " << "info file can't be written to"; return false; } QDSLockedFile dataFile( dataFileName( id ) ); if ( dataFile.exists() ) { qLog(DataSharing) << "QDSDataStore::add - data file already exists"; return false; } if ( !dataFile.openLocked( QIODevice::WriteOnly ) ) { qLog(DataSharing) << "QDSDataStore::add - " << "data file can't be written to"; return false; } // Write to files QDataStream infoDs( &infoFile ); infoDs << 1; infoDs << 0; infoDs << type.id(); dataFile.write( data ); return true; }
// This function reads the data file and records the locations of the values // in bakMap void ibis::bak::mapValues(const char* f, ibis::bak::bakMap& bmap) const { if (col == 0) return; horometer timer; if (ibis::gVerbose > 4) timer.start(); const unsigned prec = parsePrec(*col); // the precision of mapped value std::string fnm; // name of the data file dataFileName(fnm, f); if (fnm.empty()) { LOGGER(ibis::gVerbose > 0) << "Warning -- bak::mapValues failed to determine the data file " "name from \"" << (f ? f : "") << '"'; return; } uint32_t nev; ibis::bitvector mask; col->getNullMask(mask); if (col->partition() != 0) nev = col->partition()->nRows(); else nev = mask.size(); if (nev == 0) return; // need to use different types of array_t for different columns switch (col->type()) { case ibis::TEXT: case ibis::UINT: {// unsigned int array_t<uint32_t> val; if (! fnm.empty()) ibis::fileManager::instance().getFile(fnm.c_str(), val); else col->getValuesArray(&val); if (val.size() <= 0) { col->logWarning("bak::mapValues", "unable to read %s", fnm.c_str()); } else { bmap.clear(); nev = val.size(); if (nev > mask.size()) mask.adjustSize(nev, nev); ibis::bitvector::indexSet iset = mask.firstIndexSet(); uint32_t nind = iset.nIndices(); const ibis::bitvector::word_t *iix = iset.indices(); while (nind) { if (iset.isRange()) { // a range uint32_t k = (iix[1] < nev ? iix[1] : nev); for (uint32_t i = *iix; i < k; ++i) { double key = ibis::util::coarsen(val[i], prec); ibis::bak::grain& grn = bmap[key]; if (grn.loc == 0) grn.loc = new ibis::bitvector; grn.loc->setBit(i, 1); if (grn.min > val[i]) grn.min = val[i]; if (grn.max < val[i]) grn.max = val[i]; } } else if (*iix+ibis::bitvector::bitsPerLiteral() < nev) { // a list of indices for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; double key = ibis::util::coarsen(val[k], prec); ibis::bak::grain& grn = bmap[key]; if (grn.loc == 0) grn.loc = new ibis::bitvector; grn.loc->setBit(k, 1); if (grn.min > val[k]) grn.min = val[k]; if (grn.max < val[k]) grn.max = val[k]; } } else { for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; if (k < nev) { double key = ibis::util::coarsen(val[k], prec); ibis::bak::grain& grn = bmap[key]; if (grn.loc == 0) grn.loc = new ibis::bitvector; grn.loc->setBit(k, 1); if (grn.min > val[k]) grn.min = val[k]; if (grn.max < val[k]) grn.max = val[k]; } } } ++iset; nind = iset.nIndices(); if (*iix >= nev) nind = 0; } // while (nind) } break;} case ibis::INT: {// signed int array_t<int32_t> val; if (! fnm.empty()) ibis::fileManager::instance().getFile(fnm.c_str(), val); else col->getValuesArray(&val); if (val.size() <= 0) { col->logWarning("bak::mapValues", "unable to read %s", fnm.c_str()); } else { nev = val.size(); if (nev > mask.size()) mask.adjustSize(nev, nev); ibis::bitvector::indexSet iset = mask.firstIndexSet(); uint32_t nind = iset.nIndices(); const ibis::bitvector::word_t *iix = iset.indices(); while (nind) { if (iset.isRange()) { // a range uint32_t k = (iix[1] < nev ? iix[1] : nev); for (uint32_t i = *iix; i < k; ++i) { double key = ibis::util::coarsen(val[i], prec); ibis::bak::grain& grn = bmap[key]; if (grn.loc == 0) grn.loc = new ibis::bitvector; grn.loc->setBit(i, 1); if (grn.min > val[i]) grn.min = val[i]; if (grn.max < val[i]) grn.max = val[i]; } } else if (*iix+ibis::bitvector::bitsPerLiteral() < nev) { // a list of indices for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; double key = ibis::util::coarsen(val[k], prec); ibis::bak::grain& grn = bmap[key]; if (grn.loc == 0) grn.loc = new ibis::bitvector; grn.loc->setBit(k, 1); if (grn.min > val[k]) grn.min = val[k]; if (grn.max < val[k]) grn.max = val[k]; } } else { for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; if (k < nev) { double key = ibis::util::coarsen(val[k], prec); ibis::bak::grain& grn = bmap[key]; if (grn.loc == 0) grn.loc = new ibis::bitvector; grn.loc->setBit(k, 1); if (grn.min > val[k]) grn.min = val[k]; if (grn.max < val[k]) grn.max = val[k]; } } } ++iset; nind = iset.nIndices(); if (*iix >= nev) nind = 0; } // while (nind) } break;} case ibis::FLOAT: {// (4-byte) floating-point values array_t<float> val; if (! fnm.empty()) ibis::fileManager::instance().getFile(fnm.c_str(), val); else col->getValuesArray(&val); if (val.size() <= 0) { col->logWarning("bak::mapValues", "unable to read %s", fnm.c_str()); } else { nev = val.size(); if (nev > mask.size()) mask.adjustSize(nev, nev); ibis::bitvector::indexSet iset = mask.firstIndexSet(); uint32_t nind = iset.nIndices(); const ibis::bitvector::word_t *iix = iset.indices(); while (nind) { if (iset.isRange()) { // a range uint32_t k = (iix[1] < nev ? iix[1] : nev); for (uint32_t i = *iix; i < k; ++i) { double key = ibis::util::coarsen(val[i], prec); ibis::bak::grain& grn = bmap[key]; if (grn.loc == 0) grn.loc = new ibis::bitvector; grn.loc->setBit(i, 1); if (grn.min > val[i]) grn.min = val[i]; if (grn.max < val[i]) grn.max = val[i]; } } else if (*iix+ibis::bitvector::bitsPerLiteral() < nev) { // a list of indices for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; double key = ibis::util::coarsen(val[k], prec); ibis::bak::grain& grn = bmap[key]; if (grn.loc == 0) grn.loc = new ibis::bitvector; grn.loc->setBit(k, 1); if (grn.min > val[k]) grn.min = val[k]; if (grn.max < val[k]) grn.max = val[k]; } } else { for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; if (k < nev) { double key = ibis::util::coarsen(val[k], prec); ibis::bak::grain& grn = bmap[key]; if (grn.loc == 0) grn.loc = new ibis::bitvector; grn.loc->setBit(k, 1); if (grn.min > val[k]) grn.min = val[k]; if (grn.max < val[k]) grn.max = val[k]; } } } ++iset; nind = iset.nIndices(); if (*iix >= nev) nind = 0; } // while (nind) } break;} case ibis::DOUBLE: {// (8-byte) floating-point values array_t<double> val; if (! fnm.empty()) ibis::fileManager::instance().getFile(fnm.c_str(), val); else col->getValuesArray(&val); if (val.size() <= 0) { col->logWarning("bak::mapValues", "unable to read %s", fnm.c_str()); } else { nev = val.size(); if (nev > mask.size()) mask.adjustSize(nev, nev); ibis::bitvector::indexSet iset = mask.firstIndexSet(); uint32_t nind = iset.nIndices(); const ibis::bitvector::word_t *iix = iset.indices(); while (nind) { if (iset.isRange()) { // a range uint32_t k = (iix[1] < nev ? iix[1] : nev); for (uint32_t i = *iix; i < k; ++i) { double key = ibis::util::coarsen(val[i], prec); ibis::bak::grain& grn = bmap[key]; if (grn.loc == 0) grn.loc = new ibis::bitvector; grn.loc->setBit(i, 1); if (grn.min > val[i]) grn.min = val[i]; if (grn.max < val[i]) grn.max = val[i]; } } else if (*iix+ibis::bitvector::bitsPerLiteral() < nev) { // a list of indices for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; double key = ibis::util::coarsen(val[k], prec); ibis::bak::grain& grn = bmap[key]; if (grn.loc == 0) grn.loc = new ibis::bitvector; grn.loc->setBit(k, 1); if (grn.min > val[k]) grn.min = val[k]; if (grn.max < val[k]) grn.max = val[k]; } } else { for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; if (k < nev) { double key = ibis::util::coarsen(val[k], prec); ibis::bak::grain& grn = bmap[key]; if (grn.loc == 0) grn.loc = new ibis::bitvector; grn.loc->setBit(k, 1); if (grn.min > val[k]) grn.min = val[k]; if (grn.max < val[k]) grn.max = val[k]; } } } ++iset; nind = iset.nIndices(); if (*iix >= nev) nind = 0; } // while (nind) } break;} case ibis::CATEGORY: // no need for a separate index col->logWarning("bak::mapValues", "no need for binning -- should have " "a basic bitmap index already"); return; default: col->logWarning("bak::mapValues", "unable to create bins for " "this type of column"); return; } // make sure all bit vectors are the same size for (ibis::bak::bakMap::iterator it = bmap.begin(); it != bmap.end(); ++ it) { (*it).second.loc->adjustSize(0, nev); // (*it).second.loc->compress(); } // write out the current content if (ibis::gVerbose > 2) { if (ibis::gVerbose > 4) { timer.stop(); col->logMessage("bak::mapValues", "mapped %lu values to %lu " "%u-digit number%s in %g sec(elapsed)", static_cast<long unsigned>(nev), static_cast<long unsigned>(bmap.size()), prec, (bmap.size() > 1 ? "s" : ""), timer.realTime()); } else { col->logMessage("bak::mapValues", "mapped %lu values to %lu " "%lu-digit number%s", static_cast<long unsigned>(nev), static_cast<long unsigned>(bmap.size()), prec, (bmap.size() > 1 ? "s" : "")); } if (ibis::gVerbose > 6) { ibis::util::logger lg; printMap(lg(), bmap); } } } // ibis::bak::mapValues
/// Generate a new sbiad index by passing through the data twice. /// - (1) scan the data to generate a list of distinct values and their count. /// - (2) scan the data a second time to produce the bit vectors. void ibis::sbiad::construct2(const char* f, const uint32_t nbase) { { // a block to limit the scope of hst histogram hst; mapValues(f, hst); // scan the data to produce the histogram if (hst.empty()) // no data, of course no index return; // convert histogram into two arrays const uint32_t nhst = hst.size(); vals.resize(nhst); cnts.resize(nhst); histogram::const_iterator it = hst.begin(); for (uint32_t i = 0; i < nhst; ++i) { vals[i] = (*it).first; cnts[i] = (*it).second; ++ it; } } // determine the base sizes setBases(bases, vals.size(), nbase); const uint32_t nb = bases.size(); int ierr; // allocate the correct number of bitvectors uint32_t nobs = 0; for (uint32_t ii = 0; ii < nb; ++ii) nobs += bases[ii]; bits.resize(nobs); for (uint32_t ii = 0; ii < nobs; ++ii) bits[ii] = new ibis::bitvector; std::string fnm; // name of the data file dataFileName(fnm, f); nrows = col->partition()->nRows(); ibis::bitvector mask; { // name of mask file associated with the data file array_t<ibis::bitvector::word_t> arr; std::string mname(fnm); mname += ".msk"; if (ibis::fileManager::instance().getFile(mname.c_str(), arr) == 0) mask.copy(ibis::bitvector(arr)); // convert arr to a bitvector else mask.set(1, nrows); // default mask } // need to do different things for different columns switch (col->type()) { case ibis::TEXT: case ibis::UINT: {// unsigned int array_t<uint32_t> val; if (! fnm.empty()) ierr = ibis::fileManager::instance().getFile(fnm.c_str(), val); else ierr = col->getValuesArray(&val); if (ierr < 0 || val.empty()) { LOGGER(ibis::gVerbose > 0) << "Warning -- sbiad::construct2 failed to retrieve any value"; break; } if (val.size() > mask.size()) { col->logWarning("sbiad::construct", "the data file \"%s\" " "contains more elements (%lu) then expected " "(%lu)", fnm.c_str(), static_cast<long unsigned>(val.size()), static_cast<long unsigned>(mask.size())); mask.adjustSize(nrows, nrows); } ibis::bitvector::indexSet iset = mask.firstIndexSet(); uint32_t nind = iset.nIndices(); const ibis::bitvector::word_t *iix = iset.indices(); while (nind) { if (iset.isRange()) { // a range uint32_t k = (iix[1] < nrows ? iix[1] : nrows); for (uint32_t i = *iix; i < k; ++i) setBit(i, val[i]); } else if (*iix+ibis::bitvector::bitsPerLiteral() < nrows) { // a list of indices for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; setBit(k, val[k]); } } else { for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; if (k < nrows) setBit(k, val[k]); } } ++iset; nind = iset.nIndices(); if (*iix >= nrows) nind = 0; } // while (nind) break;} case ibis::INT: {// signed int array_t<int32_t> val; if (! fnm.empty()) ierr = ibis::fileManager::instance().getFile(fnm.c_str(), val); else ierr = col->getValuesArray(&val); if (ierr < 0 || val.empty()) { LOGGER(ibis::gVerbose > 0) << "Warning -- sbiad::construct2 failed to retrieve any value"; break; } if (val.size() > mask.size()) { col->logWarning("sbiad::construct", "the data file \"%s\" " "contains more elements (%lu) then expected " "(%lu)", fnm.c_str(), static_cast<long unsigned>(val.size()), static_cast<long unsigned>(mask.size())); mask.adjustSize(nrows, nrows); } ibis::bitvector::indexSet iset = mask.firstIndexSet(); uint32_t nind = iset.nIndices(); const ibis::bitvector::word_t *iix = iset.indices(); while (nind) { if (iset.isRange()) { // a range uint32_t k = (iix[1] < nrows ? iix[1] : nrows); for (uint32_t i = *iix; i < k; ++i) setBit(i, val[i]); } else if (*iix+ibis::bitvector::bitsPerLiteral() < nrows) { // a list of indices for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; setBit(k, val[k]); } } else { for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; if (k < nrows) setBit(k, val[k]); } } ++iset; nind = iset.nIndices(); if (*iix >= nrows) nind = 0; } // while (nind) break;} case ibis::ULONG: {// unsigned long int array_t<uint64_t> val; if (! fnm.empty()) ierr = ibis::fileManager::instance().getFile(fnm.c_str(), val); else ierr = col->getValuesArray(&val); if (ierr < 0 || val.empty()) { LOGGER(ibis::gVerbose > 0) << "Warning -- sbiad::construct2 failed to retrieve any value"; break; } if (val.size() > mask.size()) { col->logWarning("sbiad::construct", "the data file \"%s\" " "contains more elements (%lu) then expected " "(%lu)", fnm.c_str(), static_cast<long unsigned>(val.size()), static_cast<long unsigned>(mask.size())); mask.adjustSize(nrows, nrows); } ibis::bitvector::indexSet iset = mask.firstIndexSet(); uint32_t nind = iset.nIndices(); const ibis::bitvector::word_t *iix = iset.indices(); while (nind) { if (iset.isRange()) { // a range uint32_t k = (iix[1] < nrows ? iix[1] : nrows); for (uint32_t i = *iix; i < k; ++i) setBit(i, val[i]); } else if (*iix+ibis::bitvector::bitsPerLiteral() < nrows) { // a list of indices for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; setBit(k, val[k]); } } else { for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; if (k < nrows) setBit(k, val[k]); } } ++iset; nind = iset.nIndices(); if (*iix >= nrows) nind = 0; } // while (nind) break;} case ibis::LONG: {// signed long int array_t<int64_t> val; if (! fnm.empty()) ierr = ibis::fileManager::instance().getFile(fnm.c_str(), val); else ierr = col->getValuesArray(&val); if (ierr < 0 || val.empty()) { LOGGER(ibis::gVerbose > 0) << "Warning -- sbiad::construct2 failed to retrieve any value"; break; } if (val.size() > mask.size()) { col->logWarning("sbiad::construct", "the data file \"%s\" " "contains more elements (%lu) then expected " "(%lu)", fnm.c_str(), static_cast<long unsigned>(val.size()), static_cast<long unsigned>(mask.size())); mask.adjustSize(nrows, nrows); } ibis::bitvector::indexSet iset = mask.firstIndexSet(); uint32_t nind = iset.nIndices(); const ibis::bitvector::word_t *iix = iset.indices(); while (nind) { if (iset.isRange()) { // a range uint32_t k = (iix[1] < nrows ? iix[1] : nrows); for (uint32_t i = *iix; i < k; ++i) setBit(i, val[i]); } else if (*iix+ibis::bitvector::bitsPerLiteral() < nrows) { // a list of indices for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; setBit(k, val[k]); } } else { for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; if (k < nrows) setBit(k, val[k]); } } ++iset; nind = iset.nIndices(); if (*iix >= nrows) nind = 0; } // while (nind) break;} case ibis::USHORT: {// unsigned short int array_t<uint16_t> val; if (! fnm.empty()) ierr = ibis::fileManager::instance().getFile(fnm.c_str(), val); else ierr = col->getValuesArray(&val); if (ierr < 0 || val.empty()) { LOGGER(ibis::gVerbose > 0) << "Warning -- sbiad::construct2 failed to retrieve any value"; break; } if (val.size() > mask.size()) { col->logWarning("sbiad::construct", "the data file \"%s\" " "contains more elements (%lu) then expected " "(%lu)", fnm.c_str(), static_cast<long unsigned>(val.size()), static_cast<long unsigned>(mask.size())); mask.adjustSize(nrows, nrows); } ibis::bitvector::indexSet iset = mask.firstIndexSet(); uint32_t nind = iset.nIndices(); const ibis::bitvector::word_t *iix = iset.indices(); while (nind) { if (iset.isRange()) { // a range uint32_t k = (iix[1] < nrows ? iix[1] : nrows); for (uint32_t i = *iix; i < k; ++i) setBit(i, val[i]); } else if (*iix+ibis::bitvector::bitsPerLiteral() < nrows) { // a list of indices for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; setBit(k, val[k]); } } else { for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; if (k < nrows) setBit(k, val[k]); } } ++iset; nind = iset.nIndices(); if (*iix >= nrows) nind = 0; } // while (nind) break;} case ibis::SHORT: {// signed short int array_t<int16_t> val; if (! fnm.empty()) ierr = ibis::fileManager::instance().getFile(fnm.c_str(), val); else ierr = col->getValuesArray(&val); if (ierr < 0 || val.empty()) { LOGGER(ibis::gVerbose > 0) << "Warning -- sbiad::construct2 failed to retrieve any value"; break; } if (val.size() > mask.size()) { col->logWarning("sbiad::construct", "the data file \"%s\" " "contains more elements (%lu) then expected " "(%lu)", fnm.c_str(), static_cast<long unsigned>(val.size()), static_cast<long unsigned>(mask.size())); mask.adjustSize(nrows, nrows); } ibis::bitvector::indexSet iset = mask.firstIndexSet(); uint32_t nind = iset.nIndices(); const ibis::bitvector::word_t *iix = iset.indices(); while (nind) { if (iset.isRange()) { // a range uint32_t k = (iix[1] < nrows ? iix[1] : nrows); for (uint32_t i = *iix; i < k; ++i) setBit(i, val[i]); } else if (*iix+ibis::bitvector::bitsPerLiteral() < nrows) { // a list of indices for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; setBit(k, val[k]); } } else { for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; if (k < nrows) setBit(k, val[k]); } } ++iset; nind = iset.nIndices(); if (*iix >= nrows) nind = 0; } // while (nind) break;} case ibis::UBYTE: {// unsigned char array_t<unsigned char> val; if (! fnm.empty()) ierr = ibis::fileManager::instance().getFile(fnm.c_str(), val); else ierr = col->getValuesArray(&val); if (ierr < 0 || val.empty()) { LOGGER(ibis::gVerbose > 0) << "Warning -- sbiad::construct2 failed to retrieve any value"; break; } if (val.size() > mask.size()) { col->logWarning("sbiad::construct", "the data file \"%s\" " "contains more elements (%lu) then expected " "(%lu)", fnm.c_str(), static_cast<long unsigned>(val.size()), static_cast<long unsigned>(mask.size())); mask.adjustSize(nrows, nrows); } ibis::bitvector::indexSet iset = mask.firstIndexSet(); uint32_t nind = iset.nIndices(); const ibis::bitvector::word_t *iix = iset.indices(); while (nind) { if (iset.isRange()) { // a range uint32_t k = (iix[1] < nrows ? iix[1] : nrows); for (uint32_t i = *iix; i < k; ++i) setBit(i, val[i]); } else if (*iix+ibis::bitvector::bitsPerLiteral() < nrows) { // a list of indices for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; setBit(k, val[k]); } } else { for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; if (k < nrows) setBit(k, val[k]); } } ++iset; nind = iset.nIndices(); if (*iix >= nrows) nind = 0; } // while (nind) break;} case ibis::BYTE: {// signed char array_t<signed char> val; if (! fnm.empty()) ierr = ibis::fileManager::instance().getFile(fnm.c_str(), val); else ierr = col->getValuesArray(&val); if (ierr < 0 || val.empty()) { LOGGER(ibis::gVerbose > 0) << "Warning -- sbiad::construct2 failed to retrieve any value"; break; } if (val.size() > mask.size()) { col->logWarning("sbiad::construct", "the data file \"%s\" " "contains more elements (%lu) then expected " "(%lu)", fnm.c_str(), static_cast<long unsigned>(val.size()), static_cast<long unsigned>(mask.size())); mask.adjustSize(nrows, nrows); } ibis::bitvector::indexSet iset = mask.firstIndexSet(); uint32_t nind = iset.nIndices(); const ibis::bitvector::word_t *iix = iset.indices(); while (nind) { if (iset.isRange()) { // a range uint32_t k = (iix[1] < nrows ? iix[1] : nrows); for (uint32_t i = *iix; i < k; ++i) setBit(i, val[i]); } else if (*iix+ibis::bitvector::bitsPerLiteral() < nrows) { // a list of indices for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; setBit(k, val[k]); } } else { for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; if (k < nrows) setBit(k, val[k]); } } ++iset; nind = iset.nIndices(); if (*iix >= nrows) nind = 0; } // while (nind) break;} case ibis::FLOAT: {// (4-byte) floating-point values array_t<float> val; if (! fnm.empty()) ierr = ibis::fileManager::instance().getFile(fnm.c_str(), val); else ierr = col->getValuesArray(&val); if (ierr < 0 || val.empty()) { LOGGER(ibis::gVerbose > 0) << "Warning -- sbiad::construct2 failed to retrieve any value"; break; } if (val.size() > mask.size()) { col->logWarning("sbiad::construct", "the data file \"%s\" " "contains more elements (%lu) then expected " "(%lu)", fnm.c_str(), static_cast<long unsigned>(val.size()), static_cast<long unsigned>(mask.size())); mask.adjustSize(nrows, nrows); } ibis::bitvector::indexSet iset = mask.firstIndexSet(); uint32_t nind = iset.nIndices(); const ibis::bitvector::word_t *iix = iset.indices(); while (nind) { if (iset.isRange()) { // a range uint32_t k = (iix[1] < nrows ? iix[1] : nrows); for (uint32_t i = *iix; i < k; ++i) setBit(i, val[i]); } else if (*iix+ibis::bitvector::bitsPerLiteral() < nrows) { // a list of indices for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; setBit(k, val[k]); } } else { for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; if (k < nrows) setBit(k, val[k]); } } ++iset; nind = iset.nIndices(); if (*iix >= nrows) nind = 0; } // while (nind) break;} case ibis::DOUBLE: {// (8-byte) floating-point values array_t<double> val; if (! fnm.empty()) ierr = ibis::fileManager::instance().getFile(fnm.c_str(), val); else ierr = col->getValuesArray(&val); if (ierr < 0 || val.empty()) { LOGGER(ibis::gVerbose > 0) << "Warning -- sbiad::construct2 failed to retrieve any value"; break; } if (val.size() > mask.size()) { col->logWarning("sbiad::construct", "the data file \"%s\" " "contains more elements (%lu) then expected " "(%lu)", fnm.c_str(), static_cast<long unsigned>(val.size()), static_cast<long unsigned>(mask.size())); mask.adjustSize(nrows, nrows); } ibis::bitvector::indexSet iset = mask.firstIndexSet(); uint32_t nind = iset.nIndices(); const ibis::bitvector::word_t *iix = iset.indices(); while (nind) { if (iset.isRange()) { // a range uint32_t k = (iix[1] < nrows ? iix[1] : nrows); for (uint32_t i = *iix; i < k; ++i) setBit(i, val[i]); } else if (*iix+ibis::bitvector::bitsPerLiteral() < nrows) { // a list of indices for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; setBit(k, val[k]); } } else { for (uint32_t i = 0; i < nind; ++i) { uint32_t k = iix[i]; if (k < nrows) setBit(k, val[k]); } } ++iset; nind = iset.nIndices(); if (*iix >= nrows) nind = 0; } // while (nind) break;} case ibis::CATEGORY: // no need for a separate index col->logWarning("sbiad::ctor", "no need for another index"); return; default: col->logWarning("sbiad::ctor", "unable to create bit sbiad index " "for column type %s", ibis::TYPESTRING[(int)col->type()]); return; } // make sure all bit vectors are the same size for (uint32_t i = 0; i < nobs; ++i) { bits[i]->adjustSize(0, nrows); } // sum up the bitvectors according to interval-encoding array_t<bitvector*> beq; beq.swap(bits); try { uint32_t ke = 0; bits.clear(); for (uint32_t i = 0; i < nb; ++i) { if (bases[i] > 2) { nobs = (bases[i] - 1) / 2; bits.push_back(new ibis::bitvector); bits.back()->copy(*(beq[ke])); if (nobs > 64) bits.back()->decompress(); for (uint32_t j = ke+1; j <= ke+nobs; ++j) *(bits.back()) |= *(beq[j]); bits.back()->compress(); for (uint32_t j = 1; j < bases[i]-nobs; ++j) { bits.push_back(*(bits.back()) - *(beq[ke+j-1])); *(bits.back()) |= *(beq[ke+j+nobs]); bits.back()->compress(); } for (uint32_t j = ke; j < ke+bases[i]; ++j) { delete beq[j]; beq[j] = 0; } } else { bits.push_back(beq[ke]); if (bases[i] > 1) { delete beq[ke+1]; beq[ke+1] = 0; } } ke += bases[i]; } } catch (...) { LOGGER(ibis::gVerbose > 1) << "Warning -- column::[" << col->name() << "]::construct2 encountered an exception while converting " "to inverval encoding, cleaning up ..."; for (uint32_t i = 0; i < beq.size(); ++ i) delete beq[i]; throw; } beq.clear(); optionalUnpack(bits, col->indexSpec()); // write out the current content if (ibis::gVerbose > 8) { ibis::util::logger lg; print(lg()); } } // ibis::sbiad::construct2