bool QDSDataStore::add( const QUniqueId& id,
QFile& data,
const QMimeType& type )
{
// Create info file and copy data file (in an atomic fashion)
QDSLockedFile infoFile( infoFileName( id ) );
if ( infoFile.exists() ) {
qLog(DataSharing) << "QDSDataStore::add - info file already exists";
return false;
}
if ( !infoFile.openLocked( QIODevice::WriteOnly ) ) {
qLog(DataSharing) << "QDSDataStore::add - "
<< "info file can't be written to";
return false;
}
if ( !data.copy( dataFileName( id ) ) ) {
qLog(DataSharing) << "QDSDataStore::add - couldn't copy data file";
return false;
}
QDataStream ds(&infoFile);
ds << 1;
ds << 0;
ds << type.id();
return true;
}
int main( int argc, char *argv[] )
{
// Put the file paths back to a flatter system because I do not know if
// Windows will barf at the forward slashes and I do not have time to make this
// code check which OS it's using.
// char temp[256];
// string dataFileName(getcwd(temp, 255));
// string colorFileName(getcwd(temp, 255));
// dataFileName.append("Elevation.bin");
// colorFileName.append("rainbow.pal");
// string dataFileName("Elevation.bin");
// string colorFileName("livingston.pal");
string dataFileName("Head.bin");
string colorFileName("rainbow.pal");
cout << "Data file: " << dataFileName << endl;
cout << "Color file: " << colorFileName << endl;
// cout << "Setting up data reader" << endl;
dr = new DataReader(dataFileName, colorFileName);
// cout << "Building texture" << endl;
texels = buildTexture(); // texels allocated here
// cout << "Everything appears to work" << endl;
setup(argc, argv);
free(texels);
free(dr);
return 0;
}
bool QDSDataStore::add( const QUniqueId& id,
const QByteArray& data,
const QMimeType& type )
{
// Create info and data files (in an atomic fashion)
QDSLockedFile infoFile( infoFileName( id ) );
if ( infoFile.exists() ) {
qLog(DataSharing) << "QDSDataStore::add - info file already exists";
return false;
}
if ( !infoFile.openLocked( QIODevice::WriteOnly ) ) {
qLog(DataSharing) << "QDSDataStore::add - "
<< "info file can't be written to";
return false;
}
QDSLockedFile dataFile( dataFileName( id ) );
if ( dataFile.exists() ) {
qLog(DataSharing) << "QDSDataStore::add - data file already exists";
return false;
}
if ( !dataFile.openLocked( QIODevice::WriteOnly ) ) {
qLog(DataSharing) << "QDSDataStore::add - "
<< "data file can't be written to";
return false;
}
// Write to files
QDataStream infoDs( &infoFile );
infoDs << 1;
infoDs << 0;
infoDs << type.id();
dataFile.write( data );
return true;
}
// This function reads the data file and records the locations of the values
// in bakMap
void ibis::bak::mapValues(const char* f, ibis::bak::bakMap& bmap) const {
if (col == 0) return;
horometer timer;
if (ibis::gVerbose > 4)
timer.start();
const unsigned prec = parsePrec(*col); // the precision of mapped value
std::string fnm; // name of the data file
dataFileName(fnm, f);
if (fnm.empty()) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- bak::mapValues failed to determine the data file "
"name from \"" << (f ? f : "") << '"';
return;
}
uint32_t nev;
ibis::bitvector mask;
col->getNullMask(mask);
if (col->partition() != 0)
nev = col->partition()->nRows();
else
nev = mask.size();
if (nev == 0) return;
// need to use different types of array_t for different columns
switch (col->type()) {
case ibis::TEXT:
case ibis::UINT: {// unsigned int
array_t<uint32_t> val;
if (! fnm.empty())
ibis::fileManager::instance().getFile(fnm.c_str(), val);
else
col->getValuesArray(&val);
if (val.size() <= 0) {
col->logWarning("bak::mapValues", "unable to read %s",
fnm.c_str());
}
else {
bmap.clear();
nev = val.size();
if (nev > mask.size())
mask.adjustSize(nev, nev);
ibis::bitvector::indexSet iset = mask.firstIndexSet();
uint32_t nind = iset.nIndices();
const ibis::bitvector::word_t *iix = iset.indices();
while (nind) {
if (iset.isRange()) { // a range
uint32_t k = (iix[1] < nev ? iix[1] : nev);
for (uint32_t i = *iix; i < k; ++i) {
double key = ibis::util::coarsen(val[i], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(i, 1);
if (grn.min > val[i]) grn.min = val[i];
if (grn.max < val[i]) grn.max = val[i];
}
}
else if (*iix+ibis::bitvector::bitsPerLiteral() < nev) {
// a list of indices
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
double key = ibis::util::coarsen(val[k], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0) grn.loc = new ibis::bitvector;
grn.loc->setBit(k, 1);
if (grn.min > val[k]) grn.min = val[k];
if (grn.max < val[k]) grn.max = val[k];
}
}
else {
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
if (k < nev) {
double key = ibis::util::coarsen(val[k], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(k, 1);
if (grn.min > val[k]) grn.min = val[k];
if (grn.max < val[k]) grn.max = val[k];
}
}
}
++iset;
nind = iset.nIndices();
if (*iix >= nev) nind = 0;
} // while (nind)
}
break;}
case ibis::INT: {// signed int
array_t<int32_t> val;
if (! fnm.empty())
ibis::fileManager::instance().getFile(fnm.c_str(), val);
else
col->getValuesArray(&val);
if (val.size() <= 0) {
col->logWarning("bak::mapValues", "unable to read %s",
fnm.c_str());
}
else {
nev = val.size();
if (nev > mask.size())
mask.adjustSize(nev, nev);
ibis::bitvector::indexSet iset = mask.firstIndexSet();
uint32_t nind = iset.nIndices();
const ibis::bitvector::word_t *iix = iset.indices();
while (nind) {
if (iset.isRange()) { // a range
uint32_t k = (iix[1] < nev ? iix[1] : nev);
for (uint32_t i = *iix; i < k; ++i) {
double key = ibis::util::coarsen(val[i], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(i, 1);
if (grn.min > val[i]) grn.min = val[i];
if (grn.max < val[i]) grn.max = val[i];
}
}
else if (*iix+ibis::bitvector::bitsPerLiteral() < nev) {
// a list of indices
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
double key = ibis::util::coarsen(val[k], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(k, 1);
if (grn.min > val[k]) grn.min = val[k];
if (grn.max < val[k]) grn.max = val[k];
}
}
else {
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
if (k < nev) {
double key = ibis::util::coarsen(val[k], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(k, 1);
if (grn.min > val[k]) grn.min = val[k];
if (grn.max < val[k]) grn.max = val[k];
}
}
}
++iset;
nind = iset.nIndices();
if (*iix >= nev) nind = 0;
} // while (nind)
}
break;}
case ibis::FLOAT: {// (4-byte) floating-point values
array_t<float> val;
if (! fnm.empty())
ibis::fileManager::instance().getFile(fnm.c_str(), val);
else
col->getValuesArray(&val);
if (val.size() <= 0) {
col->logWarning("bak::mapValues", "unable to read %s",
fnm.c_str());
}
else {
nev = val.size();
if (nev > mask.size())
mask.adjustSize(nev, nev);
ibis::bitvector::indexSet iset = mask.firstIndexSet();
uint32_t nind = iset.nIndices();
const ibis::bitvector::word_t *iix = iset.indices();
while (nind) {
if (iset.isRange()) { // a range
uint32_t k = (iix[1] < nev ? iix[1] : nev);
for (uint32_t i = *iix; i < k; ++i) {
double key = ibis::util::coarsen(val[i], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(i, 1);
if (grn.min > val[i]) grn.min = val[i];
if (grn.max < val[i]) grn.max = val[i];
}
}
else if (*iix+ibis::bitvector::bitsPerLiteral() < nev) {
// a list of indices
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
double key = ibis::util::coarsen(val[k], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(k, 1);
if (grn.min > val[k]) grn.min = val[k];
if (grn.max < val[k]) grn.max = val[k];
}
}
else {
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
if (k < nev) {
double key = ibis::util::coarsen(val[k], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(k, 1);
if (grn.min > val[k]) grn.min = val[k];
if (grn.max < val[k]) grn.max = val[k];
}
}
}
++iset;
nind = iset.nIndices();
if (*iix >= nev) nind = 0;
} // while (nind)
}
break;}
case ibis::DOUBLE: {// (8-byte) floating-point values
array_t<double> val;
if (! fnm.empty())
ibis::fileManager::instance().getFile(fnm.c_str(), val);
else
col->getValuesArray(&val);
if (val.size() <= 0) {
col->logWarning("bak::mapValues", "unable to read %s",
fnm.c_str());
}
else {
nev = val.size();
if (nev > mask.size())
mask.adjustSize(nev, nev);
ibis::bitvector::indexSet iset = mask.firstIndexSet();
uint32_t nind = iset.nIndices();
const ibis::bitvector::word_t *iix = iset.indices();
while (nind) {
if (iset.isRange()) { // a range
uint32_t k = (iix[1] < nev ? iix[1] : nev);
for (uint32_t i = *iix; i < k; ++i) {
double key = ibis::util::coarsen(val[i], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(i, 1);
if (grn.min > val[i]) grn.min = val[i];
if (grn.max < val[i]) grn.max = val[i];
}
}
else if (*iix+ibis::bitvector::bitsPerLiteral() < nev) {
// a list of indices
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
double key = ibis::util::coarsen(val[k], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(k, 1);
if (grn.min > val[k]) grn.min = val[k];
if (grn.max < val[k]) grn.max = val[k];
}
}
else {
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
if (k < nev) {
double key = ibis::util::coarsen(val[k], prec);
ibis::bak::grain& grn = bmap[key];
if (grn.loc == 0)
grn.loc = new ibis::bitvector;
grn.loc->setBit(k, 1);
if (grn.min > val[k]) grn.min = val[k];
if (grn.max < val[k]) grn.max = val[k];
}
}
}
++iset;
nind = iset.nIndices();
if (*iix >= nev) nind = 0;
} // while (nind)
}
break;}
case ibis::CATEGORY: // no need for a separate index
col->logWarning("bak::mapValues", "no need for binning -- should have "
"a basic bitmap index already");
return;
default:
col->logWarning("bak::mapValues", "unable to create bins for "
"this type of column");
return;
}
// make sure all bit vectors are the same size
for (ibis::bak::bakMap::iterator it = bmap.begin();
it != bmap.end(); ++ it) {
(*it).second.loc->adjustSize(0, nev);
// (*it).second.loc->compress();
}
// write out the current content
if (ibis::gVerbose > 2) {
if (ibis::gVerbose > 4) {
timer.stop();
col->logMessage("bak::mapValues", "mapped %lu values to %lu "
"%u-digit number%s in %g sec(elapsed)",
static_cast<long unsigned>(nev),
static_cast<long unsigned>(bmap.size()),
prec, (bmap.size() > 1 ? "s" : ""),
timer.realTime());
}
else {
col->logMessage("bak::mapValues", "mapped %lu values to %lu "
"%lu-digit number%s",
static_cast<long unsigned>(nev),
static_cast<long unsigned>(bmap.size()), prec,
(bmap.size() > 1 ? "s" : ""));
}
if (ibis::gVerbose > 6) {
ibis::util::logger lg;
printMap(lg(), bmap);
}
}
} // ibis::bak::mapValues
/// Generate a new sbiad index by passing through the data twice.
/// - (1) scan the data to generate a list of distinct values and their count.
/// - (2) scan the data a second time to produce the bit vectors.
void ibis::sbiad::construct2(const char* f, const uint32_t nbase) {
{ // a block to limit the scope of hst
histogram hst;
mapValues(f, hst); // scan the data to produce the histogram
if (hst.empty()) // no data, of course no index
return;
// convert histogram into two arrays
const uint32_t nhst = hst.size();
vals.resize(nhst);
cnts.resize(nhst);
histogram::const_iterator it = hst.begin();
for (uint32_t i = 0; i < nhst; ++i) {
vals[i] = (*it).first;
cnts[i] = (*it).second;
++ it;
}
}
// determine the base sizes
setBases(bases, vals.size(), nbase);
const uint32_t nb = bases.size();
int ierr;
// allocate the correct number of bitvectors
uint32_t nobs = 0;
for (uint32_t ii = 0; ii < nb; ++ii)
nobs += bases[ii];
bits.resize(nobs);
for (uint32_t ii = 0; ii < nobs; ++ii)
bits[ii] = new ibis::bitvector;
std::string fnm; // name of the data file
dataFileName(fnm, f);
nrows = col->partition()->nRows();
ibis::bitvector mask;
{ // name of mask file associated with the data file
array_t<ibis::bitvector::word_t> arr;
std::string mname(fnm);
mname += ".msk";
if (ibis::fileManager::instance().getFile(mname.c_str(), arr) == 0)
mask.copy(ibis::bitvector(arr)); // convert arr to a bitvector
else
mask.set(1, nrows); // default mask
}
// need to do different things for different columns
switch (col->type()) {
case ibis::TEXT:
case ibis::UINT: {// unsigned int
array_t<uint32_t> val;
if (! fnm.empty())
ierr = ibis::fileManager::instance().getFile(fnm.c_str(), val);
else
ierr = col->getValuesArray(&val);
if (ierr < 0 || val.empty()) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- sbiad::construct2 failed to retrieve any value";
break;
}
if (val.size() > mask.size()) {
col->logWarning("sbiad::construct", "the data file \"%s\" "
"contains more elements (%lu) then expected "
"(%lu)", fnm.c_str(),
static_cast<long unsigned>(val.size()),
static_cast<long unsigned>(mask.size()));
mask.adjustSize(nrows, nrows);
}
ibis::bitvector::indexSet iset = mask.firstIndexSet();
uint32_t nind = iset.nIndices();
const ibis::bitvector::word_t *iix = iset.indices();
while (nind) {
if (iset.isRange()) { // a range
uint32_t k = (iix[1] < nrows ? iix[1] : nrows);
for (uint32_t i = *iix; i < k; ++i)
setBit(i, val[i]);
}
else if (*iix+ibis::bitvector::bitsPerLiteral() < nrows) {
// a list of indices
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
setBit(k, val[k]);
}
}
else {
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
if (k < nrows)
setBit(k, val[k]);
}
}
++iset;
nind = iset.nIndices();
if (*iix >= nrows)
nind = 0;
} // while (nind)
break;}
case ibis::INT: {// signed int
array_t<int32_t> val;
if (! fnm.empty())
ierr = ibis::fileManager::instance().getFile(fnm.c_str(), val);
else
ierr = col->getValuesArray(&val);
if (ierr < 0 || val.empty()) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- sbiad::construct2 failed to retrieve any value";
break;
}
if (val.size() > mask.size()) {
col->logWarning("sbiad::construct", "the data file \"%s\" "
"contains more elements (%lu) then expected "
"(%lu)", fnm.c_str(),
static_cast<long unsigned>(val.size()),
static_cast<long unsigned>(mask.size()));
mask.adjustSize(nrows, nrows);
}
ibis::bitvector::indexSet iset = mask.firstIndexSet();
uint32_t nind = iset.nIndices();
const ibis::bitvector::word_t *iix = iset.indices();
while (nind) {
if (iset.isRange()) { // a range
uint32_t k = (iix[1] < nrows ? iix[1] : nrows);
for (uint32_t i = *iix; i < k; ++i)
setBit(i, val[i]);
}
else if (*iix+ibis::bitvector::bitsPerLiteral() < nrows) {
// a list of indices
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
setBit(k, val[k]);
}
}
else {
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
if (k < nrows)
setBit(k, val[k]);
}
}
++iset;
nind = iset.nIndices();
if (*iix >= nrows)
nind = 0;
} // while (nind)
break;}
case ibis::ULONG: {// unsigned long int
array_t<uint64_t> val;
if (! fnm.empty())
ierr = ibis::fileManager::instance().getFile(fnm.c_str(), val);
else
ierr = col->getValuesArray(&val);
if (ierr < 0 || val.empty()) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- sbiad::construct2 failed to retrieve any value";
break;
}
if (val.size() > mask.size()) {
col->logWarning("sbiad::construct", "the data file \"%s\" "
"contains more elements (%lu) then expected "
"(%lu)", fnm.c_str(),
static_cast<long unsigned>(val.size()),
static_cast<long unsigned>(mask.size()));
mask.adjustSize(nrows, nrows);
}
ibis::bitvector::indexSet iset = mask.firstIndexSet();
uint32_t nind = iset.nIndices();
const ibis::bitvector::word_t *iix = iset.indices();
while (nind) {
if (iset.isRange()) { // a range
uint32_t k = (iix[1] < nrows ? iix[1] : nrows);
for (uint32_t i = *iix; i < k; ++i)
setBit(i, val[i]);
}
else if (*iix+ibis::bitvector::bitsPerLiteral() < nrows) {
// a list of indices
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
setBit(k, val[k]);
}
}
else {
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
if (k < nrows)
setBit(k, val[k]);
}
}
++iset;
nind = iset.nIndices();
if (*iix >= nrows)
nind = 0;
} // while (nind)
break;}
case ibis::LONG: {// signed long int
array_t<int64_t> val;
if (! fnm.empty())
ierr = ibis::fileManager::instance().getFile(fnm.c_str(), val);
else
ierr = col->getValuesArray(&val);
if (ierr < 0 || val.empty()) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- sbiad::construct2 failed to retrieve any value";
break;
}
if (val.size() > mask.size()) {
col->logWarning("sbiad::construct", "the data file \"%s\" "
"contains more elements (%lu) then expected "
"(%lu)", fnm.c_str(),
static_cast<long unsigned>(val.size()),
static_cast<long unsigned>(mask.size()));
mask.adjustSize(nrows, nrows);
}
ibis::bitvector::indexSet iset = mask.firstIndexSet();
uint32_t nind = iset.nIndices();
const ibis::bitvector::word_t *iix = iset.indices();
while (nind) {
if (iset.isRange()) { // a range
uint32_t k = (iix[1] < nrows ? iix[1] : nrows);
for (uint32_t i = *iix; i < k; ++i)
setBit(i, val[i]);
}
else if (*iix+ibis::bitvector::bitsPerLiteral() < nrows) {
// a list of indices
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
setBit(k, val[k]);
}
}
else {
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
if (k < nrows)
setBit(k, val[k]);
}
}
++iset;
nind = iset.nIndices();
if (*iix >= nrows)
nind = 0;
} // while (nind)
break;}
case ibis::USHORT: {// unsigned short int
array_t<uint16_t> val;
if (! fnm.empty())
ierr = ibis::fileManager::instance().getFile(fnm.c_str(), val);
else
ierr = col->getValuesArray(&val);
if (ierr < 0 || val.empty()) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- sbiad::construct2 failed to retrieve any value";
break;
}
if (val.size() > mask.size()) {
col->logWarning("sbiad::construct", "the data file \"%s\" "
"contains more elements (%lu) then expected "
"(%lu)", fnm.c_str(),
static_cast<long unsigned>(val.size()),
static_cast<long unsigned>(mask.size()));
mask.adjustSize(nrows, nrows);
}
ibis::bitvector::indexSet iset = mask.firstIndexSet();
uint32_t nind = iset.nIndices();
const ibis::bitvector::word_t *iix = iset.indices();
while (nind) {
if (iset.isRange()) { // a range
uint32_t k = (iix[1] < nrows ? iix[1] : nrows);
for (uint32_t i = *iix; i < k; ++i)
setBit(i, val[i]);
}
else if (*iix+ibis::bitvector::bitsPerLiteral() < nrows) {
// a list of indices
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
setBit(k, val[k]);
}
}
else {
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
if (k < nrows)
setBit(k, val[k]);
}
}
++iset;
nind = iset.nIndices();
if (*iix >= nrows)
nind = 0;
} // while (nind)
break;}
case ibis::SHORT: {// signed short int
array_t<int16_t> val;
if (! fnm.empty())
ierr = ibis::fileManager::instance().getFile(fnm.c_str(), val);
else
ierr = col->getValuesArray(&val);
if (ierr < 0 || val.empty()) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- sbiad::construct2 failed to retrieve any value";
break;
}
if (val.size() > mask.size()) {
col->logWarning("sbiad::construct", "the data file \"%s\" "
"contains more elements (%lu) then expected "
"(%lu)", fnm.c_str(),
static_cast<long unsigned>(val.size()),
static_cast<long unsigned>(mask.size()));
mask.adjustSize(nrows, nrows);
}
ibis::bitvector::indexSet iset = mask.firstIndexSet();
uint32_t nind = iset.nIndices();
const ibis::bitvector::word_t *iix = iset.indices();
while (nind) {
if (iset.isRange()) { // a range
uint32_t k = (iix[1] < nrows ? iix[1] : nrows);
for (uint32_t i = *iix; i < k; ++i)
setBit(i, val[i]);
}
else if (*iix+ibis::bitvector::bitsPerLiteral() < nrows) {
// a list of indices
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
setBit(k, val[k]);
}
}
else {
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
if (k < nrows)
setBit(k, val[k]);
}
}
++iset;
nind = iset.nIndices();
if (*iix >= nrows)
nind = 0;
} // while (nind)
break;}
case ibis::UBYTE: {// unsigned char
array_t<unsigned char> val;
if (! fnm.empty())
ierr = ibis::fileManager::instance().getFile(fnm.c_str(), val);
else
ierr = col->getValuesArray(&val);
if (ierr < 0 || val.empty()) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- sbiad::construct2 failed to retrieve any value";
break;
}
if (val.size() > mask.size()) {
col->logWarning("sbiad::construct", "the data file \"%s\" "
"contains more elements (%lu) then expected "
"(%lu)", fnm.c_str(),
static_cast<long unsigned>(val.size()),
static_cast<long unsigned>(mask.size()));
mask.adjustSize(nrows, nrows);
}
ibis::bitvector::indexSet iset = mask.firstIndexSet();
uint32_t nind = iset.nIndices();
const ibis::bitvector::word_t *iix = iset.indices();
while (nind) {
if (iset.isRange()) { // a range
uint32_t k = (iix[1] < nrows ? iix[1] : nrows);
for (uint32_t i = *iix; i < k; ++i)
setBit(i, val[i]);
}
else if (*iix+ibis::bitvector::bitsPerLiteral() < nrows) {
// a list of indices
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
setBit(k, val[k]);
}
}
else {
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
if (k < nrows)
setBit(k, val[k]);
}
}
++iset;
nind = iset.nIndices();
if (*iix >= nrows)
nind = 0;
} // while (nind)
break;}
case ibis::BYTE: {// signed char
array_t<signed char> val;
if (! fnm.empty())
ierr = ibis::fileManager::instance().getFile(fnm.c_str(), val);
else
ierr = col->getValuesArray(&val);
if (ierr < 0 || val.empty()) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- sbiad::construct2 failed to retrieve any value";
break;
}
if (val.size() > mask.size()) {
col->logWarning("sbiad::construct", "the data file \"%s\" "
"contains more elements (%lu) then expected "
"(%lu)", fnm.c_str(),
static_cast<long unsigned>(val.size()),
static_cast<long unsigned>(mask.size()));
mask.adjustSize(nrows, nrows);
}
ibis::bitvector::indexSet iset = mask.firstIndexSet();
uint32_t nind = iset.nIndices();
const ibis::bitvector::word_t *iix = iset.indices();
while (nind) {
if (iset.isRange()) { // a range
uint32_t k = (iix[1] < nrows ? iix[1] : nrows);
for (uint32_t i = *iix; i < k; ++i)
setBit(i, val[i]);
}
else if (*iix+ibis::bitvector::bitsPerLiteral() < nrows) {
// a list of indices
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
setBit(k, val[k]);
}
}
else {
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
if (k < nrows)
setBit(k, val[k]);
}
}
++iset;
nind = iset.nIndices();
if (*iix >= nrows)
nind = 0;
} // while (nind)
break;}
case ibis::FLOAT: {// (4-byte) floating-point values
array_t<float> val;
if (! fnm.empty())
ierr = ibis::fileManager::instance().getFile(fnm.c_str(), val);
else
ierr = col->getValuesArray(&val);
if (ierr < 0 || val.empty()) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- sbiad::construct2 failed to retrieve any value";
break;
}
if (val.size() > mask.size()) {
col->logWarning("sbiad::construct", "the data file \"%s\" "
"contains more elements (%lu) then expected "
"(%lu)", fnm.c_str(),
static_cast<long unsigned>(val.size()),
static_cast<long unsigned>(mask.size()));
mask.adjustSize(nrows, nrows);
}
ibis::bitvector::indexSet iset = mask.firstIndexSet();
uint32_t nind = iset.nIndices();
const ibis::bitvector::word_t *iix = iset.indices();
while (nind) {
if (iset.isRange()) { // a range
uint32_t k = (iix[1] < nrows ? iix[1] : nrows);
for (uint32_t i = *iix; i < k; ++i)
setBit(i, val[i]);
}
else if (*iix+ibis::bitvector::bitsPerLiteral() < nrows) {
// a list of indices
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
setBit(k, val[k]);
}
}
else {
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
if (k < nrows)
setBit(k, val[k]);
}
}
++iset;
nind = iset.nIndices();
if (*iix >= nrows)
nind = 0;
} // while (nind)
break;}
case ibis::DOUBLE: {// (8-byte) floating-point values
array_t<double> val;
if (! fnm.empty())
ierr = ibis::fileManager::instance().getFile(fnm.c_str(), val);
else
ierr = col->getValuesArray(&val);
if (ierr < 0 || val.empty()) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- sbiad::construct2 failed to retrieve any value";
break;
}
if (val.size() > mask.size()) {
col->logWarning("sbiad::construct", "the data file \"%s\" "
"contains more elements (%lu) then expected "
"(%lu)", fnm.c_str(),
static_cast<long unsigned>(val.size()),
static_cast<long unsigned>(mask.size()));
mask.adjustSize(nrows, nrows);
}
ibis::bitvector::indexSet iset = mask.firstIndexSet();
uint32_t nind = iset.nIndices();
const ibis::bitvector::word_t *iix = iset.indices();
while (nind) {
if (iset.isRange()) { // a range
uint32_t k = (iix[1] < nrows ? iix[1] : nrows);
for (uint32_t i = *iix; i < k; ++i)
setBit(i, val[i]);
}
else if (*iix+ibis::bitvector::bitsPerLiteral() < nrows) {
// a list of indices
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
setBit(k, val[k]);
}
}
else {
for (uint32_t i = 0; i < nind; ++i) {
uint32_t k = iix[i];
if (k < nrows)
setBit(k, val[k]);
}
}
++iset;
nind = iset.nIndices();
if (*iix >= nrows)
nind = 0;
} // while (nind)
break;}
case ibis::CATEGORY: // no need for a separate index
col->logWarning("sbiad::ctor", "no need for another index");
return;
default:
col->logWarning("sbiad::ctor", "unable to create bit sbiad index "
"for column type %s",
ibis::TYPESTRING[(int)col->type()]);
return;
}
// make sure all bit vectors are the same size
for (uint32_t i = 0; i < nobs; ++i) {
bits[i]->adjustSize(0, nrows);
}
// sum up the bitvectors according to interval-encoding
array_t<bitvector*> beq;
beq.swap(bits);
try {
uint32_t ke = 0;
bits.clear();
for (uint32_t i = 0; i < nb; ++i) {
if (bases[i] > 2) {
nobs = (bases[i] - 1) / 2;
bits.push_back(new ibis::bitvector);
bits.back()->copy(*(beq[ke]));
if (nobs > 64)
bits.back()->decompress();
for (uint32_t j = ke+1; j <= ke+nobs; ++j)
*(bits.back()) |= *(beq[j]);
bits.back()->compress();
for (uint32_t j = 1; j < bases[i]-nobs; ++j) {
bits.push_back(*(bits.back()) - *(beq[ke+j-1]));
*(bits.back()) |= *(beq[ke+j+nobs]);
bits.back()->compress();
}
for (uint32_t j = ke; j < ke+bases[i]; ++j) {
delete beq[j];
beq[j] = 0;
}
}
else {
bits.push_back(beq[ke]);
if (bases[i] > 1) {
delete beq[ke+1];
beq[ke+1] = 0;
}
}
ke += bases[i];
}
}
catch (...) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- column::[" << col->name()
<< "]::construct2 encountered an exception while converting "
"to inverval encoding, cleaning up ...";
for (uint32_t i = 0; i < beq.size(); ++ i)
delete beq[i];
throw;
}
beq.clear();
optionalUnpack(bits, col->indexSpec());
// write out the current content
if (ibis::gVerbose > 8) {
ibis::util::logger lg;
print(lg());
}
} // ibis::sbiad::construct2
