/// Read a sub-array.
///
/// @Note The length of the buffer is assumed to be sufficiently large
/// if the argument bufLen is less or equal to 0. The caller is
/// responsible for allocating the buffer (@c buf) of the currect size.
int64_t
ADIOS_Var::readData(void* buf, int64_t bufLen,
const uint64_t* start,
const uint64_t* count)
{
uint64_t total_elm = 1;
for (int i=0; i<getNumDimension(); i++) {
total_elm *= count[i];
#ifdef DEBUG
LOGGER(ibis::gVerbose > 5)
<< "ADIOS_Var::readData: [" << i << "] start=" << start[i]
<< ", count=" << count[i];
#endif
}
{
uint64_t total_bytes = total_elm *
adios_type_size(getType(), readValue());
if (bufLen > 0 && (uint64_t)bufLen < total_bytes) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- ADIOS_Var::readData: bufLen (" << bufLen
<< ") < total_bytes (" << total_bytes << ")";
return -1;
}
}
ADIOS_SELECTION *sel = adios_selection_boundingbox
(m_handle->ndim, start, count);
if (sel == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- ADIOS_Var::readData failed to create a selection";
return -2;
}
IBIS_BLOCK_GUARD(adios_selection_delete, sel);
int ierr = adios_schedule_read_byid
(getFile(), sel, index(), getFile()->current_step, 1, buf);
if (ierr != 0) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- ADIOS_Var::readData call to "
"adios_schedule_read_byid failed due to " << adios_errmsg();
return -3;
}
ierr = adios_perform_reads(getFile(), 1); // 1 == blocking
if (ierr != 0) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- ADIOS_Var::readData call to adios_perform_reads on "
<< getFileName() << " failed due to " << adios_errmsg();
return -4;
}
LOGGER(ibis::gVerbose > 5)
<< "ADIOS_Var::readData: competeled reading " << total_elm
<< " element" << (total_elm>1?"s":"") << " for " << getName()
<< " from " << getFileName();
return total_elm;
} // ADIOS_Var::readData
ibis::table*
ibis::jRange::select(const ibis::table::stringArray& colnames) const {
ibis::table *res = 0;
if (nrows < 0) {
int64_t ierr = count();
if (ierr < 0) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- jRange::count failed with error code"
<< ierr;
return res;
}
}
if (valr_ == 0 || orderr_ == 0 || vals_ == 0 || orders_ == 0 ||
orderr_->size() != maskr_.cnt() || orders_->size() != masks_.cnt()) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- jRange::select failed to evaluate the join";
return res;
}
if (colnames.empty() || nrows == 0) {
std::string nm = ibis::util::shortName(desc_);
res = new ibis::tabula(nm.c_str(), desc_.c_str(), nrows);
return res;
}
const uint32_t ncols = colnames.size();
std::string evt;
evt = "select ";
evt += colnames[0];
for (uint32_t j = 1; j < ncols; ++ j) {
evt += ", ";
evt += colnames[j];
}
if ((desc_[0] != 'F' && desc_[0] != 'f') ||
(desc_[1] != 'R' && desc_[1] != 'r') ||
(desc_[2] != 'O' && desc_[2] != 'o') ||
(desc_[3] != 'M' && desc_[3] != 'm'))
evt += " for ";
else
evt += ' ';
evt += desc_;
ibis::util::timer mytimer(evt.c_str());
std::map<const char*, uint32_t, ibis::lessi> namesToPos;
std::vector<uint32_t> ipToPos(colnames.size());
std::vector<const ibis::column*> ircol, iscol;
std::vector<const ibis::dictionary*> cats(colnames.size(), 0);
// identify the names from the two data partitions
for (uint32_t j = 0; j < ncols; ++ j) {
ipToPos[j] = ncols+1;
const char* cn = colnames[j];
std::string tname;
while (*cn != 0 && *cn != '.') {
tname += *cn;
++ cn;
}
if (*cn == '.') {
++ cn;
}
else { // did not find '.'
tname.erase();
cn = colnames[j];
}
int match = -1; // 0 ==> partr_, 1 ==> parts_
if (! tname.empty()) {
match = frm_->position(tname.c_str());
if (match >= static_cast<long>(frm_->size())) {
if (stricmp(tname.c_str(), partr_.name()) == 0) {
match = 0;
}
else if (stricmp(tname.c_str(), parts_.name()) == 0) {
match = 1;
}
}
}
if (match == 0) {
const ibis::column *col = partr_.getColumn(cn);
if (col != 0) {
namesToPos[colnames[j]] = j;
ipToPos[j] = ircol.size();
ircol.push_back(col);
if (col->type() == ibis::CATEGORY) {
const ibis::category *cat =
static_cast<const ibis::category*>(col);
cats[j] = cat->getDictionary();
}
else if (col->type() == ibis::UINT) {
const ibis::bord::column *bc =
dynamic_cast<const ibis::bord::column*>(col);
if (bc != 0) {
cats[j] = bc->getDictionary();
}
}
}
else {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- " << evt << " can not find column named \""
<< colnames[j] << "\" in data partition \"" << partr_.name()
<< "\"";
return res;
}
}
else if (match == 1) {
const ibis::column *col = parts_.getColumn(cn);
if (col != 0) {
namesToPos[colnames[j]] = j;
ipToPos[j] = ncols - iscol.size();
iscol.push_back(col);
if (col->type() == ibis::CATEGORY) {
const ibis::category *cat =
static_cast<const ibis::category*>(col);
cats[j] = cat->getDictionary();
}
else if (col->type() == ibis::UINT) {
const ibis::bord::column *bc =
dynamic_cast<const ibis::bord::column*>(col);
if (bc != 0) {
cats[j] = bc->getDictionary();
}
}
}
else {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- " << evt << " can not find column named \""
<< colnames[j] << "\" in data partition \""
<< parts_.name() << "\"";
return res;
}
}
else { // not prefixed with a data partition name
cn = colnames[j];
const ibis::column* col = partr_.getColumn(cn);
if (col != 0) {
ipToPos[j] = ircol.size();
ircol.push_back(col);
if (col->type() == ibis::CATEGORY) {
const ibis::category *cat =
static_cast<const ibis::category*>(col);
cats[j] = cat->getDictionary();
}
else if (col->type() == ibis::UINT) {
const ibis::bord::column *bc =
dynamic_cast<const ibis::bord::column*>(col);
if (bc != 0) {
cats[j] = bc->getDictionary();
}
}
LOGGER(ibis::gVerbose > 3)
<< evt << " encountered a column name ("
<< colnames[j] << ") that does not start with a data "
"partition name, assume it is for \"" << partr_.name()
<< "\"";
}
else {
col = parts_.getColumn(cn);
if (col != 0) {
ipToPos[j] = ncols - iscol.size();
iscol.push_back(col);
if (col->type() == ibis::CATEGORY) {
const ibis::category *cat =
static_cast<const ibis::category*>(col);
cats[j] = cat->getDictionary();
}
else if (col->type() == ibis::UINT) {
const ibis::bord::column *bc =
dynamic_cast<const ibis::bord::column*>(col);
if (bc != 0) {
cats[j] = bc->getDictionary();
}
}
LOGGER(ibis::gVerbose > 1)
<< evt << " encountered a column name (" << colnames[j]
<< ") that does not start with a data partition name, "
"assume it is for \"" << parts_.name() << "\"";
}
else {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- " << evt << " encountered a name ("
<< colnames[j] << ") that does not start with a data "
"partition name";
return res;
}
}
}
} // for (uint32_t j = 0; j < ncols;
LOGGER(ibis::gVerbose > 3)
<< evt << " -- found " << ircol.size()
<< " column" << (ircol.size() > 1 ? "s" : "") << " from "
<< partr_.name() << " and " << iscol.size() << " column"
<< (iscol.size() > 1 ? "s" : "") << " from " << parts_.name();
// change Pos values for columns in S to have offset ircol.size()
for (uint32_t j = 0; j < ncols; ++j) {
if (ipToPos[j] <= ncols && ipToPos[j] >= ircol.size())
ipToPos[j] = (ncols - ipToPos[j]) + ircol.size();
}
ibis::table::typeArray rtypes(ircol.size(), ibis::UNKNOWN_TYPE);
ibis::table::bufferArray rbuff(ircol.size(), 0);
IBIS_BLOCK_GUARD(ibis::table::freeBuffers, ibis::util::ref(rbuff),
ibis::util::ref(rtypes));
ibis::table::typeArray stypes(iscol.size(), ibis::UNKNOWN_TYPE);
ibis::table::bufferArray sbuff(iscol.size(), 0);
IBIS_BLOCK_GUARD(ibis::table::freeBuffers, ibis::util::ref(sbuff),
ibis::util::ref(stypes));
bool sane = true;
// retrieve values from r_
for (uint32_t j = 0; sane && j < ircol.size(); ++ j) {
rtypes[j] = ircol[j]->type();
switch (ircol[j]->type()) {
case ibis::BYTE:
rbuff[j] = ircol[j]->selectBytes(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<signed char>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
case ibis::UBYTE:
rbuff[j] = ircol[j]->selectUBytes(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<unsigned char>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
case ibis::SHORT:
rbuff[j] = ircol[j]->selectShorts(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<int16_t>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
case ibis::USHORT:
rbuff[j] = ircol[j]->selectUShorts(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<uint16_t>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
case ibis::INT:
rbuff[j] = ircol[j]->selectInts(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<int32_t>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
case ibis::UINT:
rbuff[j] = ircol[j]->selectUInts(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<uint32_t>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
case ibis::LONG:
rbuff[j] = ircol[j]->selectLongs(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<int64_t>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
case ibis::ULONG:
rbuff[j] = ircol[j]->selectULongs(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<uint64_t>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
case ibis::FLOAT:
rbuff[j] = ircol[j]->selectFloats(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<float>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
case ibis::DOUBLE:
rbuff[j] = ircol[j]->selectDoubles(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<double>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
case ibis::TEXT:
case ibis::CATEGORY:
rbuff[j] = ircol[j]->selectStrings(maskr_);
if (rbuff[j] != 0)
ibis::util::reorder
(*static_cast<std::vector<std::string>*>(rbuff[j]),
*orderr_);
else
sane = false;
break;
default:
sane = false;
rbuff[j] = 0;
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::select does not support column "
"type " << ibis::TYPESTRING[(int)ircol[j]->type()]
<< " (name = " << partr_.name() << "." << ircol[j]->name()
<< ")";
break;
}
}
if (! sane) {
return res;
}
// retrieve values from parts_
for (uint32_t j = 0; sane && j < iscol.size(); ++ j) {
stypes[j] = iscol[j]->type();
switch (iscol[j]->type()) {
case ibis::BYTE:
sbuff[j] = iscol[j]->selectBytes(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<signed char>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
case ibis::UBYTE:
sbuff[j] = iscol[j]->selectUBytes(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<unsigned char>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
case ibis::SHORT:
sbuff[j] = iscol[j]->selectShorts(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<int16_t>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
case ibis::USHORT:
sbuff[j] = iscol[j]->selectUShorts(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<uint16_t>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
case ibis::INT:
sbuff[j] = iscol[j]->selectInts(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<int32_t>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
case ibis::UINT:
sbuff[j] = iscol[j]->selectUInts(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<uint32_t>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
case ibis::LONG:
sbuff[j] = iscol[j]->selectLongs(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<int64_t>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
case ibis::ULONG:
sbuff[j] = iscol[j]->selectULongs(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<uint64_t>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
case ibis::FLOAT:
sbuff[j] = iscol[j]->selectFloats(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<float>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
case ibis::DOUBLE:
sbuff[j] = iscol[j]->selectDoubles(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<array_t<double>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
case ibis::TEXT:
case ibis::CATEGORY:
sbuff[j] = iscol[j]->selectStrings(masks_);
if (sbuff[j] != 0)
ibis::util::reorder
(*static_cast<std::vector<std::string>*>(sbuff[j]),
*orders_);
else
sane = false;
break;
default:
sane = false;
sbuff[j] = 0;
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::select does not support column "
"type " << ibis::TYPESTRING[(int)iscol[j]->type()]
<< " (name = " << parts_.name() << "." << iscol[j]->name()
<< ")";
break;
}
}
if (! sane) {
return res;
}
/// fill the in-memory buffer
switch (colr_.type()) {
case ibis::BYTE:
res = fillResult
(nrows, delta1_, delta2_, evt,
*static_cast<array_t<signed char>*>(valr_), rtypes, rbuff,
*static_cast<array_t<signed char>*>(vals_), stypes, sbuff,
colnames, ipToPos);
break;
case ibis::UBYTE:
res = fillResult
(nrows, delta1_, delta2_, evt,
*static_cast<array_t<unsigned char>*>(valr_), rtypes, rbuff,
*static_cast<array_t<unsigned char>*>(vals_), stypes, sbuff,
colnames, ipToPos);
break;
case ibis::SHORT:
res = fillResult
(nrows, delta1_, delta2_, evt,
*static_cast<array_t<int16_t>*>(valr_), rtypes, rbuff,
*static_cast<array_t<int16_t>*>(vals_), stypes, sbuff,
colnames, ipToPos);
break;
case ibis::USHORT:
res = fillResult
(nrows, delta1_, delta2_, evt,
*static_cast<array_t<uint16_t>*>(valr_), rtypes, rbuff,
*static_cast<array_t<uint16_t>*>(vals_), stypes, sbuff,
colnames, ipToPos);
break;
case ibis::INT:
res = fillResult
(nrows, delta1_, delta2_, evt,
*static_cast<array_t<int32_t>*>(valr_), rtypes, rbuff,
*static_cast<array_t<int32_t>*>(vals_), stypes, sbuff,
colnames, ipToPos);
break;
case ibis::UINT:
res = fillResult
(nrows, delta1_, delta2_, evt,
*static_cast<array_t<uint32_t>*>(valr_), rtypes, rbuff,
*static_cast<array_t<uint32_t>*>(vals_), stypes, sbuff,
colnames, ipToPos);
break;
case ibis::LONG:
res = fillResult
(nrows, delta1_, delta2_, evt,
*static_cast<array_t<int64_t>*>(valr_), rtypes, rbuff,
*static_cast<array_t<int64_t>*>(vals_), stypes, sbuff,
colnames, ipToPos);
break;
case ibis::ULONG:
res = fillResult
(nrows, delta1_, delta2_, evt,
*static_cast<array_t<uint64_t>*>(valr_), rtypes, rbuff,
*static_cast<array_t<uint64_t>*>(vals_), stypes, sbuff,
colnames, ipToPos);
break;
case ibis::FLOAT:
res = fillResult
(nrows, delta1_, delta2_, evt,
*static_cast<array_t<float>*>(valr_), rtypes, rbuff,
*static_cast<array_t<float>*>(vals_), stypes, sbuff,
colnames, ipToPos);
break;
case ibis::DOUBLE:
res = fillResult
(nrows, delta1_, delta2_, evt,
*static_cast<array_t<double>*>(valr_), rtypes, rbuff,
*static_cast<array_t<double>*>(vals_), stypes, sbuff,
colnames, ipToPos);
break;
default:
LOGGER(ibis::gVerbose > 0)
<< "Warning -- " << evt << " can not handle join column of type "
<< ibis::TYPESTRING[(int)colr_.type()];
}
for (unsigned j = 0; j < cats.size(); ++ j) {
if (cats[j] != 0) {
ibis::bord::column *bc = dynamic_cast<ibis::bord::column*>
(static_cast<ibis::bord*>(res)->getColumn(j));
if (bc != 0)
bc->setDictionary(cats[j]);
}
}
return res;
} // ibis::jRange::select
ibis::table*
ibis::jRange::fillResult(size_t nrows, double delta1, double delta2,
const std::string &desc,
const ibis::array_t<T>& rjcol,
const ibis::table::typeArray& rtypes,
const ibis::table::bufferArray& rbuff,
const ibis::array_t<T>& sjcol,
const ibis::table::typeArray& stypes,
const ibis::table::bufferArray& sbuff,
const ibis::table::stringArray& tcname,
const std::vector<uint32_t>& tcnpos) {
if (nrows > (rjcol.size() * sjcol.size()) ||
rtypes.size() != rbuff.size() || stypes.size() != sbuff.size() ||
tcname.size() != rtypes.size() + stypes.size() ||
tcnpos.size() != tcname.size()) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- jRange::fillResult can not proceed due "
"to invalid arguments";
return 0;
}
std::string tn = ibis::util::shortName(desc.c_str());
if (nrows == 0 || rjcol.empty() || sjcol.empty() ||
(stypes.empty() && rtypes.empty()))
return new ibis::tabula(tn.c_str(), desc.c_str(), nrows);
ibis::table::bufferArray tbuff(tcname.size());
ibis::table::typeArray ttypes(tcname.size());
IBIS_BLOCK_GUARD(ibis::table::freeBuffers, ibis::util::ref(tbuff),
ibis::util::ref(ttypes));
try {
// allocate enough space for the output table
for (size_t j = 0; j < tcname.size(); ++ j) {
if (tcnpos[j] < rtypes.size()) {
ttypes[j] = rtypes[tcnpos[j]];
tbuff[j] = ibis::table::allocateBuffer
(rtypes[tcnpos[j]], nrows);
}
else if (tcnpos[j] < rtypes.size()+stypes.size()) {
ttypes[j] = stypes[tcnpos[j]-rtypes.size()];
tbuff[j] = ibis::table::allocateBuffer
(stypes[tcnpos[j]-rtypes.size()], nrows);
}
else { // tcnpos is out of valid range
ttypes[j] = ibis::UNKNOWN_TYPE;
tbuff[j] = 0;
LOGGER(ibis::gVerbose > 0)
<< "Warning -- jRange::fillResult detects an "
"invalid tcnpos[" << j << "] = " << tcnpos[j]
<< ", should be less than " << rtypes.size()+stypes.size();
return 0;
}
}
}
catch (...) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- jRange::fillResult failed to allocate "
"sufficient memory for " << nrows << " row" << (nrows>1?"s":"")
<< " and " << rtypes.size()+stypes.size()
<< " column" << (rtypes.size()+stypes.size()>1?"s":"");
return 0;
}
size_t tind = 0; // row index into the resulting table
uint32_t ir0 = 0;
uint32_t ir1 = 0;
uint32_t is = 0;
const uint32_t nr = rjcol.size();
const uint32_t ns = sjcol.size();
while (ir0 < nr && is < ns) {
while (ir0 < nr && rjcol[ir0] < sjcol[is]+delta1)
++ ir0;
ir1 = (ir1>=ir0?ir1:ir0);
while (ir1 < nr && rjcol[ir1] <= sjcol[is]+delta2)
++ ir1;
if (ir1 > ir0) { // found matches
size_t is0 = is;
while (is < ns && sjcol[is] == sjcol[is0])
++ is;
LOGGER(ibis::gVerbose > 5)
<< "DEBUG -- jRange::fillResult: ir0=" << ir0 << ", ir1="
<< ir1 << ", is0=" << is0 << ", is1=" << is << ", rjcol["
<< ir0 << "]=" << rjcol[ir0] << ", rjcol[" << ir1 << "]="
<< rjcol[ir1] << ", sjcol[" << is0 << "]=" << sjcol[is0]
<< ", sjcol[" << is << "]=" << sjcol[is];
for (size_t jr = ir0; jr < ir1; ++ jr) {
for (size_t js = is0; js < is; ++ js) {
for (size_t jt = 0; jt < tcnpos.size(); ++ jt) {
if (tcnpos[jt] < rbuff.size()) {
ibis::bord::copyValue(rtypes[tcnpos[jt]],
tbuff[jt], tind,
rbuff[tcnpos[jt]], jr);
}
else {
ibis::bord::copyValue
(stypes[tcnpos[jt]-rtypes.size()],
tbuff[jt], tind,
sbuff[tcnpos[jt]-rtypes.size()], js);
}
} // jt
++ tind;
} // js
} // jr
}
else {
++ is;
}
} // while ...
if (tind != nrows) {
LOGGER(ibis::gVerbose >= 0)
<< "Warning -- jRange::fillResult expected to produce "
<< nrows << " row" << (nrows>1?"s":"") << ", but produced "
<< tind << " instead";
return 0;
}
LOGGER(ibis::gVerbose > 3)
<< "jRange(" << desc << ")::fillResult produced " << tind
<< " row" << (tind>1?"s":"") << " for \"" << typeid(T).name()
<< '[' << rjcol.size() << "] - " << typeid(T).name()
<< '[' << sjcol.size() << "] between " << delta1 << " and " << delta2
<< '\"';
return new ibis::bord(tn.c_str(), desc.c_str(), nrows,
tbuff, ttypes, tcname);
} // ibis::jRange::fillResult
/// Read the index contained in the file f. This function always reads all
/// bitvectors.
int ibis::skive::read(const char* f) {
std::string fnm;
indexFileName(fnm, f);
int fdes = UnixOpen(fnm.c_str(), OPEN_READONLY);
if (fdes < 0) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- skive[" << col->partition()->name() << '.'
<< col->name() << "]::read failed to open " << fnm;
return -1; // can not do anything else
}
char header[8];
IBIS_BLOCK_GUARD(UnixClose, fdes);
#if defined(_WIN32) && defined(_MSC_VER)
(void)_setmode(fdes, _O_BINARY);
#endif
int ierr = UnixRead(fdes, static_cast<void*>(header), 8);
if (ierr != 8) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- skive[" << col->partition()->name() << '.'
<< col->name() << "]::read failed to read 8 bytes from "
<< fnm;
return -2;
}
if (!(header[0] == '#' && header[1] == 'I' &&
header[2] == 'B' && header[3] == 'I' &&
header[4] == 'S' &&
(header[5] == static_cast<char>(ibis::index::SKIVE) ||
header[5] == static_cast<char>(ibis::index::SLICE)) &&
(header[6] == 8 || header[6] == 4) &&
header[7] == static_cast<char>(0))) {
if (ibis::gVerbose > 0) {
ibis::util::logger lg;
lg() << "Warning -- skive[" << col->partition()->name() << '.'
<< col->name() << "]::read the header from " << fnm << " (";
printHeader(lg(), header);
lg() << ") does not contain the expected values";
}
return -3;
}
uint32_t dim[3]; // nrows, nobs, card
size_t begin, end;
clear(); // clear the current content
ierr = UnixRead(fdes, static_cast<void*>(dim), 3*sizeof(uint32_t));
nrows = dim[0];
// read vals
begin = 8*((3*sizeof(uint32_t) + 15) / 8);
end = begin + dim[2] * sizeof(double);
{
array_t<double> dbl(fnm.c_str(), fdes, begin, end);
vals.swap(dbl);
}
// read the offsets
begin = end;
end += header[6] * (dim[1] + 1);
ierr = initOffsets(fdes, header[6], begin, dim[1]);
if (ierr < 0)
return ierr;
// cnts
begin = end;
end += sizeof(uint32_t) * dim[2];
{
array_t<uint32_t> szt(fnm.c_str(), fdes, begin, end);
cnts.swap(szt);
}
ibis::fileManager::instance().recordPages(0, end);
initBitmaps(fdes);
activate();
return 0;
} // ibis::skive::read
// the argument is the name of the directory or the file name
int ibis::skive::write(const char* dt) const {
if (vals.empty()) return -1;
std::string fnm;
indexFileName(fnm, dt);
if (fnm.empty()) {
return 0;
}
else if (0 != str && 0 != str->filename() &&
0 == fnm.compare(str->filename())) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- skive::write can not overwrite the index file \""
<< fnm << "\" while it is used as a read-only file map";
return 0;
}
else if (fname != 0 && *fname != 0 && 0 == fnm.compare(fname)) {
activate(); // read everything into memory
fname = 0; // break the link with the named file
}
ibis::fileManager::instance().flushFile(fnm.c_str());
int fdes = UnixOpen(fnm.c_str(), OPEN_WRITENEW, OPEN_FILEMODE);
if (fdes < 0) {
ibis::fileManager::instance().flushFile(fnm.c_str());
fdes = UnixOpen(fnm.c_str(), OPEN_WRITENEW, OPEN_FILEMODE);
if (fdes < 0) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- skive[" << col->partition()->name() << '.'
<< col->name() << "]::write failed to open \"" << fnm
<< "\" for writing";
return -2;
}
}
IBIS_BLOCK_GUARD(UnixClose, fdes);
#if defined(_WIN32) && defined(_MSC_VER)
(void)_setmode(fdes, _O_BINARY);
#endif
#ifdef FASTBIT_USE_LONG_OFFSETS
const bool useoffset64 = true;
#else
const bool useoffset64 = (getSerialSize()+8 > 0x80000000UL);
#endif
char header[] = "#IBIS\11\0\0";
header[5] = (char)ibis::index::SKIVE;
header[6] = (char)(useoffset64 ? 8 : 4);
int ierr = UnixWrite(fdes, header, 8);
if (ierr < 8) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- skive[" << col->partition()->name() << "."
<< col->name() << "]::write(" << fnm
<< ") failed to write the 8-byte header, ierr = " << ierr;
return -3;
}
if (useoffset64)
ierr = write64(fdes);
else
ierr = write32(fdes);
if (ierr >= 0) {
#if defined(FASTBIT_SYNC_WRITE)
#if _POSIX_FSYNC+0 > 0
(void) UnixFlush(fdes); // write to disk
#elif defined(_WIN32) && defined(_MSC_VER)
(void) _commit(fdes);
#endif
#endif
LOGGER(ibis::gVerbose > 3)
<< "skive[" << col->partition()->name() << "." << col->name()
<< "]::write wrote " << bits.size() << " bitmap"
<< (bits.size()>1?"s":"") << " to file " << fnm;
}
return ierr;
} // ibis::skive::write
/**
Write the content of the dictionary to the named file. The existing
content in the named file is overwritten. The content of the dictionary
file is laid out as follows.
\li Signature "#IBIS Dictionary " and version number (currently 0). (20
bytes)
\li N = Number of strings in the file. (4 bytes)
\li uint32_t[N]: the integer values assigned to the strings.
\li uint32_t[N+1]: the starting positions of the strings in this file.
\li the string values packed one after the other with nil terminators.
*/
int ibis::dictionary::write(const char* name) const {
if (name == 0 || *name == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- dictionary::write can not proceed with a "
"null string as the file name";
return -1;
}
if (! (code_.size() == key_.size() &&
code_.size()+1 == raw_.size())) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- dictionary::write(" << name
<< ") can not write an inconsistent dictionary, key_.size("
<< key_.size() << "), code_.size(" << code_.size()
<< "), raw_.size(" << raw_.size() << ")";
return -2;
}
FILE* fptr = fopen(name, "wb");
if (fptr == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- dictionary::write(" << name
<< ") failed to open the file ... "
<< (errno ? strerror(errno) : "no free stdio stream");
return -3;
}
IBIS_BLOCK_GUARD(fclose, fptr);
int ierr = fwrite(_fastbit_dictionary_header, 1, 20, fptr);
if (ierr != 20) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- dictionary::write(" << name
<< ") failed to write the header, fwrite returned " << ierr;
return -4;
}
const uint32_t nkeys = key_.size();
ierr = fwrite(&nkeys, sizeof(nkeys), 1, fptr);
if (ierr != 1) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- dictionary::write(" << name
<< ") failed to write the size(" << nkeys
<< "), fwrite returned " << ierr;
return -5;
}
if (nkeys == 0) // nothing else to write
return 0;
ierr = fwrite(code_.begin(), sizeof(uint32_t), nkeys, fptr);
if (ierr != (int)nkeys) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- dictionary::write(" << name
<< ") failed to write " << nkeys << " code value"
<< (nkeys>1?"s":"") << ", fwrite returned " << ierr;
return -6;
}
array_t<uint32_t> pos(nkeys+1);
ierr = fseek(fptr, sizeof(uint32_t)*(nkeys+1), SEEK_CUR);
long int tmp = ftell(fptr);
pos[0] = tmp;
for (unsigned i = 0; i < nkeys; ++ i) {
const int len = 1 + strlen(key_[i]);
ierr = fwrite(key_[i], 1, len, fptr);
LOGGER(ierr != len && ibis::gVerbose > 1)
<< "Warning -- dictionary::write(" << name
<< ") failed to write key[" << i << "]; expected fwrite to return "
<< len << ", but got " << ierr;
tmp = ftell(fptr);
pos[i+1] = tmp;
LOGGER((long int)(pos[i+1]) != tmp && ibis::gVerbose > 1)
<< "Warning -- dictionary::write(" << name
<< ") failed to store position " << tmp
<< " into a 32-bit integer; dictionary file will be unusable!";
}
// go back to write the positions
ierr = fseek(fptr, 24+4*nkeys, SEEK_SET);
LOGGER(ierr != 0 && ibis::gVerbose > 1)
<< "Warning -- dictionary::write(" << name
<< ") failed to seek to " << 24+4*nkeys
<< " to write the offsets";
ierr = fwrite(pos.begin(), 4, nkeys+1, fptr);
LOGGER(ierr != (int)(nkeys+1) && ibis::gVerbose > 1)
<< "Warning -- dictionary::write(" << name
<< ") failed to write the offsets, expected fwrite to return "
<< nkeys+1 << ", but got " << ierr;
return -7 * (ierr != (int)(nkeys+1));
} // ibis::dictionary::write
/// Read the content of the named file. The file content is read into the
/// buffer in one-shot and then digested.
int ibis::dictionary::read(const char* name) {
if (name == 0 || *name == 0) return -1;
std::string evt = "dictionary::read(";
evt += name;
evt += ')';
// open the file to read
int ierr = 0;
FILE* fptr = fopen(name, "rb");
if (fptr == 0) {
LOGGER(ibis::gVerbose > 3)
<< "Warning -- " << evt << " failed to open the file ... "
<< (errno ? strerror(errno) : "no free stdio stream");
return -2;
}
ibis::util::timer mytimer(evt.c_str(), 4);
IBIS_BLOCK_GUARD(fclose, fptr);
ierr = fseek(fptr, 0, SEEK_END); // to the end
if (ierr != 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- " << evt << " failed to seek to the end of the file";
return -3;
}
long int sz = ftell(fptr); // file size
if (sz < 24) { // must be the old style dictionary file
return readRaw(evt.c_str(), fptr);
}
else {
char header[20];
ierr = fseek(fptr, 0, SEEK_SET);
if (ierr != 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- " << evt << " failed to seek to the beginning "
"of the file";
return -4;
}
ierr = fread(header, 1, 20, fptr);
if (ierr != 20) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- " << evt << " failed to read the 20-byte header";
return -5;
}
if (header[0] == _fastbit_dictionary_header[0] &&
header[1] == _fastbit_dictionary_header[1] &&
header[2] == _fastbit_dictionary_header[2] &&
header[3] == _fastbit_dictionary_header[3] &&
header[4] == _fastbit_dictionary_header[4] &&
header[5] == _fastbit_dictionary_header[5] &&
header[6] == _fastbit_dictionary_header[6] &&
header[7] == _fastbit_dictionary_header[7] &&
header[8] == _fastbit_dictionary_header[8] &&
header[9] == _fastbit_dictionary_header[9] &&
header[10] == _fastbit_dictionary_header[10] &&
header[11] == _fastbit_dictionary_header[11] &&
header[12] == _fastbit_dictionary_header[12] &&
header[13] == _fastbit_dictionary_header[13] &&
header[14] == _fastbit_dictionary_header[14] &&
header[15] == _fastbit_dictionary_header[15] &&
header[16] == _fastbit_dictionary_header[16] &&
header[17] == _fastbit_dictionary_header[17] &&
header[18] == _fastbit_dictionary_header[18] &&
header[19] == _fastbit_dictionary_header[19]) {
// got the expected header
return readKeys(evt.c_str(), fptr);
}
else {
LOGGER(ibis::gVerbose > 2)
<< evt << " did not find the expected header, assume "
"to be an old-style dictionary";
return readRaw(evt.c_str(), fptr);
}
}
} // ibis::dictionary::read
/**
Write the content of the dictionary to the named file. The existing
content in the named file is overwritten. The content of the dictionary
file is laid out as follows.
\li Signature "#IBIS Dictionary " and version number (currently
0x020000). (20 bytes)
\li N = Number of strings in the file. (4 bytes)
\li uint64_t[N+1]: the starting positions of the strings in this file.
\li uint32_t[N]: The integer code corresponding to each string value.
\li the string values packed one after the other with their nil
terminators.
*/
int ibis::dictionary::write(const char* name) const {
std::string evt = "dictionary::write";
if (name == 0 || *name == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- " << evt << " can not proceed with a "
"null string as the file name";
return -1;
}
if (ibis::gVerbose > 1) {
evt += '(';
evt += name;
evt += ')';
}
if (key_.size() > raw_.size()) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- " << evt
<< " can not write an inconsistent dictionary, key_.size("
<< key_.size() << "), raw_.size(" << raw_.size() << ")";
return -2;
}
ibis::util::timer mytimer(evt.c_str(), 4);
FILE* fptr = fopen(name, "wb");
if (fptr == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- " << evt << " failed to open the file ... "
<< (errno ? strerror(errno) : "no free stdio stream");
return -3;
}
IBIS_BLOCK_GUARD(fclose, fptr);
int ierr = fwrite(_fastbit_dictionary_header, 1, 20, fptr);
if (ierr != 20) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- " << evt
<< " failed to write the header, fwrite returned " << ierr;
return -4;
}
const uint32_t nkeys = key_.size();
ierr = fwrite(&nkeys, sizeof(nkeys), 1, fptr);
if (ierr != 1) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- " << evt << " failed to write the size(" << nkeys
<< "), fwrite returned " << ierr;
return -5;
}
if (nkeys == 0) // nothing else to write
return 0;
mergeBuffers();
array_t<uint64_t> pos(nkeys+1);
array_t<uint32_t> qos(nkeys);
pos.clear();
qos.clear();
pos.push_back(0);
if (buffer_.size() == 1) {
for (uint32_t j = 0; j < raw_.size(); ++ j) {
if (raw_[j] != 0) {
pos.push_back(1U + strlen(raw_[j]));
qos.push_back(j);
}
}
ierr = writeBuffer(fptr, nkeys, pos, qos);
}
else {
ierr = writeKeys(fptr, nkeys, pos, qos);
}
LOGGER(ibis::gVerbose > 1)
<< evt << " complete with ierr = " << ierr;
return ierr;
} // ibis::dictionary::write
/// Read the content of the named file. The file content is read into the
/// buffer in one-shot and then digested.
///
/// This function determines the version of the dictionary and invokes the
/// necessary reading function to perform the actual reading operations.
/// Currently there are three possible version of dictioanries
/// 0x02000000 - the version produced by the current write function,
/// 0x01000000 - the version with 64-bit offsets, consecutive kyes, strings
/// are stored in key order
/// 0x00000000 - the version 32-bit offsets and stores strings in
/// sorted order.
/// unmarked - the version without a header, only has the bare strings in
/// the code order.
int ibis::dictionary::read(const char* name) {
if (name == 0 || *name == 0) return -1;
std::string evt = "dictionary::read";
if (ibis::gVerbose > 1) {
evt += '(';
evt += name;
evt += ')';
}
// open the file to read
int ierr = 0;
FILE* fptr = fopen(name, "rb");
if (fptr == 0) {
LOGGER(ibis::gVerbose > 3)
<< "Warning -- " << evt << " failed to open the file ... "
<< (errno ? strerror(errno) : "no free stdio stream");
return -2;
}
ibis::util::timer mytimer(evt.c_str(), 4);
IBIS_BLOCK_GUARD(fclose, fptr);
ierr = fseek(fptr, 0, SEEK_END); // to the end
if (ierr != 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- " << evt << " failed to seek to the end of the file";
return -3;
}
uint32_t version = 0xFFFFFFFFU;
long int sz = ftell(fptr); // file size
if (sz > 24) {
char header[20];
ierr = fseek(fptr, 0, SEEK_SET);
if (ierr != 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- " << evt << " failed to seek to the beginning "
"of the file";
return -4;
}
ierr = fread(header, 1, 20, fptr);
if (ierr != 20) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- " << evt << " failed to read the 20-byte header";
return -5;
}
if (header[0] == _fastbit_dictionary_header[0] &&
header[1] == _fastbit_dictionary_header[1] &&
header[2] == _fastbit_dictionary_header[2] &&
header[3] == _fastbit_dictionary_header[3] &&
header[4] == _fastbit_dictionary_header[4] &&
header[5] == _fastbit_dictionary_header[5] &&
header[6] == _fastbit_dictionary_header[6] &&
header[7] == _fastbit_dictionary_header[7] &&
header[8] == _fastbit_dictionary_header[8] &&
header[9] == _fastbit_dictionary_header[9] &&
header[10] == _fastbit_dictionary_header[10] &&
header[11] == _fastbit_dictionary_header[11] &&
header[12] == _fastbit_dictionary_header[12] &&
header[13] == _fastbit_dictionary_header[13] &&
header[14] == _fastbit_dictionary_header[14] &&
header[15] == _fastbit_dictionary_header[15]) {
version = (header[16] << 24 | header[17] << 16 |
header[18] << 8 | header[19]);
LOGGER(ibis::gVerbose > 3)
<< evt << " detected dictionary version 0x" << std::hex
<< version << std::dec;
}
else {
LOGGER(ibis::gVerbose > 2)
<< evt << " did not find the expected header, assume "
"to have no header (oldest version of dictioinary)";
}
}
// invoke the actual reader based on version number
switch (version) {
case 0x02000000:
ierr = readKeys2(evt.c_str(), fptr);
break;
case 0x01000000:
ierr = readKeys1(evt.c_str(), fptr);
break;
case 0x00000000:
ierr = readKeys0(evt.c_str(), fptr);
break;
default:
ierr = readRaw(evt.c_str(), fptr);
break;
}
if (ibis::gVerbose > 3) {
ibis::util::logger lg;
lg() << evt << " completed with ";
toASCII(lg());
}
return ierr;
} // ibis::dictionary::read
