int StructuralStatsDatabase::subtractStats(OperationContext &context, const NameID &id1, const NameID &id2,
const StructuralStats &stats)
{
DBXML_ASSERT(id1 != 0 || id2 == 0);
Cursor myCursor(const_cast<DbWrapper&>(db_), getTxn(context), CURSOR_WRITE, 0);
if(myCursor.error() != 0) return myCursor.error();
marshalKey(id1, id2, context.key());
int err = myCursor.get(context.key(), context.data(), DB_SET);
if(err != DB_NOTFOUND && err != 0) return err;
StructuralStats current;
if(err == DB_NOTFOUND) {
current.subtract(stats);
current.marshal(context.data(), id2 == 0);
err = myCursor.put(context.key(), context.data(), DB_KEYLAST);
} else {
current.unmarshal(context.data());
current.subtract(stats);
current.marshal(context.data(), id2 == 0);
err = myCursor.put(context.key(), context.data(), DB_CURRENT);
}
return err;
}
void DictionaryDatabase::display(OperationContext &context, ostream &out) const
{
{
Cursor myCursor(const_cast<PrimaryDatabase&>(*primary_.get()), getTxn(context), CURSOR_READ, 0);
if(myCursor.error() != 0) throw XmlException(myCursor.error());
int err = 0;
NameID id;
while((err = myCursor.get(context.key(), context.data(), DB_NEXT)) == 0) {
id.setThisFromDbtAsId(context.key());
Buffer val(context.data().data, context.data().size, true);
out << id << " -> " << val.asString(true) << endl;
}
}
{
Cursor myCursor(const_cast<SecondaryDatabase&>(*secondary_.get()), getTxn(context), CURSOR_READ, 0);
if(myCursor.error() != 0) throw XmlException(myCursor.error());
int err = 0;
NameID id;
while((err = myCursor.get(context.key(), context.data(), DB_NEXT)) == 0) {
Buffer val(context.key().data, context.key().size, true);
id.setThisFromDbt(context.data());
out << val.asString(true) << " -> " << id << endl;
}
}
}
int DocumentDatabase::getMetaData(OperationContext &context,
DictionaryDatabase *dictionary, const Name &name,
const DocID &did, XmlValue::Type &type, DbXmlDbt *metadata,
u_int32_t flags) const
{
NameID nid;
int err = dictionary->lookupIDFromName(context, name,
nid, /*define=*/false);
if(err == 0) {
Cursor cursor(const_cast<SecondaryDatabase&>(secondary_),
context.txn(), CURSOR_READ, "DocumentMetaData", flags);
u_int32_t origFlags = DB_CURSOR_GET_MASK(const_cast<SecondaryDatabase&>(secondary_),flags);
MetaDatum::setKeyDbt(did, nid, XmlValue::NONE, context.key());
DbtIn none;
none.set_flags(DB_DBT_PARTIAL); // Don't pull back the data.
err = cursor.get(context.key(), none,
(flags | DB_SET_RANGE) & ~DB_RMW);
if (err == 0) {
DocID db_did;
NameID db_nid;
MetaDatum::decodeKeyDbt(context.key(), db_did,
db_nid, type);
if(db_did == did && db_nid == nid) {
err = cursor.get(context.key(), *metadata,
origFlags|DB_CURRENT);
} else {
return DB_NOTFOUND;
}
}
}
return err;
}
int DocumentDatabase::addMetaData(OperationContext &oc,
DictionaryDatabase *dictionary,
Document &document)
{
int err = 0;
MetaData::const_iterator end = document.metaDataEnd();
MetaData::const_iterator i;
for (i = document.metaDataBegin(); err == 0 && i != end; ++i) {
NameID nid;
err = dictionary->lookupIDFromName(oc,
(*i)->getName(),
nid, /*define=*/true);
if(err == 0) {
DbtIn value;
MetaDatum::setKeyDbt(document.getID(),
nid, (*i)->getType(),
oc.key());
(*i)->setValueDbtFromThis(value);
// could throw on error
err = secondary_.put(oc.txn(), &oc.key(),
&value, 0);
}
}
if(err == 0)
for(i = document.metaDataBegin(); i != end; ++i)
(*i)->setModified(false);
return err;
}
void
NsUpgradeReader::initDocInfo()
{
// read doc node
// read namespace info
OperationContext oc;
oc.set(NULL);
const char metadataId[] =
{ NS_PROTOCOL_VERSION_COMPAT, NS_METADATA_ID, 0 };
DbtOut data((void *)metadataId, strlen(metadataId) + 1);
// reproduce old NsDocumentDatabase::getNodeRecord(), but
// cannot deadlock -- no transaction
DBXML_ASSERT(!oc.txn());
id_.setDbtFromThis(oc.key());
int ret = db_.get(oc.txn(), &oc.key(), &data, DB_GET_BOTH);
uint32_t flags = 0;
if (ret == 0) {
const xmlbyte_t *ptr = (const xmlbyte_t *) data.data;
ptr += 3; // past version and id
size_t len;
ptr += NsFormat::unmarshalInt(ptr, &flags);
if (flags & NSDOC_HASDECL) {
ptr += NsFormat::unmarshalInt(ptr, &xmlDecl_);
DBXML_ASSERT(xmlDecl_ == 0 || xmlDecl_ == 1);
}
if (flags & NSDOC_HASENCODE) {
DBXML_ASSERT(!encStr_);
encStr_ = NsUtil::nsStringDup(ptr, &len);
ptr += len;
}
if (flags & NSDOC_HASSNIFF) {
sniffStr_ = NsUtil::nsStringDup(ptr, &len);
ptr += len;
}
if (flags & NSDOC_STANDYES)
standStr_ = _standYes;
if (flags & NSDOC_STANDNO)
standStr_ = _standNo;
}
if (flags & NSDOC_NAMESPACE) {
nsInfo_ = new NsNamespaceInfo();
if (!nsInfo_)
NsUtil::nsThrowException(XmlException::NO_MEMORY_ERROR,
"new failed to allocate memory",
__FILE__, __LINE__);
nsInfo_->initialize();
const char namespaceId[] =
{ NS_PROTOCOL_VERSION_COMPAT, NS_NAMESPACE_ID, 0 };
DbtOut ndata((void *)namespaceId,
strlen(namespaceId) + 1);
id_.setDbtFromThis(oc.key());
ret = db_.get(oc.txn(), &oc.key(), &ndata, DB_GET_BOTH);
if (ret == 0) {
nsInfo_->load((const char*)
((const char *)ndata.data)+3);
}
}
}
int StructuralStatsDatabase::addStats(OperationContext &context, const StructuralStatsDatabase &sdb)
{
Cursor myCursor(db_, getTxn(context), CURSOR_WRITE, 0);
if(myCursor.error() != 0) return myCursor.error();
Cursor sdbCursor(const_cast<DbWrapper&>(sdb.db_), sdb.getTxn(context), CURSOR_READ, 0);
if(sdbCursor.error() != 0) return sdbCursor.error();
int err = 0;
StructuralStats stats;
StructuralStats current;
bool nodeStats;
while((err = sdbCursor.get(context.key(), context.data(), DB_NEXT)) == 0) {
stats.reset();
nodeStats = stats.unmarshal(context.data());
err = myCursor.get(context.key(), context.data(), DB_SET);
if(err == DB_NOTFOUND) {
stats.marshal(context.data(), nodeStats);
err = myCursor.put(context.key(), context.data(), DB_KEYLAST);
} else if(err == 0) {
current.unmarshal(context.data());
current.add(stats);
current.marshal(context.data(), nodeStats);
err = myCursor.put(context.key(), context.data(), DB_CURRENT);
} else break;
}
if(err != DB_NOTFOUND) return err;
return 0;
}
int StructuralStatsDatabase::addStats(OperationContext &context, const StructuralStatsWriteCache &cache)
{
Cursor myCursor(db_, getTxn(context), CURSOR_WRITE, 0);
if(myCursor.error() != 0) return myCursor.error();
int err = 0;
StructuralStats current;
StructuralStatsWriteCache::Map::const_iterator it = cache.cache_.begin();
for(; err == 0 && it != cache.cache_.end(); ++it) {
StructuralStatsWriteCache::InnerMap::const_iterator it2 = it->second.begin();
for(; it2 != it->second.end(); ++it2) {
marshalKey(it->first, it2->first, context.key());
err = myCursor.get(context.key(), context.data(), DB_SET);
if(err == DB_NOTFOUND) {
it2->second.marshal(context.data(), it2->first == 0);
err = myCursor.put(context.key(), context.data(), DB_KEYLAST);
} else if(err == 0) {
current.unmarshal(context.data());
current.add(it2->second);
current.marshal(context.data(), it2->first == 0);
err = myCursor.put(context.key(), context.data(), DB_CURRENT);
} else break;
}
}
if(err != DB_NOTFOUND) return err;
return 0;
}
int DictionaryDatabase::lookupIDFromName(OperationContext &context,
const Name &name,
NameID &id, bool define)
{
int err = 0;
if (name == Name::dbxml_colon_name) {
id = nidName_;
} else if (name == Name::dbxml_colon_root) {
id = nidRoot_;
}
if (id == 0) {
MutexLock ml(mutex_);
u_int32_t flags = (getTxn(context) && !define) ? DB_READ_COMMITTED : 0;
name.setDbtFromThis_SecondaryKey(context.key());
if (stringCacheLookup(context, context.key(), id))
return 0;
err = secondary_->get(getTxn(context), &context.key(), &context.data(), flags);
if (err == 0) {
id.setThisFromDbt(context.data());
stringCache_.insert(&context.key(), id.raw());
} else if (err == DB_NOTFOUND && define) {
err = defineName(context, name, id); // define from name:uri
} else {
id.reset();
}
}
return err;
}
void StructuralStatsDatabase::display(OperationContext &context, ostream &out, const DictionaryDatabase *ddb) const
{
Cursor myCursor(const_cast<DbWrapper&>(db_), getTxn(context), CURSOR_WRITE, 0);
if(myCursor.error() != 0) throw XmlException(myCursor.error());
NameID id1;
NameID id2;
StructuralStats stats;
int err = 0;
while((err = myCursor.get(context.key(), context.data(), DB_NEXT)) == 0) {
id1.reset();
id2.reset();
unmarshalKey(context.key(), id1, id2);
stats.reset();
stats.unmarshal(context.data());
if(ddb) out << ddb->lookupName(context, id1);
else out << id1;
if(id2 != 0) {
if(ddb) out << " -> " << ddb->lookupName(context, id2);
else out << " -> " << id2;
}
out << ": ";
stats.display(out);
out << endl;
}
}
int PrimaryDatabase::appendPrimary(OperationContext &context, NameID &id,
const DbXmlDbt &data, u_int32_t flags)
{
int err = put(context.txn(), &context.key(), &(const_cast<DbXmlDbt&>(data)),
flags | DB_APPEND);
if (err == 0)
id.setThisFromDbtAsId(context.key());
return err;
}
int DocumentDatabase::getAllMetaData(OperationContext &context,
DictionaryDatabase *dictionary,
Document *document,
u_int32_t flags) const
{
int err = 0;
u_int32_t orig_flags = flags;
const DocID &did = document->getID();
//
// Read each of the meta-data items from the document secondary
// database. Content is no longer considered metadata
//
Cursor cursor(const_cast<SecondaryDatabase&>(secondary_),
context.txn(), CURSOR_READ, "DocumentMetaData", flags);
orig_flags = flags = DB_CURSOR_GET_MASK(const_cast<SecondaryDatabase&>(secondary_),flags);
flags |= DB_SET_RANGE;
bool done = false;
while (!done) {
did.setDbtFromThis(context.key());
DbtIn none;
none.set_flags(DB_DBT_PARTIAL); // Don't pull back the data.
// if DB_RMW set, don't get locks on this get, wait for a match
err = cursor.get(context.key(), none, flags & (~DB_RMW));
if (err == 0) {
DocID db_did;
NameID db_nid;
XmlValue::Type type;
MetaDatum::decodeKeyDbt(context.key(), db_did,
db_nid, type);
if (did == db_did) {
Name name;
err = dictionary->lookupNameFromID(context, db_nid, name);
if(err == 0 && !document->containsMetaData(name)) {
DbtOut *data = new DbtOut();
err = cursor.get(context.key(), *data,
DB_CURRENT);
if(err == 0) document->setMetaData(
name, type, &data,
/*modified*/false);
delete data; // If not consumed by setThis..
}
flags = orig_flags | DB_NEXT;
} else {
err = 0;
done = true;
}
} else if (err == DB_NOTFOUND) {
err = 0;
done = true;
} else {
done = true;
}
}
return err;
}
int StructuralStatsDatabase::getStats(OperationContext &context, StructuralStats &stats) const
{
Cursor myCursor(const_cast<DbWrapper&>(db_), getTxn(context), CURSOR_READ, 0);
if(myCursor.error() != 0) return myCursor.error();
StructuralStats current;
// Loop over every node and every node's descendant information
int err;
while((err = myCursor.get(context.key(), context.data(), DB_NEXT)) == 0) {
current.reset();
current.unmarshal(context.data());
stats.add(current);
}
if(err != DB_NOTFOUND && err != 0) return err;
if(stats.sumSize_ == 0 && stats.numberOfNodes_ != 0) {
// Fill in an estimate for the missing size values that you get with DLS
stats.sumSize_ = NODE_SIZE * stats.numberOfNodes_;
stats.sumChildSize_ = NODE_SIZE * stats.sumNumberOfChildren_;
stats.sumDescendantSize_ = NODE_SIZE * stats.sumNumberOfDescendants_;
}
return 0;
}
// puts result in dbt, which may be context.data(), or may be user-provided
int DictionaryDatabase::lookupFromID(
OperationContext &context, DbtOut &dbt,
const NameID &id) const
{
int err = 0;
if (cache_.lookup(context, id, dbt, false))
return 0;
u_int32_t flags = (getTxn(context)) ? DB_READ_COMMITTED : 0;
// primary key is the integer value of the id (database is
// recno)
id.setDbtFromThisAsId(context.key());
MutexLock ml(mutex_);
err = primary_->get(getTxn(context), &context.key(),
&dbt, flags);
if (err == 0)
cache_.insert(id.raw(), dbt);
return err;
}
// mutex is locked if present and necessary
int DictionaryDatabase::defineStringName(OperationContext &context,
const char *name, size_t namelen,
NameID &id)
{
// Primary { id -> name\0 }
// Secondary { name -> id }
int err = 0;
id.reset();
DbtIn primaryData((void*)name, namelen + 1); // add null
DbtIn secondaryKey((void*)name, namelen); // don't add null
err = primary_->appendPrimary(context, id,
primaryData, /*no flags*/0);
if (err == 0) {
/* Cache after insert. This prevents the cache from returning bad answers
* on a transaction abort, and speeds up access to newly inserted element
* and attribute names that are needed for indexing after an insert.
*/
cache_.insert(id.raw(), primaryData);
id.setDbtFromThis(context.key());
Transaction *txn = getTxn(context);
err = secondary_->put(txn, &secondaryKey,
&context.key(),
/*no flags*/0);
if (err == 0) {
/* Add to the transaction's string cache, that way if an abort occurs
* the values can be removed from the cache.
*/
if (txn) {
DictionaryStringCache *dsc = txn->getStringCache(this, true);
dsc->insert(&secondaryKey, id.raw());
}
if (Log::isLogEnabled(Log::C_DICTIONARY, Log::L_INFO)) {
ostringstream oss;
oss << "Define new name " << id << " -> " << name;
Log::log(environment_, Log::C_DICTIONARY, Log::L_INFO,
name_.c_str(),
oss.str().c_str());
}
}
}
return err;
}
// if toRemove is non-null, it specifies a list of Name IDs
// to remove; otherwise remove all metadata for the target document
int DocumentDatabase::removeMetaData(OperationContext &oc,
const DocID &id,
std::vector<NameID> *toRemove)
{
Cursor cursor(const_cast<SecondaryDatabase&>(secondary_), oc.txn(),
CURSOR_WRITE, "DocumentMetaData_remove");
DbtIn none;
none.set_flags(DB_DBT_PARTIAL); // Don't pull back the data.
id.setDbtFromThis(oc.key());
DocID db_id;
int err = cursor.get(oc.key(), none, DB_SET_RANGE);
try {
while(err == 0) {
if (toRemove) {
NameID nm_id;
XmlValue::Type type;
MetaDatum::decodeKeyDbt(oc.key(), db_id, nm_id, type);
if ((id == db_id) && idInList(*toRemove, nm_id))
cursor.del(0);
} else {
db_id.setThisFromDbt(oc.key());
if (id == db_id)
cursor.del(0);
}
if (id != db_id) // done with document?
break;
err = cursor.get(oc.key(), none, DB_NEXT);
}
} catch (...) {
cursor.close();
throw;
}
if(err == DB_NOTFOUND) {
err = 0;
}
cursor.close();
return err;
}
int DocumentDatabase::updateMetaData(OperationContext &oc,
DictionaryDatabase *dictionary,
Document &document)
{
int err = 0;
MetaData::const_iterator end = document.metaDataEnd();
MetaData::const_iterator i;
std::vector<NameID> toRemove;
for(i = document.metaDataBegin(); err == 0 && i != end; ++i) {
if((*i)->isModified()) {
NameID nid;
err = dictionary->lookupIDFromName(oc,
(*i)->getName(),
nid, /*define=*/true);
if(err == 0) {
if ((*i)->isRemoved())
toRemove.push_back(nid);
else {
DbtIn value;
MetaDatum::setKeyDbt(document.getID(),
nid,
(*i)->getType(),
oc.key());
(*i)->setValueDbtFromThis(value);
// could throw on error
err = secondary_.put(oc.txn(),
&oc.key(),
&value, 0);
}
}
}
}
if (toRemove.size() > 0) {
err = removeMetaData(oc, document.getID(),
&toRemove);
}
if(err == 0)
for(i = document.metaDataBegin(); i != end; ++i)
(*i)->setModified(false);
return err;
}
int StructuralStatsDatabase::getStats(OperationContext &context, const NameID &id1, StructuralStats &stats) const
{
if(id1 == 0) return getStats(context, stats);
Cursor myCursor(const_cast<DbWrapper&>(db_), getTxn(context), CURSOR_READ, 0);
if(myCursor.error() != 0) return myCursor.error();
NameID cid1, cid2;
StructuralStats current;
// Find the node information
marshalKey(id1, 0, context.key());
int err = myCursor.get(context.key(), context.data(), DB_SET);
if(err == DB_NOTFOUND) return getStats(context, stats);
// Loop over the node information and all it's descendant information
while(err == 0) {
cid1.reset();
unmarshalKey(context.key(), cid1, cid2);
if(id1 != cid1) break;
current.reset();
current.unmarshal(context.data());
stats.add(current);
err = myCursor.get(context.key(), context.data(), DB_NEXT);
}
if(err != DB_NOTFOUND && err != 0) return err;
if(stats.sumSize_ == 0 && stats.numberOfNodes_ != 0) {
// Fill in an estimate for the missing size values that you get with DLS
stats.sumSize_ = NODE_SIZE * stats.numberOfNodes_;
stats.sumChildSize_ = NODE_SIZE * stats.sumNumberOfChildren_;
stats.sumDescendantSize_ = NODE_SIZE * stats.sumNumberOfDescendants_;
}
return 0;
}
int StructuralStatsDatabase::getStats(OperationContext &context, const NameID &id1, const NameID &id2,
StructuralStats &stats) const
{
if(id1 == 0) return getStats(context, stats);
if(id2 == 0) return getStats(context, id1, stats);
Cursor myCursor(const_cast<DbWrapper&>(db_), getTxn(context), CURSOR_READ, 0);
if(myCursor.error() != 0) return myCursor.error();
// Lookup the descendant information
marshalKey(id1, id2, context.key());
int err = myCursor.get(context.key(), context.data(), DB_SET);
if(err == DB_NOTFOUND) return getStats(context, id1, stats);
if(err != 0) return err;
stats.unmarshal(context.data());
// Lookup the node information
marshalKey(id1, 0, context.key());
err = myCursor.get(context.key(), context.data(), DB_SET);
DBXML_ASSERT(err != DB_NOTFOUND);
if(err != 0) return err;
stats.unmarshal(context.data());
if(stats.sumSize_ == 0 && stats.numberOfNodes_ != 0) {
// Fill in an estimate for the missing size values that you get with DLS
StructuralStats allStats;
err = getStats(context, id1, allStats);
if(err != 0) return err;
stats.sumSize_ = allStats.sumSize_;
stats.sumChildSize_ = allStats.sumChildSize_;
stats.sumDescendantSize_ = allStats.sumDescendantSize_;
}
return 0;
}
int DocumentDatabase::getContent(OperationContext &context,
Document *document, u_int32_t flags) const
{
DbtOut *data = new DbtOut();
int err = 0;
try {
document->getID().setDbtFromThis(context.key());
// NOTE: db_.get() does not throw on DB_NOTFOUND, but
// try/catch anyway...
err = getContent(context.txn(), context.key(), *data, flags);
} catch (...) {
delete data;
throw; // re-throw
}
if(err == 0 && (data->size != 0))
document->setContentAsDbt(&data); // Note: consumes data
else
delete data;
if (err == DB_NOTFOUND)
err = 0; // allow no-content documents
return err;
}
int PrimaryDatabase::getPrimary(OperationContext &context, const NameID &id,
DbtOut *data, u_int32_t flags) const
{
id.setDbtFromThis(context.key());
return get(context.txn(), &context.key(), data, flags);
}
double IndexDatabase::percentage(OperationContext &context, Operation operation, Operation gto, Operation lto, const Key &key1, const Key &key2) const
{
DbtOut &dbt1 = context.key();
DbtOut &dbt2 = context.data();
DB_KEY_RANGE krMin;
getMinKeyDbt(key1, dbt1);
key_range(context.txn(), &dbt1, &krMin, 0);
DB_KEY_RANGE krMax;
getMaxKeyDbt(key1, dbt1);
key_range(context.txn(), &dbt1, &krMax, 0);
// range is the % of the database keys that the keys for this index occupy.
double range = krMax.less - krMin.less;
double extent = 0.0;
if (range > 0.0) {
// extent is the % of the database keys that the keys for this index match this operation.
DB_KEY_RANGE kr1;
DB_KEY_RANGE kr2;
switch(operation) {
case DbWrapper::PREFIX: {
key1.setDbtFromThis(dbt1);
key_range(context.txn(), &dbt1, &kr1, 0);
getNextKeyDbt(key1, dbt2);
key_range(context.txn(), &dbt2, &kr2, 0);
extent = kr2.less - kr1.less;
break;
}
case DbWrapper::LTX:
case DbWrapper::LTE: {
key1.setDbtFromThis(dbt2);
key_range(context.txn(), &dbt2, &kr2, 0);
extent = kr2.less - krMin.less + (operation == DbWrapper::LTE ? kr2.equal : 0);
break;
}
case DbWrapper::GTX:
case DbWrapper::GTE: {
key1.setDbtFromThis(dbt1);
key_range(context.txn(), &dbt1, &kr1, 0);
extent = krMax.less + krMax.equal - kr1.less + (operation == DbWrapper::GTX ? kr1.equal : 0);
break;
}
case DbWrapper::RANGE: {
key1.setDbtFromThis(dbt1);
key_range(context.txn(), &dbt1, &kr1, 0);
key2.setDbtFromThis(dbt2);
key_range(context.txn(), &dbt2, &kr2, 0);
extent = kr2.less - kr1.less + (lto == DbWrapper::LTE ? kr2.equal : 0) + (gto == DbWrapper::GTX ? kr1.equal : 0);
break;
}
case DbWrapper::EQUALITY: {
key1.setDbtFromThis(dbt2);
key_range(context.txn(), &dbt2, &kr2, 0);
extent = kr2.equal;
break;
}
case DbWrapper::ALL: {
extent = range;
break;
}
default: {
break;
}
}
}
// Return a small percentage in the case of a zero range or extent -
// it's unlikely that zero is really the right answer
if(range == 0 || extent == 0) return 0.001;
// extent/range is the % of keys within this index that match this operation.
return extent / range;
}
int PrimaryDatabase::deletePrimary(OperationContext &context,
const NameID &id, u_int32_t flags)
{
id.setDbtFromThis(context.key());
return del(context.txn(), &context.key(), flags);
}
int PrimaryDatabase::putPrimary(OperationContext &context, const NameID &id,
const DbXmlDbt &data, u_int32_t flags)
{
id.setDbtFromThis(context.key());
return put(context.txn(), &context.key(), &(const_cast<DbXmlDbt&>(data)), flags);
}