void
NsUpgradeReader::initDocInfo()
{
// read doc node
// read namespace info
OperationContext oc;
oc.set(NULL);
const char metadataId[] =
{ NS_PROTOCOL_VERSION_COMPAT, NS_METADATA_ID, 0 };
DbtOut data((void *)metadataId, strlen(metadataId) + 1);
// reproduce old NsDocumentDatabase::getNodeRecord(), but
// cannot deadlock -- no transaction
DBXML_ASSERT(!oc.txn());
id_.setDbtFromThis(oc.key());
int ret = db_.get(oc.txn(), &oc.key(), &data, DB_GET_BOTH);
uint32_t flags = 0;
if (ret == 0) {
const xmlbyte_t *ptr = (const xmlbyte_t *) data.data;
ptr += 3; // past version and id
size_t len;
ptr += NsFormat::unmarshalInt(ptr, &flags);
if (flags & NSDOC_HASDECL) {
ptr += NsFormat::unmarshalInt(ptr, &xmlDecl_);
DBXML_ASSERT(xmlDecl_ == 0 || xmlDecl_ == 1);
}
if (flags & NSDOC_HASENCODE) {
DBXML_ASSERT(!encStr_);
encStr_ = NsUtil::nsStringDup(ptr, &len);
ptr += len;
}
if (flags & NSDOC_HASSNIFF) {
sniffStr_ = NsUtil::nsStringDup(ptr, &len);
ptr += len;
}
if (flags & NSDOC_STANDYES)
standStr_ = _standYes;
if (flags & NSDOC_STANDNO)
standStr_ = _standNo;
}
if (flags & NSDOC_NAMESPACE) {
nsInfo_ = new NsNamespaceInfo();
if (!nsInfo_)
NsUtil::nsThrowException(XmlException::NO_MEMORY_ERROR,
"new failed to allocate memory",
__FILE__, __LINE__);
nsInfo_->initialize();
const char namespaceId[] =
{ NS_PROTOCOL_VERSION_COMPAT, NS_NAMESPACE_ID, 0 };
DbtOut ndata((void *)namespaceId,
strlen(namespaceId) + 1);
id_.setDbtFromThis(oc.key());
ret = db_.get(oc.txn(), &oc.key(), &ndata, DB_GET_BOTH);
if (ret == 0) {
nsInfo_->load((const char*)
((const char *)ndata.data)+3);
}
}
}
int DocumentDatabase::reindex(const Document &document, OperationContext &oc,
bool updateStats, bool forDelete)
{
XmlManager mgr = document.getManager();
ScopedContainer sc(mgr, document.getContainerID(), true);
Container *cont = sc.getContainer();
UpdateContext uc(mgr);
Indexer &indexer = uc.getIndexer();
IndexSpecification is;
cont->getIndexSpecification(oc.txn(), is);
indexer.resetContext(cont, &oc);
KeyStash &ks = uc.getKeyStash();
ks.reset();
if (forDelete) is.set(Index::INDEXER_DELETE);
// Index the document
indexer.indexMetaData(is, document, ks, /*checkModified*/false);
ScopedPtr<NsPushEventSource> source(document.
getContentAsEventSource(oc.txn(), /*needsValidation*/false,
indexer.getContainer()->nodesIndexed()));
if (source.get()) {
indexer.initIndexContent(is, document.getID(), source.get(),
ks, updateStats, false, /*isDelete*/forDelete);
source.get()->start();
}
ks.updateIndex(oc, cont);
return 0;
}
int SyntaxDatabase::removeIndexEntries(OperationContext &context, DbWrapper &db,
void *buf, u_int32_t bufsize)
{
unsigned char keybuf[10];
// assert bufsize < 10;
memcpy(keybuf, buf, bufsize);
DbtIn key(keybuf, bufsize);
key.ulen = (bufsize);
key.dlen = (bufsize);
key.doff = (0);
key.set_flags(DB_DBT_PARTIAL);
DbtIn data(0,0);
data.set_flags(DB_DBT_PARTIAL);
data.dlen = (0);
u_int32_t flags = (context.txn() ? DB_RMW : 0);
int err;
// remove from index database
Cursor cursor(db, context.txn(), CURSOR_WRITE);
err = cursor.get(key, data, flags|DB_SET_RANGE);
while ((err == 0) && (memcmp(key.data, buf, bufsize) == 0)) {
cursor.del(0);
err = cursor.get(key, data, flags|DB_NEXT);
if (err == DB_NOTFOUND)
break;
}
if (err == DB_NOTFOUND)
err = 0;
return err;
}
// We assume that key has the correct endianness.
int IndexDatabase::getIndexEntry(OperationContext &context, const DbXmlDbt &key, IndexEntry &ie) const
{
u_int32_t flags = (context.txn()) ? DB_READ_COMMITTED : 0;
int err = get(context.txn(), &(const_cast<DbXmlDbt&>(key)), &context.data(), flags);
if(err == 0) {
ie.setThisFromDbt(context.data());
}
return err;
}
// We assume that key has the correct endianness.
int IndexDatabase::putIndexEntry(OperationContext &context, const DbXmlDbt &key, const DbXmlDbt &data)
{
int err = put(context.txn(), &(const_cast<DbXmlDbt&>(key)),
&(const_cast<DbXmlDbt&>(data)), DB_NODUPDATA);
if(err == DB_KEYEXIST) err= 0; // Suppress duplicate data error, it's OK really.
return err;
}
// We assume that key has the correct endianness.
int IndexDatabase::updateIndexEntry(OperationContext &context, const DbXmlDbt &key, const DbXmlDbt &data)
{
Cursor cursor(*this, context.txn(), CURSOR_WRITE);
int err = cursor.error();
if(err == 0) {
// We want to do a partial retrieval of 0 bytes,
// to get the cursor in the right position
DbXmlDbt tmp;
tmp.data = data.data;
tmp.size = data.size;
tmp.doff = 0;
tmp.dlen = 0;
tmp.set_flags(DB_DBT_PARTIAL);
// Find the existing entry
err = cursor.get(const_cast<DbXmlDbt&>(key), tmp, DB_GET_BOTH);
if(err == 0) {
err = cursor.del( /*no flags*/0);
} else if(err == DB_NOTFOUND) {
// If we can't find the index entry already, just add it
err = 0;
}
if(err == 0) {
err = cursor.put(const_cast<DbXmlDbt&>(key), const_cast<DbXmlDbt&>(data), DB_NODUPDATA);
if(err == DB_KEYEXIST) err = 0; // Suppress duplicate data error, it's OK really.
}
}
return err;
}
int SyntaxDatabase::updateStatistics(OperationContext &context, DbtIn &key, const KeyStatistics &statistics)
{
Cursor cursor(*statistics_.get(), context.txn(), CURSOR_WRITE);
int err = cursor.get(key, context.data(), DB_SET | (context.txn() ? DB_RMW : 0)); // could throw on error
if (err == 0) {
KeyStatistics existing;
existing.setThisFromDbt(context.data());
existing.add(statistics);
existing.setDbtFromThis(context.data());
err = cursor.put(key, context.data(), DB_CURRENT); // could throw on error
} else if (err == DB_NOTFOUND) {
statistics.setDbtFromThis(context.data());
err = cursor.put(key, context.data(), DB_KEYFIRST); // could throw on error
}
return err;
}
int DocumentDatabase::getMetaData(OperationContext &context,
DictionaryDatabase *dictionary, const Name &name,
const DocID &did, XmlValue::Type &type, DbXmlDbt *metadata,
u_int32_t flags) const
{
NameID nid;
int err = dictionary->lookupIDFromName(context, name,
nid, /*define=*/false);
if(err == 0) {
Cursor cursor(const_cast<SecondaryDatabase&>(secondary_),
context.txn(), CURSOR_READ, "DocumentMetaData", flags);
u_int32_t origFlags = DB_CURSOR_GET_MASK(const_cast<SecondaryDatabase&>(secondary_),flags);
MetaDatum::setKeyDbt(did, nid, XmlValue::NONE, context.key());
DbtIn none;
none.set_flags(DB_DBT_PARTIAL); // Don't pull back the data.
err = cursor.get(context.key(), none,
(flags | DB_SET_RANGE) & ~DB_RMW);
if (err == 0) {
DocID db_did;
NameID db_nid;
MetaDatum::decodeKeyDbt(context.key(), db_did,
db_nid, type);
if(db_did == did && db_nid == nid) {
err = cursor.get(context.key(), *metadata,
origFlags|DB_CURRENT);
} else {
return DB_NOTFOUND;
}
}
}
return err;
}
int DocumentDatabase::addMetaData(OperationContext &oc,
DictionaryDatabase *dictionary,
Document &document)
{
int err = 0;
MetaData::const_iterator end = document.metaDataEnd();
MetaData::const_iterator i;
for (i = document.metaDataBegin(); err == 0 && i != end; ++i) {
NameID nid;
err = dictionary->lookupIDFromName(oc,
(*i)->getName(),
nid, /*define=*/true);
if(err == 0) {
DbtIn value;
MetaDatum::setKeyDbt(document.getID(),
nid, (*i)->getType(),
oc.key());
(*i)->setValueDbtFromThis(value);
// could throw on error
err = secondary_.put(oc.txn(), &oc.key(),
&value, 0);
}
}
if(err == 0)
for(i = document.metaDataBegin(); i != end; ++i)
(*i)->setModified(false);
return err;
}
int PrimaryDatabase::appendPrimary(OperationContext &context, NameID &id,
const DbXmlDbt &data, u_int32_t flags)
{
int err = put(context.txn(), &context.key(), &(const_cast<DbXmlDbt&>(data)),
flags | DB_APPEND);
if (err == 0)
id.setThisFromDbtAsId(context.key());
return err;
}
int DocumentDatabase::getAllMetaData(OperationContext &context,
DictionaryDatabase *dictionary,
Document *document,
u_int32_t flags) const
{
int err = 0;
u_int32_t orig_flags = flags;
const DocID &did = document->getID();
//
// Read each of the meta-data items from the document secondary
// database. Content is no longer considered metadata
//
Cursor cursor(const_cast<SecondaryDatabase&>(secondary_),
context.txn(), CURSOR_READ, "DocumentMetaData", flags);
orig_flags = flags = DB_CURSOR_GET_MASK(const_cast<SecondaryDatabase&>(secondary_),flags);
flags |= DB_SET_RANGE;
bool done = false;
while (!done) {
did.setDbtFromThis(context.key());
DbtIn none;
none.set_flags(DB_DBT_PARTIAL); // Don't pull back the data.
// if DB_RMW set, don't get locks on this get, wait for a match
err = cursor.get(context.key(), none, flags & (~DB_RMW));
if (err == 0) {
DocID db_did;
NameID db_nid;
XmlValue::Type type;
MetaDatum::decodeKeyDbt(context.key(), db_did,
db_nid, type);
if (did == db_did) {
Name name;
err = dictionary->lookupNameFromID(context, db_nid, name);
if(err == 0 && !document->containsMetaData(name)) {
DbtOut *data = new DbtOut();
err = cursor.get(context.key(), *data,
DB_CURRENT);
if(err == 0) document->setMetaData(
name, type, &data,
/*modified*/false);
delete data; // If not consumed by setThis..
}
flags = orig_flags | DB_NEXT;
} else {
err = 0;
done = true;
}
} else if (err == DB_NOTFOUND) {
err = 0;
done = true;
} else {
done = true;
}
}
return err;
}
// We assume that key has the correct endianness.
int IndexDatabase::delIndexEntry(OperationContext &context, const DbXmlDbt &key, const DbXmlDbt &data)
{
Cursor cursor(*this, context.txn(), CURSOR_WRITE);
int err = cursor.error();
if (err == 0) {
err = cursor.get(const_cast<DbXmlDbt&>(key), const_cast<DbXmlDbt&>(data), DB_GET_BOTH);
if (err == 0) {
err = cursor.del( /*no flags*/0);
}
}
return err;
}
// if toRemove is non-null, it specifies a list of Name IDs
// to remove; otherwise remove all metadata for the target document
int DocumentDatabase::removeMetaData(OperationContext &oc,
const DocID &id,
std::vector<NameID> *toRemove)
{
Cursor cursor(const_cast<SecondaryDatabase&>(secondary_), oc.txn(),
CURSOR_WRITE, "DocumentMetaData_remove");
DbtIn none;
none.set_flags(DB_DBT_PARTIAL); // Don't pull back the data.
id.setDbtFromThis(oc.key());
DocID db_id;
int err = cursor.get(oc.key(), none, DB_SET_RANGE);
try {
while(err == 0) {
if (toRemove) {
NameID nm_id;
XmlValue::Type type;
MetaDatum::decodeKeyDbt(oc.key(), db_id, nm_id, type);
if ((id == db_id) && idInList(*toRemove, nm_id))
cursor.del(0);
} else {
db_id.setThisFromDbt(oc.key());
if (id == db_id)
cursor.del(0);
}
if (id != db_id) // done with document?
break;
err = cursor.get(oc.key(), none, DB_NEXT);
}
} catch (...) {
cursor.close();
throw;
}
if(err == DB_NOTFOUND) {
err = 0;
}
cursor.close();
return err;
}
int DocumentDatabase::updateMetaData(OperationContext &oc,
DictionaryDatabase *dictionary,
Document &document)
{
int err = 0;
MetaData::const_iterator end = document.metaDataEnd();
MetaData::const_iterator i;
std::vector<NameID> toRemove;
for(i = document.metaDataBegin(); err == 0 && i != end; ++i) {
if((*i)->isModified()) {
NameID nid;
err = dictionary->lookupIDFromName(oc,
(*i)->getName(),
nid, /*define=*/true);
if(err == 0) {
if ((*i)->isRemoved())
toRemove.push_back(nid);
else {
DbtIn value;
MetaDatum::setKeyDbt(document.getID(),
nid,
(*i)->getType(),
oc.key());
(*i)->setValueDbtFromThis(value);
// could throw on error
err = secondary_.put(oc.txn(),
&oc.key(),
&value, 0);
}
}
}
}
if (toRemove.size() > 0) {
err = removeMetaData(oc, document.getID(),
&toRemove);
}
if(err == 0)
for(i = document.metaDataBegin(); i != end; ++i)
(*i)->setModified(false);
return err;
}
int DocumentDatabase::getContent(OperationContext &context,
Document *document, u_int32_t flags) const
{
DbtOut *data = new DbtOut();
int err = 0;
try {
document->getID().setDbtFromThis(context.key());
// NOTE: db_.get() does not throw on DB_NOTFOUND, but
// try/catch anyway...
err = getContent(context.txn(), context.key(), *data, flags);
} catch (...) {
delete data;
throw; // re-throw
}
if(err == 0 && (data->size != 0))
document->setContentAsDbt(&data); // Note: consumes data
else
delete data;
if (err == DB_NOTFOUND)
err = 0; // allow no-content documents
return err;
}
int PrimaryDatabase::deletePrimary(OperationContext &context,
const NameID &id, u_int32_t flags)
{
id.setDbtFromThis(context.key());
return del(context.txn(), &context.key(), flags);
}
int PrimaryDatabase::putPrimary(OperationContext &context, const NameID &id,
const DbXmlDbt &data, u_int32_t flags)
{
id.setDbtFromThis(context.key());
return put(context.txn(), &context.key(), &(const_cast<DbXmlDbt&>(data)), flags);
}
double IndexDatabase::percentage(OperationContext &context, Operation operation, Operation gto, Operation lto, const Key &key1, const Key &key2) const
{
DbtOut &dbt1 = context.key();
DbtOut &dbt2 = context.data();
DB_KEY_RANGE krMin;
getMinKeyDbt(key1, dbt1);
key_range(context.txn(), &dbt1, &krMin, 0);
DB_KEY_RANGE krMax;
getMaxKeyDbt(key1, dbt1);
key_range(context.txn(), &dbt1, &krMax, 0);
// range is the % of the database keys that the keys for this index occupy.
double range = krMax.less - krMin.less;
double extent = 0.0;
if (range > 0.0) {
// extent is the % of the database keys that the keys for this index match this operation.
DB_KEY_RANGE kr1;
DB_KEY_RANGE kr2;
switch(operation) {
case DbWrapper::PREFIX: {
key1.setDbtFromThis(dbt1);
key_range(context.txn(), &dbt1, &kr1, 0);
getNextKeyDbt(key1, dbt2);
key_range(context.txn(), &dbt2, &kr2, 0);
extent = kr2.less - kr1.less;
break;
}
case DbWrapper::LTX:
case DbWrapper::LTE: {
key1.setDbtFromThis(dbt2);
key_range(context.txn(), &dbt2, &kr2, 0);
extent = kr2.less - krMin.less + (operation == DbWrapper::LTE ? kr2.equal : 0);
break;
}
case DbWrapper::GTX:
case DbWrapper::GTE: {
key1.setDbtFromThis(dbt1);
key_range(context.txn(), &dbt1, &kr1, 0);
extent = krMax.less + krMax.equal - kr1.less + (operation == DbWrapper::GTX ? kr1.equal : 0);
break;
}
case DbWrapper::RANGE: {
key1.setDbtFromThis(dbt1);
key_range(context.txn(), &dbt1, &kr1, 0);
key2.setDbtFromThis(dbt2);
key_range(context.txn(), &dbt2, &kr2, 0);
extent = kr2.less - kr1.less + (lto == DbWrapper::LTE ? kr2.equal : 0) + (gto == DbWrapper::GTX ? kr1.equal : 0);
break;
}
case DbWrapper::EQUALITY: {
key1.setDbtFromThis(dbt2);
key_range(context.txn(), &dbt2, &kr2, 0);
extent = kr2.equal;
break;
}
case DbWrapper::ALL: {
extent = range;
break;
}
default: {
break;
}
}
}
// Return a small percentage in the case of a zero range or extent -
// it's unlikely that zero is really the right answer
if(range == 0 || extent == 0) return 0.001;
// extent/range is the % of keys within this index that match this operation.
return extent / range;
}
int KeyStash::updateIndex(OperationContext &context, Container *container) const
{
statistics_.reset();
int err = 0;
//
// We iterate over the key stash, adding, or deleting the keys
// from the indexes. The key buffer comes out of the key stash
// with the correct endianness.
//
Entry *entry, *otherEntry = NULL;
EntryAction action;
Index index;
DbtIn key, data;
// Add each entry to the index.
EntrySet::const_iterator end = keys_.end();
for(EntrySet::const_iterator i = keys_.begin();
i != end && err == 0; ++i) {
entry = *i;
if (entry->isDeleted())
continue;
action = entry->index.indexerAdd() ? ENTRY_ADD : ENTRY_DELETE;
if(entry->otherEntry != 0) {
// It's an update
otherEntry = entry->otherEntry;
// If this is a delete then swap, as we need the add
if(action == ENTRY_DELETE) {
otherEntry = entry;
entry = entry->otherEntry;
}
action = ENTRY_UPDATE;
}
index = entry->index;
entry->getKey(key);
entry->getData(data);
if (Log::isLogEnabled(Log::C_INDEXER, Log::L_DEBUG))
logEntry(key, data, index, action, context, *container);
bool isEqualityKey =
(index.getKey() == Index::KEY_EQUALITY);
SyntaxDatabase *database = container->
getIndexDB((Syntax::Type)index.getSyntax(),
context.txn(), true);
bool duplicate = true;
switch(action) {
case ENTRY_ADD: {
if (isEqualityKey)
duplicate = database->getIndexDB()->
exists(context.txn(), key);
// endianness of key buffer is correct
err = database->getIndexDB()->
putIndexEntry(context, key, data);
break;
}
case ENTRY_DELETE: {
err = database->getIndexDB()->
delIndexEntry(context, key, data);
if (isEqualityKey)
duplicate = database->getIndexDB()->
exists(context.txn(), key);
break;
}
case ENTRY_UPDATE: {
err = database->getIndexDB()->
updateIndexEntry(context, key, data);
break;
}
}
if (err == 0) {
if(action == ENTRY_UPDATE) {
// other entry ends up as the delete
DBXML_ASSERT(!(otherEntry->index.indexerAdd()));
// We need to use both the delete and add keys
// in the statistics if this is an update
statistics_.addToKeyStatistics(index, key, data,
false);
#ifndef DBXML_DONT_DELETE_STATS
otherEntry->getKey(key);
otherEntry->getData(data);
statistics_.addToKeyStatistics(otherEntry->index, key, data,
false);
#endif
} else
#ifdef DBXML_DONT_DELETE_STATS
if (action == ENTRY_ADD)
#endif
{
statistics_.addToKeyStatistics(index, key, data,
!duplicate);
}
}
// eat DB_NOTFOUND errors
if (err == DB_NOTFOUND)
err = 0;
}
//
// Update the index key statistics stored in the container.
//
if (err == 0) {
err = statistics_.updateContainer(context, *container);
}
if (err && Log::isLogEnabled(Log::C_INDEXER, Log::L_INFO)) {
ostringstream oss;
oss << "Error updating indexes: " << err;
container->log(Log::C_INDEXER, Log::L_INFO, oss);
}
return err;
}
int PrimaryDatabase::getPrimary(OperationContext &context, const NameID &id,
DbtOut *data, u_int32_t flags) const
{
id.setDbtFromThis(context.key());
return get(context.txn(), &context.key(), data, flags);
}
