예제 #1
0
파일: NsUpgrade.cpp 프로젝트: kanbang/Colt
void
NsUpgradeReader::initDocInfo()
{
    // read doc node
    // read namespace info
    OperationContext oc;
    oc.set(NULL);
    const char metadataId[] =
    { NS_PROTOCOL_VERSION_COMPAT, NS_METADATA_ID, 0 };
    DbtOut data((void *)metadataId, strlen(metadataId) + 1);
    // reproduce old NsDocumentDatabase::getNodeRecord(), but
    // cannot deadlock -- no transaction
    DBXML_ASSERT(!oc.txn());
    id_.setDbtFromThis(oc.key());
    int ret = db_.get(oc.txn(), &oc.key(), &data, DB_GET_BOTH);
    uint32_t flags = 0;
    if (ret == 0) {
        const xmlbyte_t *ptr = (const xmlbyte_t *) data.data;
        ptr += 3;  // past version and id
        size_t len;
        ptr += NsFormat::unmarshalInt(ptr, &flags);
        if (flags & NSDOC_HASDECL) {
            ptr += NsFormat::unmarshalInt(ptr, &xmlDecl_);
            DBXML_ASSERT(xmlDecl_ == 0 || xmlDecl_ == 1);
        }
        if (flags & NSDOC_HASENCODE) {
            DBXML_ASSERT(!encStr_);
            encStr_ = NsUtil::nsStringDup(ptr, &len);
            ptr += len;
        }
        if (flags & NSDOC_HASSNIFF) {
            sniffStr_ = NsUtil::nsStringDup(ptr, &len);
            ptr += len;
        }
        if (flags & NSDOC_STANDYES)
            standStr_ = _standYes;
        if (flags & NSDOC_STANDNO)
            standStr_ = _standNo;
    }
    if (flags & NSDOC_NAMESPACE) {
        nsInfo_ = new NsNamespaceInfo();
        if (!nsInfo_)
            NsUtil::nsThrowException(XmlException::NO_MEMORY_ERROR,
                                     "new failed to allocate memory",
                                     __FILE__, __LINE__);
        nsInfo_->initialize();
        const char namespaceId[] =
        { NS_PROTOCOL_VERSION_COMPAT, NS_NAMESPACE_ID, 0 };

        DbtOut ndata((void *)namespaceId,
                     strlen(namespaceId) + 1);

        id_.setDbtFromThis(oc.key());
        ret = db_.get(oc.txn(), &oc.key(), &ndata, DB_GET_BOTH);
        if (ret == 0) {
            nsInfo_->load((const char*)
                          ((const char *)ndata.data)+3);
        }
    }
}
예제 #2
0
int DocumentDatabase::reindex(const Document &document, OperationContext &oc,
		bool updateStats, bool forDelete)
{
        XmlManager mgr = document.getManager();
        ScopedContainer sc(mgr, document.getContainerID(), true);
        Container *cont = sc.getContainer();
        UpdateContext uc(mgr);
        Indexer &indexer = uc.getIndexer();
        IndexSpecification is;
        cont->getIndexSpecification(oc.txn(), is);
        indexer.resetContext(cont, &oc);
        KeyStash &ks = uc.getKeyStash();
        ks.reset();
        if (forDelete) is.set(Index::INDEXER_DELETE);
        // Index the document
	indexer.indexMetaData(is, document, ks, /*checkModified*/false);
	ScopedPtr<NsPushEventSource> source(document.
		getContentAsEventSource(oc.txn(), /*needsValidation*/false,
					indexer.getContainer()->nodesIndexed()));
	if (source.get()) {
		indexer.initIndexContent(is, document.getID(), source.get(),
			ks, updateStats, false, /*isDelete*/forDelete);
		source.get()->start();
	}
        ks.updateIndex(oc, cont);
        return 0;
}
예제 #3
0
int SyntaxDatabase::removeIndexEntries(OperationContext &context, DbWrapper &db,
				       void *buf, u_int32_t bufsize)
{
	unsigned char keybuf[10];
	// assert bufsize < 10;
	memcpy(keybuf, buf, bufsize);
	DbtIn key(keybuf, bufsize);
	key.ulen = (bufsize);
	key.dlen = (bufsize);
	key.doff = (0);
	key.set_flags(DB_DBT_PARTIAL);
	DbtIn data(0,0);
	data.set_flags(DB_DBT_PARTIAL);
	data.dlen = (0);
		
	u_int32_t flags = (context.txn() ? DB_RMW : 0);
	int err;

	// remove from index database
	Cursor cursor(db, context.txn(), CURSOR_WRITE);
	err = cursor.get(key, data, flags|DB_SET_RANGE);
	while ((err == 0) && (memcmp(key.data, buf, bufsize) == 0)) {
		cursor.del(0);
		err = cursor.get(key, data, flags|DB_NEXT);
		if (err == DB_NOTFOUND)
			break;
	}
	if (err == DB_NOTFOUND)
		err = 0;
	return err;
}
예제 #4
0
// We assume that key has the correct endianness.
int IndexDatabase::getIndexEntry(OperationContext &context, const DbXmlDbt &key, IndexEntry &ie) const
{
	u_int32_t flags = (context.txn()) ? DB_READ_COMMITTED : 0;
	int err = get(context.txn(), &(const_cast<DbXmlDbt&>(key)), &context.data(), flags);
	if(err == 0) {
		ie.setThisFromDbt(context.data());
	}
	return err;
}
예제 #5
0
// We assume that key has the correct endianness.
int IndexDatabase::putIndexEntry(OperationContext &context, const DbXmlDbt &key, const DbXmlDbt &data)
{
	int err = put(context.txn(), &(const_cast<DbXmlDbt&>(key)),
		      &(const_cast<DbXmlDbt&>(data)), DB_NODUPDATA);
	if(err == DB_KEYEXIST) err= 0; // Suppress duplicate data error, it's OK really.
	return err;
}
예제 #6
0
// We assume that key has the correct endianness.
int IndexDatabase::updateIndexEntry(OperationContext &context, const DbXmlDbt &key, const DbXmlDbt &data)
{
	Cursor cursor(*this, context.txn(), CURSOR_WRITE);
	int err = cursor.error();
	if(err == 0) {
		// We want to do a partial retrieval of 0 bytes,
		// to get the cursor in the right position
		DbXmlDbt tmp;
		tmp.data = data.data;
		tmp.size = data.size;
		tmp.doff = 0; 
		tmp.dlen = 0;
		tmp.set_flags(DB_DBT_PARTIAL);

		// Find the existing entry
		err = cursor.get(const_cast<DbXmlDbt&>(key), tmp, DB_GET_BOTH);

		if(err == 0) {
			err = cursor.del( /*no flags*/0);
		} else if(err == DB_NOTFOUND) {
			// If we can't find the index entry already, just add it
			err = 0;
		}

		if(err == 0) {
			err = cursor.put(const_cast<DbXmlDbt&>(key), const_cast<DbXmlDbt&>(data), DB_NODUPDATA);
			if(err == DB_KEYEXIST) err = 0; // Suppress duplicate data error, it's OK really.
		}
	}
	return err;
}
예제 #7
0
int SyntaxDatabase::updateStatistics(OperationContext &context, DbtIn &key, const KeyStatistics &statistics)
{
	Cursor cursor(*statistics_.get(), context.txn(), CURSOR_WRITE);
	int err = cursor.get(key, context.data(), DB_SET | (context.txn() ? DB_RMW : 0)); // could throw on error
	if (err == 0) {
		KeyStatistics existing;
		existing.setThisFromDbt(context.data());
		existing.add(statistics);
		existing.setDbtFromThis(context.data());
		err = cursor.put(key, context.data(), DB_CURRENT); // could throw on error
	} else if (err == DB_NOTFOUND) {
		statistics.setDbtFromThis(context.data());
		err = cursor.put(key, context.data(), DB_KEYFIRST); // could throw on error
	}
	return err;
}
예제 #8
0
int DocumentDatabase::getMetaData(OperationContext &context,
	DictionaryDatabase *dictionary, const Name &name,
	const DocID &did, XmlValue::Type &type, DbXmlDbt *metadata,
	u_int32_t flags) const
{
	NameID nid;
	int err = dictionary->lookupIDFromName(context, name,
					       nid, /*define=*/false);
	if(err == 0) {
		Cursor cursor(const_cast<SecondaryDatabase&>(secondary_),
			context.txn(), CURSOR_READ, "DocumentMetaData", flags);
		u_int32_t origFlags = DB_CURSOR_GET_MASK(const_cast<SecondaryDatabase&>(secondary_),flags);
		MetaDatum::setKeyDbt(did, nid, XmlValue::NONE, context.key());
		DbtIn none;
		none.set_flags(DB_DBT_PARTIAL); // Don't pull back the data.
		err = cursor.get(context.key(), none,
				 (flags | DB_SET_RANGE) & ~DB_RMW);
		if (err == 0) {
			DocID db_did;
			NameID db_nid;
			MetaDatum::decodeKeyDbt(context.key(), db_did,
						db_nid, type);
			if(db_did == did && db_nid == nid) {
				err = cursor.get(context.key(), *metadata,
						 origFlags|DB_CURRENT);
			} else {
				return DB_NOTFOUND;
			}
		}
	}

	return err;
}
예제 #9
0
int DocumentDatabase::addMetaData(OperationContext &oc,
				  DictionaryDatabase *dictionary,
				  Document &document)
{
	int err = 0;
	MetaData::const_iterator end = document.metaDataEnd();
	MetaData::const_iterator i;
	for (i = document.metaDataBegin(); err == 0 && i != end; ++i) {
		NameID nid;
		err = dictionary->lookupIDFromName(oc,
						   (*i)->getName(),
						   nid, /*define=*/true);
		if(err == 0) {
			DbtIn value;
			MetaDatum::setKeyDbt(document.getID(),
					     nid, (*i)->getType(),
					     oc.key());
			(*i)->setValueDbtFromThis(value);
			// could throw on error
			err = secondary_.put(oc.txn(), &oc.key(),
					     &value, 0);
		}
	}
	if(err == 0)
		for(i = document.metaDataBegin(); i != end; ++i)
			(*i)->setModified(false);
	return err;
}
예제 #10
0
int PrimaryDatabase::appendPrimary(OperationContext &context, NameID &id,
				   const DbXmlDbt &data, u_int32_t flags)
{
	int err = put(context.txn(), &context.key(), &(const_cast<DbXmlDbt&>(data)),
		      flags | DB_APPEND);
	if (err == 0)
		id.setThisFromDbtAsId(context.key());
	return err;
}
예제 #11
0
int DocumentDatabase::getAllMetaData(OperationContext &context,
				     DictionaryDatabase *dictionary,
				     Document *document,
				     u_int32_t flags) const
{
	int err = 0;
	u_int32_t orig_flags = flags;
	const DocID &did = document->getID();

	//
	// Read each of the meta-data items from the document secondary
	// database.  Content is no longer considered metadata
	//
	Cursor cursor(const_cast<SecondaryDatabase&>(secondary_),
		context.txn(), CURSOR_READ, "DocumentMetaData", flags);
	orig_flags = flags = DB_CURSOR_GET_MASK(const_cast<SecondaryDatabase&>(secondary_),flags);
	flags |= DB_SET_RANGE;
	bool done = false;
	while (!done) {
		did.setDbtFromThis(context.key());
		DbtIn none;
		none.set_flags(DB_DBT_PARTIAL); // Don't pull back the data.
		// if DB_RMW set, don't get locks on this get, wait for a match
		err = cursor.get(context.key(), none, flags & (~DB_RMW));
		if (err == 0) {
			DocID db_did;
			NameID db_nid;
			XmlValue::Type type;
			MetaDatum::decodeKeyDbt(context.key(), db_did,
						db_nid, type);
			if (did == db_did) {
				Name name;
				err = dictionary->lookupNameFromID(context, db_nid, name);
				if(err == 0 && !document->containsMetaData(name)) {
					DbtOut *data = new DbtOut();
					err = cursor.get(context.key(), *data,
							 DB_CURRENT);
					if(err == 0) document->setMetaData(
						name, type, &data,
						/*modified*/false);
					delete data; // If not consumed by setThis..
				}
				flags = orig_flags | DB_NEXT;
			} else {
				err = 0;
				done = true;
			}
		} else if (err == DB_NOTFOUND) {
			err = 0;
			done = true;
		} else {
			done = true;
		}
	}
	return err;
}
예제 #12
0
// We assume that key has the correct endianness.
int IndexDatabase::delIndexEntry(OperationContext &context, const DbXmlDbt &key, const DbXmlDbt &data)
{
	Cursor cursor(*this, context.txn(), CURSOR_WRITE);
	int err = cursor.error();
	if (err == 0) {
		err = cursor.get(const_cast<DbXmlDbt&>(key), const_cast<DbXmlDbt&>(data), DB_GET_BOTH);
		if (err == 0) {
			err = cursor.del( /*no flags*/0);
		}
	}
	return err;
}
예제 #13
0
// if toRemove is non-null, it specifies a list of Name IDs
// to remove; otherwise remove all metadata for the target document
int DocumentDatabase::removeMetaData(OperationContext &oc,
				     const DocID &id,
				     std::vector<NameID> *toRemove)
{
	Cursor cursor(const_cast<SecondaryDatabase&>(secondary_), oc.txn(),
		      CURSOR_WRITE, "DocumentMetaData_remove");

	DbtIn none;
	none.set_flags(DB_DBT_PARTIAL); // Don't pull back the data.

	id.setDbtFromThis(oc.key());

	DocID db_id;
	int err = cursor.get(oc.key(), none, DB_SET_RANGE);
	try {
		while(err == 0) {
			if (toRemove) {
				NameID nm_id;
				XmlValue::Type type;
				MetaDatum::decodeKeyDbt(oc.key(), db_id, nm_id, type);
				if ((id == db_id) && idInList(*toRemove, nm_id))
					cursor.del(0);
			} else {
				db_id.setThisFromDbt(oc.key());
				if (id == db_id)
					cursor.del(0);
			}
			if (id != db_id) // done with document?
				break;
			err = cursor.get(oc.key(), none, DB_NEXT);
		}
	} catch (...) {
		cursor.close();
		throw;
	}
	if(err == DB_NOTFOUND) {
		err = 0;
	}
	cursor.close();
	return err;
}
예제 #14
0
int DocumentDatabase::updateMetaData(OperationContext &oc,
				     DictionaryDatabase *dictionary,
				     Document &document)
{
	int err = 0;
	MetaData::const_iterator end = document.metaDataEnd();
	MetaData::const_iterator i;
	std::vector<NameID> toRemove;
	for(i = document.metaDataBegin(); err == 0 && i != end; ++i) {
		if((*i)->isModified()) {
			NameID nid;
			err = dictionary->lookupIDFromName(oc,
							   (*i)->getName(),
							   nid, /*define=*/true);
			if(err == 0) {
				if ((*i)->isRemoved())
					toRemove.push_back(nid);
				else {
					DbtIn value;
					MetaDatum::setKeyDbt(document.getID(),
							     nid,
							     (*i)->getType(),
							     oc.key());
					(*i)->setValueDbtFromThis(value);
					// could throw on error
					err = secondary_.put(oc.txn(),
							     &oc.key(),
							     &value, 0);
				}
			}
		}
	}
	if (toRemove.size() > 0) {
		err = removeMetaData(oc, document.getID(),
				     &toRemove);
	}
	if(err == 0)
		for(i = document.metaDataBegin(); i != end; ++i)
			(*i)->setModified(false);
	return err;
}
예제 #15
0
int DocumentDatabase::getContent(OperationContext &context,
				 Document *document, u_int32_t flags) const
{
	DbtOut *data = new DbtOut();
	int err = 0;
	try {
		document->getID().setDbtFromThis(context.key());
		// NOTE: db_.get() does not throw on DB_NOTFOUND, but
		// try/catch anyway...
		err = getContent(context.txn(), context.key(), *data, flags);
	} catch (...) {
		delete data;
		throw; // re-throw
	}
 
	if(err == 0 && (data->size != 0)) 
		document->setContentAsDbt(&data); // Note: consumes data
	else 
		delete data;
	if (err == DB_NOTFOUND)
		err = 0; // allow no-content documents
	return err;
}
예제 #16
0
int PrimaryDatabase::deletePrimary(OperationContext &context,
				   const NameID &id, u_int32_t flags)
{
	id.setDbtFromThis(context.key());
	return del(context.txn(), &context.key(), flags);
}
예제 #17
0
int PrimaryDatabase::putPrimary(OperationContext &context, const NameID &id,
				const DbXmlDbt &data, u_int32_t flags)
{
	id.setDbtFromThis(context.key());
	return put(context.txn(), &context.key(), &(const_cast<DbXmlDbt&>(data)), flags);
}
예제 #18
0
double IndexDatabase::percentage(OperationContext &context, Operation operation, Operation gto, Operation lto, const Key &key1, const Key &key2) const
{
	DbtOut &dbt1 = context.key();
	DbtOut &dbt2 = context.data();

	DB_KEY_RANGE krMin;
	getMinKeyDbt(key1, dbt1);
	key_range(context.txn(), &dbt1, &krMin, 0);

	DB_KEY_RANGE krMax;
	getMaxKeyDbt(key1, dbt1);
	key_range(context.txn(), &dbt1, &krMax, 0);

	// range is the % of the database keys that the keys for this index occupy.
	double range = krMax.less - krMin.less;
	double extent = 0.0;

	if (range > 0.0) {
		// extent is the % of the database keys that the keys for this index match this operation.
		DB_KEY_RANGE kr1;
		DB_KEY_RANGE kr2;

		switch(operation) {
		case DbWrapper::PREFIX: {
			key1.setDbtFromThis(dbt1);
			key_range(context.txn(), &dbt1, &kr1, 0);
			getNextKeyDbt(key1, dbt2);
			key_range(context.txn(), &dbt2, &kr2, 0);
			extent = kr2.less - kr1.less;
			break;
		}
		case DbWrapper::LTX:
		case DbWrapper::LTE: {
			key1.setDbtFromThis(dbt2);
			key_range(context.txn(), &dbt2, &kr2, 0);
			extent = kr2.less - krMin.less + (operation == DbWrapper::LTE ? kr2.equal : 0);
			break;
		}
		case DbWrapper::GTX:
		case DbWrapper::GTE: {
			key1.setDbtFromThis(dbt1);
			key_range(context.txn(), &dbt1, &kr1, 0);
			extent = krMax.less + krMax.equal - kr1.less + (operation == DbWrapper::GTX ? kr1.equal : 0);
			break;
		}
		case DbWrapper::RANGE: {
			key1.setDbtFromThis(dbt1);
			key_range(context.txn(), &dbt1, &kr1, 0);
			key2.setDbtFromThis(dbt2);
			key_range(context.txn(), &dbt2, &kr2, 0);
			extent = kr2.less - kr1.less + (lto == DbWrapper::LTE ? kr2.equal : 0) + (gto == DbWrapper::GTX ? kr1.equal : 0);
			break;
		}
		case DbWrapper::EQUALITY: {
			key1.setDbtFromThis(dbt2);
			key_range(context.txn(), &dbt2, &kr2, 0);
			extent = kr2.equal;
			break;
		}
		case DbWrapper::ALL: {
			extent = range;
			break;
		}
		default: {
			break;
		}
		}
	}

	// Return a small percentage in the case of a zero range or extent -
	// it's unlikely that zero is really the right answer
	if(range == 0 || extent == 0) return 0.001;

	// extent/range is the % of keys within this index that match this operation.
	return extent / range;
}
예제 #19
0
int KeyStash::updateIndex(OperationContext &context, Container *container) const
{
	statistics_.reset();

	int err = 0;

	//
	// We iterate over the key stash, adding, or deleting the keys
	// from the indexes. The key buffer comes out of the key stash
	// with the correct endianness.
	//
	Entry *entry, *otherEntry = NULL;
	EntryAction action;
	Index index;
	DbtIn key, data;
	// Add each entry to the index.
	EntrySet::const_iterator end = keys_.end();
	for(EntrySet::const_iterator i = keys_.begin();
	    i != end && err == 0; ++i) {
		entry = *i;
		if (entry->isDeleted())
			continue;
		action = entry->index.indexerAdd() ? ENTRY_ADD : ENTRY_DELETE;
		if(entry->otherEntry != 0) {
			// It's an update
			otherEntry = entry->otherEntry;
			// If this is a delete then swap, as we need the add
			if(action == ENTRY_DELETE) {
				otherEntry = entry;
				entry = entry->otherEntry;
			}
			action = ENTRY_UPDATE;
		}

		index = entry->index;
		entry->getKey(key);
		entry->getData(data);
		
		
		if (Log::isLogEnabled(Log::C_INDEXER, Log::L_DEBUG))
			logEntry(key, data, index, action, context, *container);
		bool isEqualityKey =
			(index.getKey() == Index::KEY_EQUALITY);
		SyntaxDatabase *database = container->
			getIndexDB((Syntax::Type)index.getSyntax(),
				   context.txn(), true);
		bool duplicate = true;
		switch(action) {
		case ENTRY_ADD: {
			if (isEqualityKey)
				duplicate = database->getIndexDB()->
					exists(context.txn(), key);
			// endianness of key buffer is correct
			err = database->getIndexDB()->
				putIndexEntry(context, key, data);
			break;
		}
		case ENTRY_DELETE: {
			err = database->getIndexDB()->
				delIndexEntry(context, key, data);
			if (isEqualityKey)
				duplicate = database->getIndexDB()->
					exists(context.txn(), key);
			break;
		}
		case ENTRY_UPDATE: {
			err = database->getIndexDB()->
				updateIndexEntry(context, key, data);
			break;
		}
		}
		if (err == 0) {
			if(action == ENTRY_UPDATE) {
				// other entry ends up as the delete
				DBXML_ASSERT(!(otherEntry->index.indexerAdd()));
				// We need to use both the delete and add keys
				// in the statistics if this is an update
				statistics_.addToKeyStatistics(index, key, data,
							       false);
#ifndef DBXML_DONT_DELETE_STATS
				otherEntry->getKey(key);
				otherEntry->getData(data);
				statistics_.addToKeyStatistics(otherEntry->index, key, data,
							       false);
#endif
			} else 
#ifdef DBXML_DONT_DELETE_STATS
				if (action == ENTRY_ADD)
#endif
			{
				statistics_.addToKeyStatistics(index, key, data,
					!duplicate);
			}
		}
		// eat DB_NOTFOUND errors
		if (err == DB_NOTFOUND)
			err = 0;
	}
	//
	// Update the index key statistics stored in the container.
	//
	if (err == 0) {
		err = statistics_.updateContainer(context, *container);
	}
	if (err && Log::isLogEnabled(Log::C_INDEXER, Log::L_INFO)) {
		ostringstream oss;
		oss << "Error updating indexes: " << err;
		container->log(Log::C_INDEXER, Log::L_INFO, oss);
	}
	return err;
}
예제 #20
0
int PrimaryDatabase::getPrimary(OperationContext &context, const NameID &id,
				DbtOut *data, u_int32_t flags) const
{
	id.setDbtFromThis(context.key());
	return get(context.txn(), &context.key(), data, flags);
}