pair<vector<BSONObj>, vector<BSONObj>> IndexAccessMethod::setDifference(const BSONObjSet& left,
const BSONObjSet& right) {
// Two iterators to traverse the two sets in sorted order.
auto leftIt = left.begin();
auto rightIt = right.begin();
vector<BSONObj> onlyLeft;
vector<BSONObj> onlyRight;
while (leftIt != left.end() && rightIt != right.end()) {
const int cmp = leftIt->woCompare(*rightIt);
if (cmp == 0) {
// 'leftIt' and 'rightIt' compare equal using woCompare(), but may not be identical,
// which should result in an index change.
if (!leftIt->binaryEqual(*rightIt)) {
onlyLeft.push_back(*leftIt);
onlyRight.push_back(*rightIt);
}
++leftIt;
++rightIt;
continue;
} else if (cmp > 0) {
onlyRight.push_back(*rightIt);
++rightIt;
} else {
onlyLeft.push_back(*leftIt);
++leftIt;
}
}
// Add the rest of 'left' to 'onlyLeft', and the rest of 'right' to 'onlyRight', if any.
onlyLeft.insert(onlyLeft.end(), leftIt, left.end());
onlyRight.insert(onlyRight.end(), rightIt, right.end());
return {std::move(onlyLeft), std::move(onlyRight)};
}
void getKeys(const BSONObj& obj, BSONObjSet& keys) const {
verify(_fields.size() >= 1);
BSONObjSet keysToAdd;
// We output keys in the same order as the fields we index.
for (size_t i = 0; i < _fields.size(); ++i) {
const IndexedField &field = _fields[i];
// First, we get the keys that this field adds. Either they're added literally from
// the value of the field, or they're transformed if the field is geo.
BSONElementSet fieldElements;
// false means Don't expand the last array, duh.
obj.getFieldsDotted(field.name, fieldElements, false);
BSONObjSet keysForThisField;
if (IndexedField::GEO == field.type) {
getGeoKeys(fieldElements, &keysForThisField);
} else if (IndexedField::LITERAL == field.type) {
getLiteralKeys(fieldElements, &keysForThisField);
} else {
verify(0);
}
// We expect there to be _spec->_missingField() present in the keys if data is
// missing. So, this should be non-empty.
verify(!keysForThisField.empty());
// We take the Cartesian product of all of the keys. This requires that we have
// some keys to take the Cartesian product with. If keysToAdd.empty(), we
// initialize it.
if (keysToAdd.empty()) {
keysToAdd = keysForThisField;
continue;
}
BSONObjSet updatedKeysToAdd;
for (BSONObjSet::const_iterator it = keysToAdd.begin(); it != keysToAdd.end();
++it) {
for (BSONObjSet::const_iterator newIt = keysForThisField.begin();
newIt!= keysForThisField.end(); ++newIt) {
BSONObjBuilder b;
b.appendElements(*it);
b.append(newIt->firstElement());
updatedKeysToAdd.insert(b.obj());
}
}
keysToAdd = updatedKeysToAdd;
}
if (keysToAdd.size() > _params.maxKeysPerInsert) {
warning() << "insert of geo object generated lots of keys (" << keysToAdd.size()
<< ") consider creating larger buckets. obj="
<< obj;
}
for (BSONObjSet::const_iterator it = keysToAdd.begin(); it != keysToAdd.end(); ++it) {
keys.insert(*it);
}
}
void S2AccessMethod::getKeys(const BSONObj& obj, BSONObjSet* keys) {
BSONObjSet keysToAdd;
// We output keys in the same order as the fields we index.
BSONObjIterator i(_descriptor->keyPattern());
while (i.more()) {
BSONElement e = i.next();
// First, we get the keys that this field adds. Either they're added literally from
// the value of the field, or they're transformed if the field is geo.
BSONElementSet fieldElements;
// false means Don't expand the last array, duh.
obj.getFieldsDotted(e.fieldName(), fieldElements, false);
BSONObjSet keysForThisField;
if (IndexNames::GEO_2DSPHERE == e.valuestr()) {
// We can't ever return documents that don't have geometry so don't bother indexing
// them.
if (fieldElements.empty()) { return; }
getGeoKeys(obj, fieldElements, &keysForThisField);
} else {
getLiteralKeys(fieldElements, &keysForThisField);
}
// We expect there to be the missing field element present in the keys if data is
// missing. So, this should be non-empty.
verify(!keysForThisField.empty());
// We take the Cartesian product of all of the keys. This requires that we have
// some keys to take the Cartesian product with. If keysToAdd.empty(), we
// initialize it.
if (keysToAdd.empty()) {
keysToAdd = keysForThisField;
continue;
}
BSONObjSet updatedKeysToAdd;
for (BSONObjSet::const_iterator it = keysToAdd.begin(); it != keysToAdd.end();
++it) {
for (BSONObjSet::const_iterator newIt = keysForThisField.begin();
newIt!= keysForThisField.end(); ++newIt) {
BSONObjBuilder b;
b.appendElements(*it);
b.append(newIt->firstElement());
updatedKeysToAdd.insert(b.obj());
}
}
keysToAdd = updatedKeysToAdd;
}
if (keysToAdd.size() > _params.maxKeysPerInsert) {
warning() << "insert of geo object generated lots of keys (" << keysToAdd.size()
<< ") consider creating larger buckets. obj="
<< obj;
}
*keys = keysToAdd;
}
Status IndexAccessMethod::BulkBuilder::insert(OperationContext* txn,
const BSONObj& obj,
const RecordId& loc,
const InsertDeleteOptions& options,
int64_t* numInserted) {
BSONObjSet keys = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
MultikeyPaths multikeyPaths;
_real->getKeys(obj, &keys, &multikeyPaths);
_everGeneratedMultipleKeys = _everGeneratedMultipleKeys || (keys.size() > 1);
if (!multikeyPaths.empty()) {
if (_indexMultikeyPaths.empty()) {
_indexMultikeyPaths = multikeyPaths;
} else {
invariant(_indexMultikeyPaths.size() == multikeyPaths.size());
for (size_t i = 0; i < multikeyPaths.size(); ++i) {
_indexMultikeyPaths[i].insert(multikeyPaths[i].begin(), multikeyPaths[i].end());
}
}
}
for (BSONObjSet::iterator it = keys.begin(); it != keys.end(); ++it) {
_sorter->add(*it, loc);
_keysInserted++;
}
if (NULL != numInserted) {
*numInserted += keys.size();
}
return Status::OK();
}
void NamespaceDetails::ColdIndexer::build() {
Lock::assertWriteLocked(_d->_ns);
if (_isSecondaryIndex) {
IndexDetails::Builder builder(*_idx);
const int indexNum = _d->idxNo(*_idx);
for (shared_ptr<Cursor> cursor(BasicCursor::make(_d));
cursor->ok(); cursor->advance()) {
BSONObj pk = cursor->currPK();
BSONObj obj = cursor->current();
BSONObjSet keys;
_idx->getKeysFromObject(obj, keys);
if (keys.size() > 1) {
_d->setIndexIsMultikey(indexNum);
}
for (BSONObjSet::const_iterator ki = keys.begin(); ki != keys.end(); ++ki) {
builder.insertPair(*ki, &pk, obj);
}
killCurrentOp.checkForInterrupt(); // uasserts if we should stop
}
builder.done();
// If the index is unique, check all adjacent keys for a duplicate.
if (_idx->unique()) {
_d->checkIndexUniqueness(*_idx);
}
}
}
/* add keys to index idxNo for a new record */
static void addKeysToIndex(const char *ns, NamespaceDetails *d, int idxNo, BSONObj& obj,
DiskLoc recordLoc, bool dupsAllowed) {
IndexDetails& idx = d->idx(idxNo);
BSONObjSet keys;
idx.getKeysFromObject(obj, keys);
if( keys.empty() )
return;
BSONObj order = idx.keyPattern();
IndexInterface& ii = idx.idxInterface();
Ordering ordering = Ordering::make(order);
int n = 0;
for ( BSONObjSet::iterator i=keys.begin(); i != keys.end(); i++ ) {
if( ++n == 2 ) {
d->setIndexIsMultikey(ns, idxNo);
}
verify( !recordLoc.isNull() );
try {
ii.bt_insert(idx.head, recordLoc, *i, ordering, dupsAllowed, idx);
}
catch (AssertionException& e) {
if( e.getCode() == 10287 && idxNo == d->nIndexes ) {
DEV log() << "info: caught key already in index on bg indexing (ok)" << endl;
continue;
}
if( !dupsAllowed ) {
// dup key exception, presumably.
throw;
}
problem() << " caught assertion addKeysToIndex " << idx.indexNamespace() << " " << obj["_id"] << endl;
}
}
}
// Return keys in l that are not in r.
// Lifted basically verbatim from elsewhere.
static void setDifference(const BSONObjSet &l, const BSONObjSet &r, vector<BSONObj*> *diff) {
// l and r must use the same ordering spec.
verify(l.key_comp().order() == r.key_comp().order());
BSONObjSet::const_iterator i = l.begin();
BSONObjSet::const_iterator j = r.begin();
while ( 1 ) {
if ( i == l.end() )
break;
while ( j != r.end() && j->woCompare( *i ) < 0 )
j++;
if ( j == r.end() || i->woCompare(*j) != 0 ) {
const BSONObj *jo = &*i;
diff->push_back( (BSONObj *) jo );
}
i++;
}
}
// Find the keys for obj, put them in the tree pointing to loc
Status BtreeBasedAccessMethod::insert(OperationContext* txn,
const BSONObj& obj,
const DiskLoc& loc,
const InsertDeleteOptions& options,
int64_t* numInserted) {
*numInserted = 0;
BSONObjSet keys;
// Delegate to the subclass.
getKeys(obj, &keys);
Status ret = Status::OK();
for (BSONObjSet::const_iterator i = keys.begin(); i != keys.end(); ++i) {
Status status = _newInterface->insert(txn, *i, loc, options.dupsAllowed);
// Everything's OK, carry on.
if (status.isOK()) {
++*numInserted;
continue;
}
// Error cases.
if (ErrorCodes::KeyTooLong == status.code()) {
// Ignore this error if we're on a secondary.
if (!txn->isPrimaryFor(_btreeState->ns())) {
continue;
}
// The user set a parameter to ignore key too long errors.
if (!failIndexKeyTooLong) {
continue;
}
}
if (ErrorCodes::UniqueIndexViolation == status.code()) {
// We ignore it for some reason in BG indexing.
if (!_btreeState->isReady()) {
DEV log() << "info: key already in index during bg indexing (ok)\n";
continue;
}
}
// Clean up after ourselves.
for (BSONObjSet::const_iterator j = keys.begin(); j != i; ++j) {
removeOneKey(txn, *j, loc);
*numInserted = 0;
}
return status;
}
if (*numInserted > 1) {
_btreeState->setMultikey( txn );
}
return ret;
}
AllowedIndicesFilter::AllowedIndicesFilter(const BSONObjSet& indexKeyPatterns,
const stdx::unordered_set<std::string>& indexNames)
: indexKeyPatterns(SimpleBSONObjComparator::kInstance.makeBSONObjSet()),
indexNames(indexNames) {
for (BSONObjSet::const_iterator i = indexKeyPatterns.begin(); i != indexKeyPatterns.end();
++i) {
const BSONObj& indexKeyPattern = *i;
this->indexKeyPatterns.insert(indexKeyPattern.getOwned());
}
}
Status BtreeBasedAccessMethod::touch(const BSONObj& obj) {
BSONObjSet keys;
getKeys(obj, &keys);
boost::scoped_ptr<BtreeInterface::Cursor> cursor(_newInterface->newCursor(1));
for (BSONObjSet::const_iterator i = keys.begin(); i != keys.end(); ++i) {
cursor->locate(*i, DiskLoc());
}
return Status::OK();
}
Status IndexAccessMethod::touch(OperationContext* txn, const BSONObj& obj) {
BSONObjSet keys;
getKeys(obj, &keys);
boost::scoped_ptr<SortedDataInterface::Cursor> cursor(_newInterface->newCursor(txn, 1));
for (BSONObjSet::const_iterator i = keys.begin(); i != keys.end(); ++i) {
cursor->locate(*i, RecordId());
}
return Status::OK();
}
Status IndexAccessMethod::touch(OperationContext* txn, const BSONObj& obj) {
BSONObjSet keys;
getKeys(obj, &keys);
std::unique_ptr<SortedDataInterface::Cursor> cursor(_newInterface->newCursor(txn));
for (BSONObjSet::const_iterator i = keys.begin(); i != keys.end(); ++i) {
cursor->seekExact(*i);
}
return Status::OK();
}
Status BtreeBasedAccessMethod::touch(const BSONObj& obj) {
BSONObjSet keys;
getKeys(obj, &keys);
DiskLoc loc;
int keyPos;
for (BSONObjSet::const_iterator i = keys.begin(); i != keys.end(); ++i) {
_newInterface->locate(*i, DiskLoc(), 1, &loc, &keyPos);
}
return Status::OK();
}
BSONObj keyTooLong(const BSONObj& a, void* data) {
BSONObj index = a[0]["index"].Obj();
BSONObj doc = a[0]["doc"].Obj();
BSONObjSet keys;
getKeysForUpgradeChecking(index, doc, &keys);
for (BSONObjSet::const_iterator key = keys.begin(); key != keys.end(); ++key) {
if (key->objsize() > 1024) {
return BSON("" << true);
}
}
return BSON("" << false);
}
TEST( FTSIndexFormat, Simple1 ) {
FTSSpec spec( FTSSpec::fixSpec( BSON( "key" << BSON( "data" << "text" ) ) ) );
BSONObjSet keys;
FTSIndexFormat::getKeys( spec, BSON( "data" << "cat sat" ), &keys );
ASSERT_EQUALS( 2U, keys.size() );
for ( BSONObjSet::const_iterator i = keys.begin(); i != keys.end(); ++i ) {
BSONObj key = *i;
ASSERT_EQUALS( 2, key.nFields() );
ASSERT_EQUALS( String, key.firstElement().type() );
}
}