void getKeys(const BSONObj& obj, BSONObjSet& keys) const { verify(_fields.size() >= 1); BSONObjSet keysToAdd; // We output keys in the same order as the fields we index. for (size_t i = 0; i < _fields.size(); ++i) { const IndexedField &field = _fields[i]; // First, we get the keys that this field adds. Either they're added literally from // the value of the field, or they're transformed if the field is geo. BSONElementSet fieldElements; // false means Don't expand the last array, duh. obj.getFieldsDotted(field.name, fieldElements, false); BSONObjSet keysForThisField; if (IndexedField::GEO == field.type) { getGeoKeys(fieldElements, &keysForThisField); } else if (IndexedField::LITERAL == field.type) { getLiteralKeys(fieldElements, &keysForThisField); } else { verify(0); } // We expect there to be _spec->_missingField() present in the keys if data is // missing. So, this should be non-empty. verify(!keysForThisField.empty()); // We take the Cartesian product of all of the keys. This requires that we have // some keys to take the Cartesian product with. If keysToAdd.empty(), we // initialize it. if (keysToAdd.empty()) { keysToAdd = keysForThisField; continue; } BSONObjSet updatedKeysToAdd; for (BSONObjSet::const_iterator it = keysToAdd.begin(); it != keysToAdd.end(); ++it) { for (BSONObjSet::const_iterator newIt = keysForThisField.begin(); newIt!= keysForThisField.end(); ++newIt) { BSONObjBuilder b; b.appendElements(*it); b.append(newIt->firstElement()); updatedKeysToAdd.insert(b.obj()); } } keysToAdd = updatedKeysToAdd; } if (keysToAdd.size() > _params.maxKeysPerInsert) { warning() << "insert of geo object generated lots of keys (" << keysToAdd.size() << ") consider creating larger buckets. obj=" << obj; } for (BSONObjSet::const_iterator it = keysToAdd.begin(); it != keysToAdd.end(); ++it) { keys.insert(*it); } }
void HashedIndexType::getKeys( const BSONObj &obj, BSONObjSet &keys ) const { string hashedFieldCopy = string( _hashedField ); const char* hashedFieldCopyPtr = hashedFieldCopy.c_str(); BSONElement fieldVal = obj.getFieldDottedOrArray( hashedFieldCopyPtr ); uassert( 16244 , "Error: hashed indexes do not currently support array values" , fieldVal.type() != Array ); if ( ! fieldVal.eoo() ) { BSONObj key = BSON( "" << makeSingleKey( fieldVal , _seed , _hashVersion ) ); keys.insert( key ); } else if (! _isSparse ) { keys.insert( _missingKey.copy() ); } }
BSONObjSet DocumentSourceLookUp::getOutputSorts() { BSONObjSet out; BSONObjSet inputSort = pSource->getOutputSorts(); std::string asPath = _as.getPath(false); for (auto&& sortObj : inputSort) { // Truncate each sortObj at the '_as' path. BSONObjBuilder outputSort; for (BSONElement fieldSort : sortObj) { if (fieldSort.fieldNameStringData() == asPath) { break; } outputSort.append(fieldSort); } BSONObj outSortObj = outputSort.obj(); if (!outSortObj.isEmpty()) { out.insert(outSortObj); } } return out; }
BSONObjSet DocumentSource::truncateSortSet(const BSONObjSet& sorts, const std::set<std::string>& fields) { BSONObjSet out = SimpleBSONObjComparator::kInstance.makeBSONObjSet(); for (auto&& sort : sorts) { BSONObjBuilder outputSort; for (auto&& key : sort) { auto keyName = key.fieldNameStringData(); bool shouldAppend = true; for (auto&& field : fields) { if (keyName == field || keyName.startsWith(field + '.')) { shouldAppend = false; break; } } if (!shouldAppend) { break; } outputSort.append(key); } BSONObj outSortObj = outputSort.obj(); if (!outSortObj.isEmpty()) { out.insert(outSortObj); } } return out; }
void S2AccessMethod::getKeys(const BSONObj& obj, BSONObjSet* keys) { BSONObjSet keysToAdd; // We output keys in the same order as the fields we index. BSONObjIterator i(_descriptor->keyPattern()); while (i.more()) { BSONElement e = i.next(); // First, we get the keys that this field adds. Either they're added literally from // the value of the field, or they're transformed if the field is geo. BSONElementSet fieldElements; // false means Don't expand the last array, duh. obj.getFieldsDotted(e.fieldName(), fieldElements, false); BSONObjSet keysForThisField; if (IndexNames::GEO_2DSPHERE == e.valuestr()) { // We can't ever return documents that don't have geometry so don't bother indexing // them. if (fieldElements.empty()) { return; } getGeoKeys(obj, fieldElements, &keysForThisField); } else { getLiteralKeys(fieldElements, &keysForThisField); } // We expect there to be the missing field element present in the keys if data is // missing. So, this should be non-empty. verify(!keysForThisField.empty()); // We take the Cartesian product of all of the keys. This requires that we have // some keys to take the Cartesian product with. If keysToAdd.empty(), we // initialize it. if (keysToAdd.empty()) { keysToAdd = keysForThisField; continue; } BSONObjSet updatedKeysToAdd; for (BSONObjSet::const_iterator it = keysToAdd.begin(); it != keysToAdd.end(); ++it) { for (BSONObjSet::const_iterator newIt = keysForThisField.begin(); newIt!= keysForThisField.end(); ++newIt) { BSONObjBuilder b; b.appendElements(*it); b.append(newIt->firstElement()); updatedKeysToAdd.insert(b.obj()); } } keysToAdd = updatedKeysToAdd; } if (keysToAdd.size() > _params.maxKeysPerInsert) { warning() << "insert of geo object generated lots of keys (" << keysToAdd.size() << ") consider creating larger buckets. obj=" << obj; } *keys = keysToAdd; }
// static Status SetFilter::set(OperationContext* opCtx, QuerySettings* querySettings, PlanCache* planCache, const string& ns, const BSONObj& cmdObj) { // indexes - required BSONElement indexesElt = cmdObj.getField("indexes"); if (indexesElt.eoo()) { return Status(ErrorCodes::BadValue, "required field indexes missing"); } if (indexesElt.type() != mongo::Array) { return Status(ErrorCodes::BadValue, "required field indexes must be an array"); } vector<BSONElement> indexesEltArray = indexesElt.Array(); if (indexesEltArray.empty()) { return Status(ErrorCodes::BadValue, "required field indexes must contain at least one index"); } BSONObjSet indexes = SimpleBSONObjComparator::kInstance.makeBSONObjSet(); stdx::unordered_set<std::string> indexNames; for (vector<BSONElement>::const_iterator i = indexesEltArray.begin(); i != indexesEltArray.end(); ++i) { const BSONElement& elt = *i; if (elt.type() == BSONType::Object) { BSONObj obj = elt.Obj(); if (obj.isEmpty()) { return Status(ErrorCodes::BadValue, "index specification cannot be empty"); } indexes.insert(obj.getOwned()); } else if (elt.type() == BSONType::String) { indexNames.insert(elt.String()); } else { return Status(ErrorCodes::BadValue, "each item in indexes must be an object or string"); } } auto statusWithCQ = PlanCacheCommand::canonicalize(opCtx, ns, cmdObj); if (!statusWithCQ.isOK()) { return statusWithCQ.getStatus(); } unique_ptr<CanonicalQuery> cq = std::move(statusWithCQ.getValue()); // Add allowed indices to query settings, overriding any previous entries. querySettings->setAllowedIndices(*cq, planCache->computeKey(*cq), indexes, indexNames); // Remove entry from plan cache. planCache->remove(*cq).transitional_ignore(); LOG(0) << "Index filter set on " << ns << " " << redact(cq->toStringShort()) << " " << indexesElt; return Status::OK(); }
void getKeys( const BSONObj &obj, BSONObjSet &keys ) const { if ( _spec._indexType.get() ) { //plugin (eg geo) _spec._indexType->getKeys( obj , keys ); return; } vector<const char*> fieldNames( _spec._fieldNames ); vector<BSONElement> fixed( _spec._fixed ); _getKeys( fieldNames , fixed , obj, keys ); if ( keys.empty() && ! _spec._sparse ) keys.insert( _spec._nullKey ); }
// Build a new BSONObj with root in it. If e is non-empty, append that to the key. Insert // the BSONObj into keys. void addKey(const string& root, const BSONElement& e, BSONObjSet& keys) const { BSONObjBuilder buf; buf.append("", root); if (e.eoo()) buf.appendNull(""); else buf.appendAs(e, ""); keys.insert(buf.obj()); }
void _add( const BSONObj& obj, const string& root , const BSONElement& e , BSONObjSet& keys ) const { BSONObjBuilder buf; buf.append( "" , root ); if ( e.eoo() ) buf.appendNull( "" ); else buf.appendAs( e , "" ); BSONObj key = buf.obj(); GEOQUADDEBUG( obj << "\n\t" << root << "\n\t" << key ); keys.insert( key ); }
BSONObjSet DocumentSource::allPrefixes(BSONObj obj) { BSONObjSet out = SimpleBSONObjComparator::kInstance.makeBSONObjSet(); BSONObj last = {}; for (auto&& field : obj) { BSONObjBuilder builder(last.objsize() + field.size()); builder.appendElements(last); builder.append(field); last = builder.obj(); out.insert(last); } return out; }
// PD_TRACE_DECLARE_FUNCTION ( SDB__IXMKEYGEN_GETKEYS, "_ixmKeyGenerator::getKeys" ) INT32 getKeys ( const BSONObj &obj, BSONObjSet &keys, BSONElement *pArrEle ) const { INT32 rc = SDB_OK ; PD_TRACE_ENTRY ( SDB__IXMKEYGEN_GETKEYS ); SDB_ASSERT( _keygen, "spec can't be NULL" ) ; SDB_ASSERT( !_keygen->_fieldNames.empty(), "can not be empty" ) ; vector<const CHAR*> fieldNames ( _keygen->_fieldNames ) ; BSONElement arrEle ; try { rc = _getKeys( fieldNames, obj, keys, &arrEle ) ; } catch ( std::exception &e ) { PD_LOG( PDERROR, "unexpected err:%s", e.what() ) ; rc = SDB_INVALIDARG ; goto error ; } if ( SDB_OK != rc ) { PD_LOG ( PDERROR, "Failed to generate key from object: %s", obj.toString().c_str() ) ; goto error ; } if ( keys.empty() ) { keys.insert ( _keygen->_undefinedKey ) ; } if ( NULL != pArrEle && !arrEle.eoo() ) { *pArrEle = arrEle ; } done : PD_TRACE_EXITRC ( SDB__IXMKEYGEN_GETKEYS, rc ); return rc ; error : goto done ; }
// PD_TRACE_DECLARE_FUNCTION ( SDB__IXMKEYGEN__GENKEYSWITHNORMALELE, "_ixmKeyGenerator::_genKeyWithNormalEle" ) INT32 _genKeyWithNormalEle( BSONElement *keyELes, UINT32 eleNum, BSONObjSet &keys ) const { PD_TRACE_ENTRY ( SDB__IXMKEYGEN__GENKEYSWITHNORMALELE ); INT32 rc = SDB_OK ; BSONObjBuilder builder ; for ( UINT32 i = 0; i < eleNum; i++ ) { BSONElement &e = keyELes[i] ; if ( e.eoo() ) { builder.appendAs( gUndefinedElt, "" ) ; } else { builder.appendAs( e, "" ) ; } } keys.insert( builder.obj() ) ; PD_TRACE_EXITRC ( SDB__IXMKEYGEN__GENKEYSWITHNORMALELE, rc ); return rc ; }
void ExpressionKeysPrivate::getS2Keys(const BSONObj& obj, const BSONObj& keyPattern, const S2IndexingParams& params, BSONObjSet* keys) { BSONObjSet keysToAdd; // Does one of our documents have a geo field? bool haveGeoField = false; // We output keys in the same order as the fields we index. BSONObjIterator i(keyPattern); while (i.more()) { BSONElement e = i.next(); // First, we get the keys that this field adds. Either they're added literally from // the value of the field, or they're transformed if the field is geo. BSONElementSet fieldElements; // false means Don't expand the last array, duh. obj.getFieldsDotted(e.fieldName(), fieldElements, false); BSONObjSet keysForThisField; if (IndexNames::GEO_2DSPHERE == e.valuestr()) { if (params.indexVersion >= S2_INDEX_VERSION_2) { // For >= V2, // geo: null, // geo: undefined // geo: [] // should all behave like there is no geo field. So we look for these cases and // throw out the field elements if we find them. if (1 == fieldElements.size()) { BSONElement elt = *fieldElements.begin(); // Get the :null and :undefined cases. if (elt.isNull() || Undefined == elt.type()) { fieldElements.clear(); } else if (elt.isABSONObj()) { // And this is the :[] case. BSONObj obj = elt.Obj(); if (0 == obj.nFields()) { fieldElements.clear(); } } } // >= V2 2dsphere indices require that at least one geo field to be present in a // document in order to index it. if (fieldElements.size() > 0) { haveGeoField = true; } } getS2GeoKeys(obj, fieldElements, params, &keysForThisField); } else { getS2LiteralKeys(fieldElements, params.collator, &keysForThisField); } // We expect there to be the missing field element present in the keys if data is // missing. So, this should be non-empty. verify(!keysForThisField.empty()); // We take the Cartesian product of all of the keys. This requires that we have // some keys to take the Cartesian product with. If keysToAdd.empty(), we // initialize it. if (keysToAdd.empty()) { keysToAdd = keysForThisField; continue; } BSONObjSet updatedKeysToAdd; for (BSONObjSet::const_iterator it = keysToAdd.begin(); it != keysToAdd.end(); ++it) { for (BSONObjSet::const_iterator newIt = keysForThisField.begin(); newIt != keysForThisField.end(); ++newIt) { BSONObjBuilder b; b.appendElements(*it); b.append(newIt->firstElement()); updatedKeysToAdd.insert(b.obj()); } } keysToAdd = updatedKeysToAdd; } // Make sure that if we're >= V2 there's at least one geo field present in the doc. if (params.indexVersion >= S2_INDEX_VERSION_2) { if (!haveGeoField) { return; } } if (keysToAdd.size() > params.maxKeysPerInsert) { warning() << "Insert of geo object generated a high number of keys." << " num keys: " << keysToAdd.size() << " obj inserted: " << obj; } *keys = keysToAdd; }
void ExpressionKeysPrivate::getS2Keys(const BSONObj& obj, const BSONObj& keyPattern, const S2IndexingParams& params, BSONObjSet* keys, MultikeyPaths* multikeyPaths) { BSONObjSet keysToAdd = SimpleBSONObjComparator::kInstance.makeBSONObjSet(); // Does one of our documents have a geo field? bool haveGeoField = false; if (multikeyPaths) { invariant(multikeyPaths->empty()); multikeyPaths->resize(keyPattern.nFields()); } size_t posInIdx = 0; // We output keys in the same order as the fields we index. for (const auto keyElem : keyPattern) { // First, we get the keys that this field adds. Either they're added literally from // the value of the field, or they're transformed if the field is geo. BSONElementSet fieldElements; const bool expandArrayOnTrailingField = false; std::set<size_t>* arrayComponents = multikeyPaths ? &(*multikeyPaths)[posInIdx] : nullptr; dps::extractAllElementsAlongPath( obj, keyElem.fieldName(), fieldElements, expandArrayOnTrailingField, arrayComponents); // Trailing array values aren't being expanded, so we still need to determine whether the // last component of the indexed path 'keyElem.fieldName()' causes the index to be multikey. // We say that it does if // (a) the last component of the indexed path ever refers to an array value (regardless of // the number of array elements) // (b) the last component of the indexed path ever refers to GeoJSON data that requires // multiple cells for its covering. bool lastPathComponentCausesIndexToBeMultikey; BSONObjSet keysForThisField = SimpleBSONObjComparator::kInstance.makeBSONObjSet(); if (IndexNames::GEO_2DSPHERE == keyElem.valuestr()) { if (params.indexVersion >= S2_INDEX_VERSION_2) { // For >= V2, // geo: null, // geo: undefined // geo: [] // should all behave like there is no geo field. So we look for these cases and // throw out the field elements if we find them. if (1 == fieldElements.size()) { BSONElement elt = *fieldElements.begin(); // Get the :null and :undefined cases. if (elt.isNull() || Undefined == elt.type()) { fieldElements.clear(); } else if (elt.isABSONObj()) { // And this is the :[] case. BSONObj obj = elt.Obj(); if (0 == obj.nFields()) { fieldElements.clear(); } } } // >= V2 2dsphere indices require that at least one geo field to be present in a // document in order to index it. if (fieldElements.size() > 0) { haveGeoField = true; } } lastPathComponentCausesIndexToBeMultikey = getS2GeoKeys(obj, fieldElements, params, &keysForThisField); } else { lastPathComponentCausesIndexToBeMultikey = getS2LiteralKeys(fieldElements, params.collator, &keysForThisField); } // We expect there to be the missing field element present in the keys if data is // missing. So, this should be non-empty. verify(!keysForThisField.empty()); if (multikeyPaths && lastPathComponentCausesIndexToBeMultikey) { const size_t pathLengthOfThisField = FieldRef{keyElem.fieldNameStringData()}.numParts(); invariant(pathLengthOfThisField > 0); (*multikeyPaths)[posInIdx].insert(pathLengthOfThisField - 1); } // We take the Cartesian product of all of the keys. This requires that we have // some keys to take the Cartesian product with. If keysToAdd.empty(), we // initialize it. if (keysToAdd.empty()) { keysToAdd = keysForThisField; ++posInIdx; continue; } BSONObjSet updatedKeysToAdd = SimpleBSONObjComparator::kInstance.makeBSONObjSet(); for (BSONObjSet::const_iterator it = keysToAdd.begin(); it != keysToAdd.end(); ++it) { for (BSONObjSet::const_iterator newIt = keysForThisField.begin(); newIt != keysForThisField.end(); ++newIt) { BSONObjBuilder b; b.appendElements(*it); b.append(newIt->firstElement()); updatedKeysToAdd.insert(b.obj()); } } keysToAdd = updatedKeysToAdd; ++posInIdx; } // Make sure that if we're >= V2 there's at least one geo field present in the doc. if (params.indexVersion >= S2_INDEX_VERSION_2) { if (!haveGeoField) { return; } } if (keysToAdd.size() > params.maxKeysPerInsert) { warning() << "Insert of geo object generated a high number of keys." << " num keys: " << keysToAdd.size() << " obj inserted: " << redact(obj); } *keys = keysToAdd; }
/** * @param fieldNames - fields to index, may be postfixes in recursive calls * @param fixed - values that have already been identified for their index fields * @param obj - object from which keys should be extracted, based on names in fieldNames * @param keys - set where index keys are written * @param numNotFound - number of index fields that have already been identified as missing * @param array - array from which keys should be extracted, based on names in fieldNames * If obj and array are both nonempty, obj will be one of the elements of array. */ void _getKeys( vector<const char*> fieldNames , vector<BSONElement> fixed , const BSONObj &obj, BSONObjSet &keys, int numNotFound = 0, const BSONObj &array = BSONObj() ) const { BSONElement arrElt; set<unsigned> arrIdxs; bool mayExpandArrayUnembedded = true; for( unsigned i = 0; i < fieldNames.size(); ++i ) { if ( *fieldNames[ i ] == '\0' ) { continue; } bool arrayNestedArray; // Extract element matching fieldName[ i ] from object xor array. BSONElement e = extractNextElement( obj, array, fieldNames[ i ], arrayNestedArray ); if ( e.eoo() ) { // if field not present, set to null fixed[ i ] = _spec._nullElt; // done expanding this field name fieldNames[ i ] = ""; numNotFound++; } else if ( e.type() == Array ) { arrIdxs.insert( i ); if ( arrElt.eoo() ) { // we only expand arrays on a single path -- track the path here arrElt = e; } else if ( e.rawdata() != arrElt.rawdata() ) { // enforce single array path here assertParallelArrays( e.fieldName(), arrElt.fieldName() ); } if ( arrayNestedArray ) { mayExpandArrayUnembedded = false; } } else { // not an array - no need for further expansion fixed[ i ] = e; } } if ( arrElt.eoo() ) { // No array, so generate a single key. if ( _spec._sparse && numNotFound == _spec._nFields ) { return; } BSONObjBuilder b(_spec._sizeTracker); for( vector< BSONElement >::iterator i = fixed.begin(); i != fixed.end(); ++i ) { b.appendAs( *i, "" ); } keys.insert( b.obj() ); } else if ( arrElt.embeddedObject().firstElement().eoo() ) { // Empty array, so set matching fields to undefined. _getKeysArrEltFixed( fieldNames, fixed, _spec._undefinedElt, keys, numNotFound, arrElt, arrIdxs, true ); } else { // Non empty array that can be expanded, so generate a key for each member. BSONObj arrObj = arrElt.embeddedObject(); BSONObjIterator i( arrObj ); while( i.more() ) { _getKeysArrEltFixed( fieldNames, fixed, i.next(), keys, numNotFound, arrElt, arrIdxs, mayExpandArrayUnembedded ); } } }
void _getKeys( vector<const char*> fieldNames , vector<BSONElement> fixed , const BSONObj &obj, BSONObjSet &keys ) const { BSONElement arrElt; unsigned arrIdx = ~0; int numNotFound = 0; for( unsigned i = 0; i < fieldNames.size(); ++i ) { if ( *fieldNames[ i ] == '\0' ) continue; BSONElement e = obj.getFieldDottedOrArray( fieldNames[ i ] ); if ( e.eoo() ) { e = _spec._nullElt; // no matching field numNotFound++; } if ( e.type() != Array ) fieldNames[ i ] = ""; // no matching field or non-array match if ( *fieldNames[ i ] == '\0' ) fixed[ i ] = e; // no need for further object expansion (though array expansion still possible) if ( e.type() == Array && arrElt.eoo() ) { // we only expand arrays on a single path -- track the path here arrIdx = i; arrElt = e; } // enforce single array path here if ( e.type() == Array && e.rawdata() != arrElt.rawdata() ) { assertParallelArrays( e.fieldName(), arrElt.fieldName() ); } } bool allFound = true; // have we found elements for all field names in the key spec? for( vector<const char*>::const_iterator i = fieldNames.begin(); i != fieldNames.end(); ++i ) { if ( **i != '\0' ) { allFound = false; break; } } if ( _spec._sparse && numNotFound == _spec._nFields ) { // we didn't find any fields // so we're not going to index this document return; } bool insertArrayNull = false; if ( allFound ) { if ( arrElt.eoo() ) { // no terminal array element to expand BSONObjBuilder b(_spec._sizeTracker); for( vector< BSONElement >::iterator i = fixed.begin(); i != fixed.end(); ++i ) b.appendAs( *i, "" ); keys.insert( b.obj() ); } else { // terminal array element to expand, so generate all keys BSONObjIterator i( arrElt.embeddedObject() ); if ( i.more() ) { while( i.more() ) { BSONObjBuilder b(_spec._sizeTracker); for( unsigned j = 0; j < fixed.size(); ++j ) { if ( j == arrIdx ) b.appendAs( i.next(), "" ); else b.appendAs( fixed[ j ], "" ); } keys.insert( b.obj() ); } } else if ( fixed.size() > 1 ) { insertArrayNull = true; } } } else { // nonterminal array element to expand, so recurse verify( !arrElt.eoo() ); BSONObjIterator i( arrElt.embeddedObject() ); if ( i.more() ) { while( i.more() ) { BSONElement e = i.next(); if ( e.type() == Object ) { _getKeys( fieldNames, fixed, e.embeddedObject(), keys ); } } } else { insertArrayNull = true; } } if ( insertArrayNull ) { // x : [] - need to insert undefined BSONObjBuilder b(_spec._sizeTracker); for( unsigned j = 0; j < fixed.size(); ++j ) { if ( j == arrIdx ) { b.appendUndefined( "" ); } else { BSONElement e = fixed[j]; if ( e.eoo() ) b.appendNull( "" ); else b.appendAs( e , "" ); } } keys.insert( b.obj() ); } }