static bool dirFuncSetOption( directive_t *dir, dir_table_enum parm )
//*******************************************************************
{
char *str;
/* unused parameters */ (void)parm;
if( dirHasOperand( dir ) ) {
assert( dir->num_operands == 1 );
assert( dir->operand_list->type == DIROP_LINE );
str = STRING_CONTENT( dir->operand_list );
if( optionString( str, "at" ) ) {
_DirSet( AT );
} else if( optionString( str, "noat" ) ) {
_DirUnSet( AT );
} else if( optionString( str, "macro" ) ) {
_DirSet( MACRO );
} else if( optionString( str, "nomacro" ) ) {
_DirUnSet( MACRO );
} else if( optionString( str, "reorder" ) ) {
_DirSet( REORDER );
} else if( optionString( str, "noreorder" ) ) {
_DirUnSet( REORDER );
} else if( optionString( str, "volatile" ) ) {
// ignore this for now
// _DirSet( VOLATILE );
} else if( optionString( str, "novolatile" ) ) {
// ignore this for now
// _DirUnSet( VOLATILE );
} else if( optionString( str, "move" ) ) {
// ignore this for now
// _DirSet( MOVE );
} else if( optionString( str, "nomove" ) ) {
// ignore this for now
// _DirUnSet( MOVE );
} else {
Error( IMPROPER_SET_DIRECTIVE, str );
}
}
return( true );
}
int
main(int argc, char *argv[])
{
FILE *output = stderr;
char temp[80];
const char *separate_str = DEFAULT_SEPARATE_STR;
bool esc_pressed = FALSE;
int offset = 1;
int offset_add;
int retval = DLG_EXIT_OK;
int done;
int j;
eOptions code;
const Mode *modePtr;
#ifndef HAVE_COLOR
int use_shadow = FALSE; /* ignore corresponding option */
#endif
#if defined(ENABLE_NLS)
/* initialize locale support */
setlocale(LC_ALL, "");
bindtextdomain(PACKAGE, LOCALEDIR);
textdomain(PACKAGE);
#elif defined(HAVE_SETLOCALE)
(void) setlocale(LC_ALL, "");
#endif
program = argv[0];
dialog_vars.output = output;
if (argc == 2) { /* if we don't want clear screen */
switch (lookupOption(argv[1], 7)) {
case o_print_maxsize:
(void) initscr();
fprintf(output, "MaxSize: %d, %d\n", SLINES, SCOLS);
end_dialog();
break;
case o_print_version:
fprintf(output, "Version: %s\n", VERSION);
break;
default:
case o_help:
Help();
break;
}
return 0;
}
if (argc < 2) {
Help();
}
init_dialog();
#ifdef HAVE_RC_FILE
if (!strcmp(argv[1], "--create-rc")) {
if (argc != 3) {
sprintf(temp, "Expected a filename for %s", argv[1]);
Usage(temp);
}
end_dialog();
create_rc(argv[2]);
return 0;
}
#endif
while (offset < argc && !esc_pressed) {
memset(&dialog_vars, 0, sizeof(dialog_vars));
dialog_vars.aspect_ratio = DEFAULT_ASPECT_RATIO;
dialog_vars.tab_len = TAB_LEN;
dialog_vars.output = output;
done = FALSE;
while (offset < argc && !done) { /* Common options */
switch (lookupOption(argv[offset], 1)) {
case o_title:
dialog_vars.title = optionString(argv, &offset);
break;
case o_backtitle:
dialog_vars.backtitle = optionString(argv, &offset);
break;
case o_separate_widget:
separate_str = optionString(argv, &offset);
break;
case o_separate_output:
dialog_vars.separate_output = TRUE;
break;
case o_cr_wrap:
dialog_vars.cr_wrap = TRUE;
break;
case o_no_kill:
dialog_vars.cant_kill = TRUE;
break;
case o_nocancel:
dialog_vars.nocancel = TRUE;
break;
case o_size_err:
dialog_vars.size_err = TRUE;
break;
case o_beep:
dialog_vars.beep_signal = TRUE;
break;
case o_beep_after:
dialog_vars.beep_after_signal = TRUE;
break;
case o_shadow:
use_shadow = TRUE;
break;
case o_defaultno:
defaultno = TRUE;
break;
case o_default_item:
dialog_vars.default_item = optionString(argv, &offset);
break;
case o_item_help:
dialog_vars.item_help = TRUE;
break;
case o_no_shadow:
use_shadow = FALSE;
break;
case o_print_size:
dialog_vars.print_siz = TRUE;
break;
case o_print_maxsize:
fprintf(output, "MaxSize: %d, %d\n", SLINES, SCOLS);
break;
case o_print_version:
fprintf(output, "Version: %s\n", VERSION);
break;
case o_tab_correct:
dialog_vars.tab_correct = TRUE;
break;
case o_sleep:
dialog_vars.sleep_secs = optionValue(argv, &offset);
break;
case o_stderr:
dialog_vars.output = output = stderr;
break;
case o_stdout:
dialog_vars.output = output = stdout;
break;
case o_tab_len:
dialog_vars.tab_len = optionValue(argv, &offset);
break;
case o_aspect:
dialog_vars.aspect_ratio = optionValue(argv, &offset);
break;
case o_begin:
dialog_vars.begin_set = TRUE;
dialog_vars.begin_y = optionValue(argv, &offset);
dialog_vars.begin_x = optionValue(argv, &offset);
break;
case o_clear:
if (argc == 2) { /* we only want to clear the screen */
killall_bg(&retval);
(void) refresh();
end_dialog();
return 0;
}
dialog_vars.dlg_clear_screen = TRUE;
break;
case o_noitem:
case o_fullbutton:
/* ignore */
break;
default: /* no more common options */
done = TRUE;
break;
}
if (!done)
offset++;
}
for (j = 1; j < argc; j++) {
if (strncmp(argv[j - 1], "--", 2) == 0 &&
strcmp(argv[j - 1], "--backtitle") != 0 &&
strcmp(argv[j - 1], "--title") != 0) {
dlg_trim_string(argv[j]);
}
}
if (argv[offset] == NULL) {
Usage("Expected a box option");
}
if (lookupOption(argv[offset], 2) != o_checklist
&& dialog_vars.separate_output) {
sprintf(temp, "Expected --checklist, not %.20s", argv[offset]);
Usage(temp);
}
if (dialog_vars.aspect_ratio == 0)
dialog_vars.aspect_ratio = DEFAULT_ASPECT_RATIO;
put_backtitle();
/* use a table to look for the requested mode, to avoid code duplication */
modePtr = 0;
if ((code = lookupOption(argv[offset], 2)) != o_unknown)
modePtr = lookupMode(code);
if (modePtr == 0) {
sprintf(temp, "Unknown option %.20s", argv[offset]);
Usage(temp);
}
if (arg_rest(&argv[offset]) < modePtr->argmin) {
sprintf(temp, "Expected at least %d tokens for %.20s, have %d",
modePtr->argmin - 1, argv[offset],
arg_rest(&argv[offset]) - 1);
Usage(temp);
}
if (modePtr->argmax && arg_rest(&argv[offset]) > modePtr->argmax) {
sprintf(temp,
"Expected no more than %d tokens for %.20s, have %d",
modePtr->argmax - 1, argv[offset],
arg_rest(&argv[offset]) - 1);
Usage(temp);
}
retval = (*(modePtr->jumper)) (dialog_vars.title, argv + offset, &offset_add);
offset += offset_add;
if (retval == DLG_EXIT_ESC) {
esc_pressed = TRUE;
} else {
if (dialog_vars.beep_after_signal)
(void) beep();
if (dialog_vars.sleep_secs)
(void) napms(dialog_vars.sleep_secs * 1000);
if (offset < argc) {
switch (lookupOption(argv[offset], 7)) {
case o_and_widget:
(void) fputs(separate_str, output);
offset++;
break;
case o_unknown:
sprintf(temp, "Expected --and-widget, not %.20s",
argv[offset]);
Usage(temp);
break;
default:
/* if we got a cancel, etc., stop chaining */
if (retval != DLG_EXIT_OK)
esc_pressed = TRUE;
else
dialog_vars.dlg_clear_screen = TRUE;
break;
}
}
if (dialog_vars.dlg_clear_screen)
dialog_clear();
}
}
killall_bg(&retval);
(void) refresh();
end_dialog();
return retval; /* assume this is the same as exit(retval) */
}
static int
parseCommandLineOptions(int argc0, char **argv, int *compile)
{ GET_LD
int argc = argc0;
for( ; argc > 0 && (argv[0][0] == '-' || argv[0][0] == '+'); argc--, argv++ )
{ char *s = &argv[0][1];
if ( streq(s, "-" ) ) /* swipl <plargs> -- <app-args> */
{ break;
}
if ( streq(s, "tty") ) /* +/-tty */
{ if ( s[-1] == '+' )
setPrologFlagMask(PLFLAG_TTY_CONTROL);
else
clearPrologFlagMask(PLFLAG_TTY_CONTROL);
continue;
} else if ( isoption(s, "nosignals") )
{ clearPrologFlagMask(PLFLAG_SIGNALS);
continue;
} else if ( isoption(s, "nodebug") )
{ clearPrologFlagMask(PLFLAG_DEBUGINFO);
continue;
} else if ( streq(s, "-quiet") )
{ GD->options.silent = TRUE;
continue;
}
if ( *s == '-' )
{ const char *optval;
s++;
if ( (optval=is_longopt(s, "pldoc")) )
{ GD->options.pldoc_server = store_string(optval);
} else if ( is_longopt(s, "home") )
{ /* already handled */
#ifdef __WINDOWS__
} else if ( (optval=is_longopt(s, "win_app")) )
{ GD->options.win_app = TRUE;
#endif
} else if ( (optval=is_longopt(s, "traditional")) )
{ setTraditional();
}
continue; /* don't handle --long=value */
}
while(*s)
{ switch(*s)
{ case 'd': if (argc > 1)
{ prolog_debug_from_string(argv[1], TRUE);
argc--, argv++;
} else
return -1;
break;
case 'p': optionList(&GD->options.search_paths);
break;
case 'O': GD->cmdline.optimise = TRUE; /* see initFeatures() */
break;
case 'x':
case 'o': optionString(GD->options.compileOut);
break;
case 'f': optionString(GD->options.initFile);
break;
case 'F': optionString(GD->options.systemInitFile);
break;
case 'l':
case 's': optionList(&GD->options.scriptFiles);
break;
case 'g': optionString(GD->options.goal);
break;
case 't': optionString(GD->options.topLevel);
break;
case 'c': *compile = TRUE;
break;
case 'b': GD->bootsession = TRUE;
break;
case 'q': GD->options.silent = TRUE;
break;
case 'L':
case 'G':
case 'T':
case 'A':
case 'H':
{ uintptr_t size = memarea_limit(&s[1]);
if ( size == MEMAREA_INVALID_SIZE )
return -1;
switch(*s)
{ case 'L': GD->options.localSize = size; goto next;
case 'G': GD->options.globalSize = size; goto next;
case 'T': GD->options.trailSize = size; goto next;
case 'H':
case 'A':
Sdprintf("% Warning: -%csize is no longer supported\n", *s);
goto next;
}
}
}
s++;
}
next:;
}
return argc0-argc;
}
// static
void QueryPlanner::plan(const CanonicalQuery& query,
const QueryPlannerParams& params,
vector<QuerySolution*>* out) {
QLOG() << "=============================\n"
<< "Beginning planning, options = " << optionString(params.options) << endl
<< "Canonical query:\n" << query.toString() << endl
<< "============================="
<< endl;
// The shortcut formerly known as IDHACK. See if it's a simple _id query. If so we might
// just make an ixscan over the _id index and bypass the rest of planning entirely.
if (!query.getParsed().isExplain() && !query.getParsed().showDiskLoc()
&& isSimpleIdQuery(query.getParsed().getFilter())
&& !query.getParsed().hasOption(QueryOption_CursorTailable)) {
// See if we can find an _id index.
for (size_t i = 0; i < params.indices.size(); ++i) {
if (isIdIndex(params.indices[i].keyPattern)) {
const IndexEntry& index = params.indices[i];
QLOG() << "IDHACK using index " << index.toString() << endl;
// If so, we make a simple scan to find the doc.
IndexScanNode* isn = new IndexScanNode();
isn->indexKeyPattern = index.keyPattern;
isn->indexIsMultiKey = index.multikey;
isn->direction = 1;
isn->bounds.isSimpleRange = true;
BSONObj key = getKeyFromQuery(index.keyPattern, query.getParsed().getFilter());
isn->bounds.startKey = isn->bounds.endKey = key;
isn->bounds.endKeyInclusive = true;
isn->computeProperties();
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, isn);
if (NULL != soln) {
out->push_back(soln);
QLOG() << "IDHACK solution is:\n" << (*out)[0]->toString() << endl;
// And that's it.
return;
}
}
}
}
for (size_t i = 0; i < params.indices.size(); ++i) {
QLOG() << "idx " << i << " is " << params.indices[i].toString() << endl;
}
bool canTableScan = !(params.options & QueryPlannerParams::NO_TABLE_SCAN);
// If the query requests a tailable cursor, the only solution is a collscan + filter with
// tailable set on the collscan. TODO: This is a policy departure. Previously I think you
// could ask for a tailable cursor and it just tried to give you one. Now, we fail if we
// can't provide one. Is this what we want?
if (query.getParsed().hasOption(QueryOption_CursorTailable)) {
if (!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)
&& canTableScan) {
QuerySolution* soln = buildCollscanSoln(query, true, params);
if (NULL != soln) {
out->push_back(soln);
}
}
return;
}
// The hint can be $natural: 1. If this happens, output a collscan. It's a weird way of
// saying "table scan for two, please."
if (!query.getParsed().getHint().isEmpty()) {
BSONElement natural = query.getParsed().getHint().getFieldDotted("$natural");
if (!natural.eoo()) {
QLOG() << "forcing a table scan due to hinted $natural\n";
if (canTableScan) {
QuerySolution* soln = buildCollscanSoln(query, false, params);
if (NULL != soln) {
out->push_back(soln);
}
}
return;
}
}
// NOR and NOT we can't handle well with indices. If we see them here, they weren't
// rewritten to remove the negation. Just output a collscan for those.
if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::NOT)
|| QueryPlannerCommon::hasNode(query.root(), MatchExpression::NOR)) {
// If there's a near predicate, we can't handle this.
// TODO: Should canonicalized query detect this?
if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)) {
warning() << "Can't handle NOT/NOR with GEO_NEAR";
return;
}
QLOG() << "NOT/NOR in plan, just outtping a collscan\n";
if (canTableScan) {
QuerySolution* soln = buildCollscanSoln(query, false, params);
if (NULL != soln) {
out->push_back(soln);
}
}
return;
}
// Figure out what fields we care about.
unordered_set<string> fields;
QueryPlannerIXSelect::getFields(query.root(), "", &fields);
for (unordered_set<string>::const_iterator it = fields.begin(); it != fields.end(); ++it) {
QLOG() << "predicate over field " << *it << endl;
}
// Filter our indices so we only look at indices that are over our predicates.
vector<IndexEntry> relevantIndices;
// Hints require us to only consider the hinted index.
BSONObj hintIndex = query.getParsed().getHint();
// Snapshot is a form of a hint. If snapshot is set, try to use _id index to make a real
// plan. If that fails, just scan the _id index.
if (query.getParsed().isSnapshot()) {
// Find the ID index in indexKeyPatterns. It's our hint.
for (size_t i = 0; i < params.indices.size(); ++i) {
if (isIdIndex(params.indices[i].keyPattern)) {
hintIndex = params.indices[i].keyPattern;
break;
}
}
}
size_t hintIndexNumber = numeric_limits<size_t>::max();
if (!hintIndex.isEmpty()) {
// Sigh. If the hint is specified it might be using the index name.
BSONElement firstHintElt = hintIndex.firstElement();
if (str::equals("$hint", firstHintElt.fieldName()) && String == firstHintElt.type()) {
string hintName = firstHintElt.String();
for (size_t i = 0; i < params.indices.size(); ++i) {
if (params.indices[i].name == hintName) {
QLOG() << "hint by name specified, restricting indices to "
<< params.indices[i].keyPattern.toString() << endl;
relevantIndices.clear();
relevantIndices.push_back(params.indices[i]);
hintIndexNumber = i;
hintIndex = params.indices[i].keyPattern;
break;
}
}
}
else {
for (size_t i = 0; i < params.indices.size(); ++i) {
if (0 == params.indices[i].keyPattern.woCompare(hintIndex)) {
relevantIndices.clear();
relevantIndices.push_back(params.indices[i]);
QLOG() << "hint specified, restricting indices to " << hintIndex.toString()
<< endl;
hintIndexNumber = i;
break;
}
}
}
if (hintIndexNumber == numeric_limits<size_t>::max()) {
// This is supposed to be an error.
warning() << "Can't find hint for " << hintIndex.toString();
return;
}
}
else {
QLOG() << "Finding relevant indices\n";
QueryPlannerIXSelect::findRelevantIndices(fields, params.indices, &relevantIndices);
}
for (size_t i = 0; i < relevantIndices.size(); ++i) {
QLOG() << "relevant idx " << i << " is " << relevantIndices[i].toString() << endl;
}
// Figure out how useful each index is to each predicate.
// query.root() is now annotated with RelevantTag(s).
QueryPlannerIXSelect::rateIndices(query.root(), "", relevantIndices);
QLOG() << "rated tree" << endl;
QLOG() << query.root()->toString() << endl;
// If there is a GEO_NEAR it must have an index it can use directly.
// XXX: move into data access?
MatchExpression* gnNode = NULL;
if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR, &gnNode)) {
// No index for GEO_NEAR? No query.
RelevantTag* tag = static_cast<RelevantTag*>(gnNode->getTag());
if (0 == tag->first.size() && 0 == tag->notFirst.size()) {
return;
}
GeoNearMatchExpression* gnme = static_cast<GeoNearMatchExpression*>(gnNode);
vector<size_t> newFirst;
// 2d + GEO_NEAR is annoying. Because 2d's GEO_NEAR isn't streaming we have to embed
// the full query tree inside it as a matcher.
for (size_t i = 0; i < tag->first.size(); ++i) {
// GEO_NEAR has a non-2d index it can use. We can deal w/that in normal planning.
if (!is2DIndex(relevantIndices[tag->first[i]].keyPattern)) {
newFirst.push_back(i);
continue;
}
// If we're here, GEO_NEAR has a 2d index. We create a 2dgeonear plan with the
// entire tree as a filter, if possible.
GeoNear2DNode* solnRoot = new GeoNear2DNode();
solnRoot->nq = gnme->getData();
if (MatchExpression::GEO_NEAR != query.root()->matchType()) {
// root is an AND, clone and delete the GEO_NEAR child.
MatchExpression* filterTree = query.root()->shallowClone();
verify(MatchExpression::AND == filterTree->matchType());
bool foundChild = false;
for (size_t i = 0; i < filterTree->numChildren(); ++i) {
if (MatchExpression::GEO_NEAR == filterTree->getChild(i)->matchType()) {
foundChild = true;
filterTree->getChildVector()->erase(filterTree->getChildVector()->begin() + i);
break;
}
}
verify(foundChild);
solnRoot->filter.reset(filterTree);
}
solnRoot->numWanted = query.getParsed().getNumToReturn();
if (0 == solnRoot->numWanted) {
solnRoot->numWanted = 100;
}
solnRoot->indexKeyPattern = relevantIndices[tag->first[i]].keyPattern;
// Remove the 2d index. 2d can only be the first field, and we know there is
// only one GEO_NEAR, so we don't care if anyone else was assigned it; it'll
// only be first for gnNode.
tag->first.erase(tag->first.begin() + i);
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot);
if (NULL != soln) {
out->push_back(soln);
}
}
// Continue planning w/non-2d indices tagged for this pred.
tag->first.swap(newFirst);
if (0 == tag->first.size() && 0 == tag->notFirst.size()) {
return;
}
}
// Likewise, if there is a TEXT it must have an index it can use directly.
MatchExpression* textNode;
if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT, &textNode)) {
RelevantTag* tag = static_cast<RelevantTag*>(textNode->getTag());
if (0 == tag->first.size() && 0 == tag->notFirst.size()) {
return;
}
}
// If we have any relevant indices, we try to create indexed plans.
if (0 < relevantIndices.size()) {
// The enumerator spits out trees tagged with IndexTag(s).
PlanEnumerator isp(query.root(), &relevantIndices);
isp.init();
MatchExpression* rawTree;
while (isp.getNext(&rawTree)) {
QLOG() << "about to build solntree from tagged tree:\n" << rawTree->toString()
<< endl;
// This can fail if enumeration makes a mistake.
QuerySolutionNode* solnRoot =
QueryPlannerAccess::buildIndexedDataAccess(query, rawTree, false, relevantIndices);
if (NULL == solnRoot) { continue; }
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot);
if (NULL != soln) {
QLOG() << "Planner: adding solution:\n" << soln->toString() << endl;
out->push_back(soln);
}
}
}
QLOG() << "Planner: outputted " << out->size() << " indexed solutions.\n";
// An index was hinted. If there are any solutions, they use the hinted index. If not, we
// scan the entire index to provide results and output that as our plan. This is the
// desired behavior when an index is hinted that is not relevant to the query.
if (!hintIndex.isEmpty() && (0 == out->size())) {
QuerySolution* soln = buildWholeIXSoln(params.indices[hintIndexNumber], query, params);
if (NULL != soln) {
QLOG() << "Planner: outputting soln that uses hinted index as scan." << endl;
out->push_back(soln);
}
return;
}
// If a sort order is requested, there may be an index that provides it, even if that
// index is not over any predicates in the query.
//
// XXX XXX: Can we do this even if the index is sparse? Might we miss things?
if (!query.getParsed().getSort().isEmpty()
&& !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)
&& !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) {
// See if we have a sort provided from an index already.
bool usingIndexToSort = false;
for (size_t i = 0; i < out->size(); ++i) {
QuerySolution* soln = (*out)[i];
if (!soln->hasSortStage) {
usingIndexToSort = true;
break;
}
}
if (!usingIndexToSort) {
for (size_t i = 0; i < params.indices.size(); ++i) {
const BSONObj& kp = params.indices[i].keyPattern;
if (providesSort(query, kp)) {
QLOG() << "Planner: outputting soln that uses index to provide sort."
<< endl;
QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params);
if (NULL != soln) {
out->push_back(soln);
break;
}
}
if (providesSort(query, QueryPlannerCommon::reverseSortObj(kp))) {
QLOG() << "Planner: outputting soln that uses (reverse) index "
<< "to provide sort." << endl;
QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params, -1);
if (NULL != soln) {
out->push_back(soln);
break;
}
}
}
}
}
// TODO: Do we always want to offer a collscan solution?
// XXX: currently disabling the always-use-a-collscan in order to find more planner bugs.
if ( !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)
&& !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)
&& ((params.options & QueryPlannerParams::INCLUDE_COLLSCAN) || (0 == out->size() && canTableScan)))
{
QuerySolution* collscan = buildCollscanSoln(query, false, params);
if (NULL != collscan) {
out->push_back(collscan);
QLOG() << "Planner: outputting a collscan:\n";
QLOG() << collscan->toString() << endl;
}
}
}
HoI4::Event::Event(const std::string& theType, std::istream& theStream):
type(theType)
{
registerKeyword(std::regex("id"), [this](const std::string& unused, std::istream& theStream)
{
commonItems::singleString idString(theStream);
id = idString.getString();
}
);
registerKeyword(std::regex("title"), [this](const std::string& unused, std::istream& theStream)
{
commonItems::singleString titleString(theStream);
title = titleString.getString();
}
);
registerKeyword(std::regex("desc"), [this](const std::string& unused, std::istream& theStream)
{
commonItems::stringOfItem descriptionString(theStream);
descriptions.push_back("desc " + descriptionString.getString());
}
);
registerKeyword(std::regex("picture"), [this](const std::string& unused, std::istream& theStream)
{
commonItems::singleString pictureString(theStream);
picture = pictureString.getString();
}
);
registerKeyword(std::regex("major"), [this](const std::string& unused, std::istream& theStream)
{
commonItems::singleString majorString(theStream);
majorEvent = (majorString.getString() == "yes");
}
);
registerKeyword(std::regex("is_triggered_only"), [this](const std::string& unused, std::istream& theStream)
{
commonItems::singleString triggeredString(theStream);
triggeredOnly = (triggeredString.getString() == "yes");
}
);
registerKeyword(std::regex("hidden"), [this](const std::string& unused, std::istream& theStream)
{
commonItems::singleString hiddenString(theStream);
hidden = (hiddenString.getString() == "yes");
}
);
registerKeyword(std::regex("trigger"), [this](const std::string& unused, std::istream& theStream)
{
commonItems::stringOfObject triggerString(theStream);
trigger = triggerString.getString();
}
);
registerKeyword(std::regex("mean_time_to_happen"), [this](const std::string& unused, std::istream& theStream)
{
commonItems::stringOfObject MTTHString(theStream);
meanTimeToHappen = MTTHString.getString();
}
);
registerKeyword(std::regex("immediate"), [this](const std::string& unused, std::istream& theStream)
{
commonItems::stringOfObject immediateString(theStream);
immediate = immediateString.getString();
}
);
registerKeyword(std::regex("option"), [this](const std::string& unused, std::istream& theStream)
{
commonItems::stringOfObject optionString(theStream);
options.push_back(optionString.getString());
}
);
parseStream(theStream);
}
// static
Status QueryPlanner::plan(const CanonicalQuery& query,
const QueryPlannerParams& params,
std::vector<QuerySolution*>* out) {
QLOG() << "=============================\n"
<< "Beginning planning, options = " << optionString(params.options) << endl
<< "Canonical query:\n" << query.toString() << endl
<< "============================="
<< endl;
for (size_t i = 0; i < params.indices.size(); ++i) {
QLOG() << "idx " << i << " is " << params.indices[i].toString() << endl;
}
bool canTableScan = !(params.options & QueryPlannerParams::NO_TABLE_SCAN);
// If the query requests a tailable cursor, the only solution is a collscan + filter with
// tailable set on the collscan. TODO: This is a policy departure. Previously I think you
// could ask for a tailable cursor and it just tried to give you one. Now, we fail if we
// can't provide one. Is this what we want?
if (query.getParsed().hasOption(QueryOption_CursorTailable)) {
if (!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)
&& canTableScan) {
QuerySolution* soln = buildCollscanSoln(query, true, params);
if (NULL != soln) {
out->push_back(soln);
}
}
return Status::OK();
}
// The hint can be $natural: 1. If this happens, output a collscan. It's a weird way of
// saying "table scan for two, please."
if (!query.getParsed().getHint().isEmpty()) {
BSONElement natural = query.getParsed().getHint().getFieldDotted("$natural");
if (!natural.eoo()) {
QLOG() << "forcing a table scan due to hinted $natural\n";
// min/max are incompatible with $natural.
if (canTableScan && query.getParsed().getMin().isEmpty()
&& query.getParsed().getMax().isEmpty()) {
QuerySolution* soln = buildCollscanSoln(query, false, params);
if (NULL != soln) {
out->push_back(soln);
}
}
return Status::OK();
}
}
// Figure out what fields we care about.
unordered_set<string> fields;
QueryPlannerIXSelect::getFields(query.root(), "", &fields);
for (unordered_set<string>::const_iterator it = fields.begin(); it != fields.end(); ++it) {
QLOG() << "predicate over field " << *it << endl;
}
// Filter our indices so we only look at indices that are over our predicates.
vector<IndexEntry> relevantIndices;
// Hints require us to only consider the hinted index.
BSONObj hintIndex = query.getParsed().getHint();
// Snapshot is a form of a hint. If snapshot is set, try to use _id index to make a real
// plan. If that fails, just scan the _id index.
if (query.getParsed().isSnapshot()) {
// Find the ID index in indexKeyPatterns. It's our hint.
for (size_t i = 0; i < params.indices.size(); ++i) {
if (isIdIndex(params.indices[i].keyPattern)) {
hintIndex = params.indices[i].keyPattern;
break;
}
}
}
size_t hintIndexNumber = numeric_limits<size_t>::max();
if (hintIndex.isEmpty()) {
QueryPlannerIXSelect::findRelevantIndices(fields, params.indices, &relevantIndices);
}
else {
// Sigh. If the hint is specified it might be using the index name.
BSONElement firstHintElt = hintIndex.firstElement();
if (str::equals("$hint", firstHintElt.fieldName()) && String == firstHintElt.type()) {
string hintName = firstHintElt.String();
for (size_t i = 0; i < params.indices.size(); ++i) {
if (params.indices[i].name == hintName) {
QLOG() << "hint by name specified, restricting indices to "
<< params.indices[i].keyPattern.toString() << endl;
relevantIndices.clear();
relevantIndices.push_back(params.indices[i]);
hintIndexNumber = i;
hintIndex = params.indices[i].keyPattern;
break;
}
}
}
else {
for (size_t i = 0; i < params.indices.size(); ++i) {
if (0 == params.indices[i].keyPattern.woCompare(hintIndex)) {
relevantIndices.clear();
relevantIndices.push_back(params.indices[i]);
QLOG() << "hint specified, restricting indices to " << hintIndex.toString()
<< endl;
hintIndexNumber = i;
break;
}
}
}
if (hintIndexNumber == numeric_limits<size_t>::max()) {
return Status(ErrorCodes::BadValue, "bad hint");
}
}
// Deal with the .min() and .max() query options. If either exist we can only use an index
// that matches the object inside.
if (!query.getParsed().getMin().isEmpty() || !query.getParsed().getMax().isEmpty()) {
BSONObj minObj = query.getParsed().getMin();
BSONObj maxObj = query.getParsed().getMax();
// This is the index into params.indices[...] that we use.
size_t idxNo = numeric_limits<size_t>::max();
// If there's an index hinted we need to be able to use it.
if (!hintIndex.isEmpty()) {
if (!minObj.isEmpty() && !indexCompatibleMaxMin(minObj, hintIndex)) {
QLOG() << "minobj doesnt work w hint";
return Status(ErrorCodes::BadValue,
"hint provided does not work with min query");
}
if (!maxObj.isEmpty() && !indexCompatibleMaxMin(maxObj, hintIndex)) {
QLOG() << "maxobj doesnt work w hint";
return Status(ErrorCodes::BadValue,
"hint provided does not work with max query");
}
idxNo = hintIndexNumber;
}
else {
// No hinted index, look for one that is compatible (has same field names and
// ordering thereof).
for (size_t i = 0; i < params.indices.size(); ++i) {
const BSONObj& kp = params.indices[i].keyPattern;
BSONObj toUse = minObj.isEmpty() ? maxObj : minObj;
if (indexCompatibleMaxMin(toUse, kp)) {
idxNo = i;
break;
}
}
}
if (idxNo == numeric_limits<size_t>::max()) {
QLOG() << "Can't find relevant index to use for max/min query";
// Can't find an index to use, bail out.
return Status(ErrorCodes::BadValue,
"unable to find relevant index for max/min query");
}
// maxObj can be empty; the index scan just goes until the end. minObj can't be empty
// though, so if it is, we make a minKey object.
if (minObj.isEmpty()) {
BSONObjBuilder bob;
bob.appendMinKey("");
minObj = bob.obj();
}
else {
// Must strip off the field names to make an index key.
minObj = stripFieldNames(minObj);
}
if (!maxObj.isEmpty()) {
// Must strip off the field names to make an index key.
maxObj = stripFieldNames(maxObj);
}
QLOG() << "max/min query using index " << params.indices[idxNo].toString() << endl;
// Make our scan and output.
QuerySolutionNode* solnRoot = QueryPlannerAccess::makeIndexScan(params.indices[idxNo],
query,
params,
minObj,
maxObj);
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot);
if (NULL != soln) {
out->push_back(soln);
}
return Status::OK();
}
for (size_t i = 0; i < relevantIndices.size(); ++i) {
QLOG() << "relevant idx " << i << " is " << relevantIndices[i].toString() << endl;
}
// Figure out how useful each index is to each predicate.
// query.root() is now annotated with RelevantTag(s).
QueryPlannerIXSelect::rateIndices(query.root(), "", relevantIndices);
QLOG() << "rated tree" << endl;
QLOG() << query.root()->toString() << endl;
// If there is a GEO_NEAR it must have an index it can use directly.
// XXX: move into data access?
MatchExpression* gnNode = NULL;
if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR, &gnNode)) {
// No index for GEO_NEAR? No query.
RelevantTag* tag = static_cast<RelevantTag*>(gnNode->getTag());
if (0 == tag->first.size() && 0 == tag->notFirst.size()) {
QLOG() << "unable to find index for $geoNear query" << endl;
return Status(ErrorCodes::BadValue, "unable to find index for $geoNear query");
}
GeoNearMatchExpression* gnme = static_cast<GeoNearMatchExpression*>(gnNode);
vector<size_t> newFirst;
// 2d + GEO_NEAR is annoying. Because 2d's GEO_NEAR isn't streaming we have to embed
// the full query tree inside it as a matcher.
for (size_t i = 0; i < tag->first.size(); ++i) {
// GEO_NEAR has a non-2d index it can use. We can deal w/that in normal planning.
if (!is2DIndex(relevantIndices[tag->first[i]].keyPattern)) {
newFirst.push_back(i);
continue;
}
// If we're here, GEO_NEAR has a 2d index. We create a 2dgeonear plan with the
// entire tree as a filter, if possible.
GeoNear2DNode* solnRoot = new GeoNear2DNode();
solnRoot->nq = gnme->getData();
if (NULL != query.getProj()) {
solnRoot->addPointMeta = query.getProj()->wantGeoNearPoint();
solnRoot->addDistMeta = query.getProj()->wantGeoNearDistance();
}
if (MatchExpression::GEO_NEAR != query.root()->matchType()) {
// root is an AND, clone and delete the GEO_NEAR child.
MatchExpression* filterTree = query.root()->shallowClone();
verify(MatchExpression::AND == filterTree->matchType());
bool foundChild = false;
for (size_t i = 0; i < filterTree->numChildren(); ++i) {
if (MatchExpression::GEO_NEAR == filterTree->getChild(i)->matchType()) {
foundChild = true;
filterTree->getChildVector()->erase(filterTree->getChildVector()->begin() + i);
break;
}
}
verify(foundChild);
solnRoot->filter.reset(filterTree);
}
solnRoot->numWanted = query.getParsed().getNumToReturn();
if (0 == solnRoot->numWanted) {
solnRoot->numWanted = 100;
}
solnRoot->indexKeyPattern = relevantIndices[tag->first[i]].keyPattern;
// Remove the 2d index. 2d can only be the first field, and we know there is
// only one GEO_NEAR, so we don't care if anyone else was assigned it; it'll
// only be first for gnNode.
tag->first.erase(tag->first.begin() + i);
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot);
if (NULL != soln) {
out->push_back(soln);
}
}
// Continue planning w/non-2d indices tagged for this pred.
tag->first.swap(newFirst);
if (0 == tag->first.size() && 0 == tag->notFirst.size()) {
return Status::OK();
}
}
// Likewise, if there is a TEXT it must have an index it can use directly.
MatchExpression* textNode;
if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT, &textNode)) {
RelevantTag* tag = static_cast<RelevantTag*>(textNode->getTag());
if (0 == tag->first.size() && 0 == tag->notFirst.size()) {
return Status::OK();
}
}
// If we have any relevant indices, we try to create indexed plans.
if (0 < relevantIndices.size()) {
// The enumerator spits out trees tagged with IndexTag(s).
PlanEnumeratorParams enumParams;
enumParams.intersect = params.options & QueryPlannerParams::INDEX_INTERSECTION;
enumParams.root = query.root();
enumParams.indices = &relevantIndices;
PlanEnumerator isp(enumParams);
isp.init();
MatchExpression* rawTree;
// XXX: have limit on # of indexed solns we'll consider. We could have a perverse
// query and index that could make n^2 very unpleasant.
while (isp.getNext(&rawTree)) {
QLOG() << "about to build solntree from tagged tree:\n" << rawTree->toString()
<< endl;
// This can fail if enumeration makes a mistake.
QuerySolutionNode* solnRoot =
QueryPlannerAccess::buildIndexedDataAccess(query, rawTree, false, relevantIndices);
if (NULL == solnRoot) { continue; }
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot);
if (NULL != soln) {
QLOG() << "Planner: adding solution:\n" << soln->toString() << endl;
out->push_back(soln);
}
}
}
QLOG() << "Planner: outputted " << out->size() << " indexed solutions.\n";
// An index was hinted. If there are any solutions, they use the hinted index. If not, we
// scan the entire index to provide results and output that as our plan. This is the
// desired behavior when an index is hinted that is not relevant to the query.
if (!hintIndex.isEmpty()) {
if (0 == out->size()) {
QuerySolution* soln = buildWholeIXSoln(params.indices[hintIndexNumber], query, params);
verify(NULL != soln);
QLOG() << "Planner: outputting soln that uses hinted index as scan." << endl;
out->push_back(soln);
}
return Status::OK();
}
// If a sort order is requested, there may be an index that provides it, even if that
// index is not over any predicates in the query.
//
if (!query.getParsed().getSort().isEmpty()
&& !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)
&& !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) {
// See if we have a sort provided from an index already.
bool usingIndexToSort = false;
for (size_t i = 0; i < out->size(); ++i) {
QuerySolution* soln = (*out)[i];
if (!soln->hasSortStage) {
usingIndexToSort = true;
break;
}
}
if (!usingIndexToSort) {
for (size_t i = 0; i < params.indices.size(); ++i) {
const IndexEntry& index = params.indices[i];
if (index.sparse) {
continue;
}
const BSONObj kp = LiteParsedQuery::normalizeSortOrder(index.keyPattern);
if (providesSort(query, kp)) {
QLOG() << "Planner: outputting soln that uses index to provide sort."
<< endl;
QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params);
if (NULL != soln) {
out->push_back(soln);
break;
}
}
if (providesSort(query, QueryPlannerCommon::reverseSortObj(kp))) {
QLOG() << "Planner: outputting soln that uses (reverse) index "
<< "to provide sort." << endl;
QuerySolution* soln = buildWholeIXSoln(params.indices[i], query, params, -1);
if (NULL != soln) {
out->push_back(soln);
break;
}
}
}
}
}
// TODO: Do we always want to offer a collscan solution?
// XXX: currently disabling the always-use-a-collscan in order to find more planner bugs.
if ( !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)
&& !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)
&& hintIndex.isEmpty()
&& ((params.options & QueryPlannerParams::INCLUDE_COLLSCAN) || (0 == out->size() && canTableScan)))
{
QuerySolution* collscan = buildCollscanSoln(query, false, params);
if (NULL != collscan) {
out->push_back(collscan);
QLOG() << "Planner: outputting a collscan:\n";
QLOG() << collscan->toString() << endl;
}
}
return Status::OK();
}
// static
StatusWith<std::vector<std::unique_ptr<QuerySolution>>> QueryPlanner::plan(
const CanonicalQuery& query, const QueryPlannerParams& params) {
LOG(5) << "Beginning planning..." << endl
<< "=============================" << endl
<< "Options = " << optionString(params.options) << endl
<< "Canonical query:" << endl
<< redact(query.toString()) << "=============================";
std::vector<std::unique_ptr<QuerySolution>> out;
for (size_t i = 0; i < params.indices.size(); ++i) {
LOG(5) << "Index " << i << " is " << params.indices[i].toString();
}
const bool canTableScan = !(params.options & QueryPlannerParams::NO_TABLE_SCAN);
const bool isTailable = query.getQueryRequest().isTailable();
// If the query requests a tailable cursor, the only solution is a collscan + filter with
// tailable set on the collscan.
if (isTailable) {
if (!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) && canTableScan) {
auto soln = buildCollscanSoln(query, isTailable, params);
if (soln) {
out.push_back(std::move(soln));
}
}
return {std::move(out)};
}
// The hint or sort can be $natural: 1. If this happens, output a collscan. If both
// a $natural hint and a $natural sort are specified, then the direction of the collscan
// is determined by the sign of the sort (not the sign of the hint).
if (!query.getQueryRequest().getHint().isEmpty() ||
!query.getQueryRequest().getSort().isEmpty()) {
BSONObj hintObj = query.getQueryRequest().getHint();
BSONObj sortObj = query.getQueryRequest().getSort();
BSONElement naturalHint = dps::extractElementAtPath(hintObj, "$natural");
BSONElement naturalSort = dps::extractElementAtPath(sortObj, "$natural");
// A hint overrides a $natural sort. This means that we don't force a table
// scan if there is a $natural sort with a non-$natural hint.
if (!naturalHint.eoo() || (!naturalSort.eoo() && hintObj.isEmpty())) {
LOG(5) << "Forcing a table scan due to hinted $natural";
// min/max are incompatible with $natural.
if (canTableScan && query.getQueryRequest().getMin().isEmpty() &&
query.getQueryRequest().getMax().isEmpty()) {
auto soln = buildCollscanSoln(query, isTailable, params);
if (soln) {
out.push_back(std::move(soln));
}
}
return {std::move(out)};
}
}
// Figure out what fields we care about.
unordered_set<string> fields;
QueryPlannerIXSelect::getFields(query.root(), "", &fields);
for (unordered_set<string>::const_iterator it = fields.begin(); it != fields.end(); ++it) {
LOG(5) << "Predicate over field '" << *it << "'";
}
// Filter our indices so we only look at indices that are over our predicates.
vector<IndexEntry> relevantIndices;
// Hints require us to only consider the hinted index.
// If index filters in the query settings were used to override
// the allowed indices for planning, we should not use the hinted index
// requested in the query.
BSONObj hintIndex;
if (!params.indexFiltersApplied) {
hintIndex = query.getQueryRequest().getHint();
}
// If snapshot is set, default to collscanning. If the query param SNAPSHOT_USE_ID is set,
// snapshot is a form of a hint, so try to use _id index to make a real plan. If that fails,
// just scan the _id index.
//
// Don't do this if the query is a geonear or text as as text search queries must be answered
// using full text indices and geoNear queries must be answered using geospatial indices.
if (query.getQueryRequest().isSnapshot()) {
RARELY {
warning() << "The snapshot option is deprecated. See "
"http://dochub.mongodb.org/core/snapshot-deprecation";
}
if (!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR) &&
!QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) {
const bool useIXScan = params.options & QueryPlannerParams::SNAPSHOT_USE_ID;
if (!useIXScan) {
auto soln = buildCollscanSoln(query, isTailable, params);
if (soln) {
out.push_back(std::move(soln));
}
return {std::move(out)};
} else {
// Find the ID index in indexKeyPatterns. It's our hint.
for (size_t i = 0; i < params.indices.size(); ++i) {
if (isIdIndex(params.indices[i].keyPattern)) {
hintIndex = params.indices[i].keyPattern;
break;
}
}
}
}
}
static int
parseCommandLineOptions(int argc0, char **argv, int *compile)
{ int argc = argc0;
for( ; argc > 0 && (argv[0][0] == '-' || argv[0][0] == '+'); argc--, argv++ )
{ char *s = &argv[0][1];
if ( streq(s, "-" ) ) /* pl <plargs> -- <app-args> */
break;
if ( *s == '-' )
continue; /* don't handle --long=value */
if ( streq(s, "tty") ) /* +/-tty */
{ if ( s[-1] == '+' )
setFeatureMask(TTY_CONTROL_FEATURE);
else
clearFeatureMask(TTY_CONTROL_FEATURE);
continue;
}
while(*s)
{ switch(*s)
{ case 'd': if (argc > 1)
{ GD->debug_level = atoi(argv[1]);
argc--, argv++;
} else
return -1;
break;
case 'p': if (!argc) /* handled in Prolog */
return -1;
argc--, argv++;
break;
case 'O': GD->cmdline.optimise = TRUE; /* see initFeatures() */
break;
case 'x':
case 'o': optionString(GD->options.compileOut);
break;
case 'f': optionString(GD->options.initFile);
break;
case 'F': optionString(GD->options.systemInitFile);
break;
case 's': optionString(GD->options.scriptFile);
break;
case 'g': optionString(GD->options.goal);
break;
case 't': optionString(GD->options.topLevel);
break;
case 'c': *compile = TRUE;
break;
case 'b': GD->bootsession = TRUE;
break;
case 'q': GD->options.silent = TRUE;
break;
case 'L':
case 'G':
case 'T':
case 'A':
case 'H':
{ unsigned long size = memarea_limit(&s[1]);
if ( size == MEMAREA_INVALID_SIZE )
return -1;
switch(*s)
{ case 'L': GD->options.localSize = size; goto next;
case 'G': GD->options.globalSize = size; goto next;
case 'T': GD->options.trailSize = size; goto next;
case 'A': GD->options.argumentSize = size; goto next;
case 'H': GD->options.heapSize = size; goto next;
}
}
}
s++;
}
next:;
}
return argc0-argc;
}
// static
Status QueryPlanner::plan(const CanonicalQuery& query,
const QueryPlannerParams& params,
std::vector<QuerySolution*>* out) {
QLOG() << "Beginning planning..." << endl
<< "=============================" << endl
<< "Options = " << optionString(params.options) << endl
<< "Canonical query:" << endl << query.toString()
<< "=============================" << endl;
for (size_t i = 0; i < params.indices.size(); ++i) {
QLOG() << "Index " << i << " is " << params.indices[i].toString() << endl;
}
bool canTableScan = !(params.options & QueryPlannerParams::NO_TABLE_SCAN);
// If the query requests a tailable cursor, the only solution is a collscan + filter with
// tailable set on the collscan. TODO: This is a policy departure. Previously I think you
// could ask for a tailable cursor and it just tried to give you one. Now, we fail if we
// can't provide one. Is this what we want?
if (query.getParsed().hasOption(QueryOption_CursorTailable)) {
if (!QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)
&& canTableScan) {
QuerySolution* soln = buildCollscanSoln(query, true, params);
if (NULL != soln) {
out->push_back(soln);
}
}
return Status::OK();
}
// The hint or sort can be $natural: 1. If this happens, output a collscan. If both
// a $natural hint and a $natural sort are specified, then the direction of the collscan
// is determined by the sign of the sort (not the sign of the hint).
if (!query.getParsed().getHint().isEmpty() || !query.getParsed().getSort().isEmpty()) {
BSONObj hintObj = query.getParsed().getHint();
BSONObj sortObj = query.getParsed().getSort();
BSONElement naturalHint = hintObj.getFieldDotted("$natural");
BSONElement naturalSort = sortObj.getFieldDotted("$natural");
// A hint overrides a $natural sort. This means that we don't force a table
// scan if there is a $natural sort with a non-$natural hint.
if (!naturalHint.eoo() || (!naturalSort.eoo() && hintObj.isEmpty())) {
QLOG() << "Forcing a table scan due to hinted $natural\n";
// min/max are incompatible with $natural.
if (canTableScan && query.getParsed().getMin().isEmpty()
&& query.getParsed().getMax().isEmpty()) {
QuerySolution* soln = buildCollscanSoln(query, false, params);
if (NULL != soln) {
out->push_back(soln);
}
}
return Status::OK();
}
}
// Figure out what fields we care about.
unordered_set<string> fields;
QueryPlannerIXSelect::getFields(query.root(), "", &fields);
for (unordered_set<string>::const_iterator it = fields.begin(); it != fields.end(); ++it) {
QLOG() << "Predicate over field '" << *it << "'" << endl;
}
// Filter our indices so we only look at indices that are over our predicates.
vector<IndexEntry> relevantIndices;
// Hints require us to only consider the hinted index.
// If index filters in the query settings were used to override
// the allowed indices for planning, we should not use the hinted index
// requested in the query.
BSONObj hintIndex;
if (!params.indexFiltersApplied) {
hintIndex = query.getParsed().getHint();
}
// Snapshot is a form of a hint. If snapshot is set, try to use _id index to make a real
// plan. If that fails, just scan the _id index.
if (query.getParsed().isSnapshot()) {
// Find the ID index in indexKeyPatterns. It's our hint.
for (size_t i = 0; i < params.indices.size(); ++i) {
if (isIdIndex(params.indices[i].keyPattern)) {
hintIndex = params.indices[i].keyPattern;
break;
}
}
}
size_t hintIndexNumber = numeric_limits<size_t>::max();
if (hintIndex.isEmpty()) {
QueryPlannerIXSelect::findRelevantIndices(fields, params.indices, &relevantIndices);
}
else {
// Sigh. If the hint is specified it might be using the index name.
BSONElement firstHintElt = hintIndex.firstElement();
if (str::equals("$hint", firstHintElt.fieldName()) && String == firstHintElt.type()) {
string hintName = firstHintElt.String();
for (size_t i = 0; i < params.indices.size(); ++i) {
if (params.indices[i].name == hintName) {
QLOG() << "Hint by name specified, restricting indices to "
<< params.indices[i].keyPattern.toString() << endl;
relevantIndices.clear();
relevantIndices.push_back(params.indices[i]);
hintIndexNumber = i;
hintIndex = params.indices[i].keyPattern;
break;
}
}
}
else {
for (size_t i = 0; i < params.indices.size(); ++i) {
if (0 == params.indices[i].keyPattern.woCompare(hintIndex)) {
relevantIndices.clear();
relevantIndices.push_back(params.indices[i]);
QLOG() << "Hint specified, restricting indices to " << hintIndex.toString()
<< endl;
hintIndexNumber = i;
break;
}
}
}
if (hintIndexNumber == numeric_limits<size_t>::max()) {
return Status(ErrorCodes::BadValue, "bad hint");
}
}
// Deal with the .min() and .max() query options. If either exist we can only use an index
// that matches the object inside.
if (!query.getParsed().getMin().isEmpty() || !query.getParsed().getMax().isEmpty()) {
BSONObj minObj = query.getParsed().getMin();
BSONObj maxObj = query.getParsed().getMax();
// This is the index into params.indices[...] that we use.
size_t idxNo = numeric_limits<size_t>::max();
// If there's an index hinted we need to be able to use it.
if (!hintIndex.isEmpty()) {
if (!minObj.isEmpty() && !indexCompatibleMaxMin(minObj, hintIndex)) {
QLOG() << "Minobj doesn't work with hint";
return Status(ErrorCodes::BadValue,
"hint provided does not work with min query");
}
if (!maxObj.isEmpty() && !indexCompatibleMaxMin(maxObj, hintIndex)) {
QLOG() << "Maxobj doesn't work with hint";
return Status(ErrorCodes::BadValue,
"hint provided does not work with max query");
}
idxNo = hintIndexNumber;
}
else {
// No hinted index, look for one that is compatible (has same field names and
// ordering thereof).
for (size_t i = 0; i < params.indices.size(); ++i) {
const BSONObj& kp = params.indices[i].keyPattern;
BSONObj toUse = minObj.isEmpty() ? maxObj : minObj;
if (indexCompatibleMaxMin(toUse, kp)) {
idxNo = i;
break;
}
}
}
if (idxNo == numeric_limits<size_t>::max()) {
QLOG() << "Can't find relevant index to use for max/min query";
// Can't find an index to use, bail out.
return Status(ErrorCodes::BadValue,
"unable to find relevant index for max/min query");
}
// maxObj can be empty; the index scan just goes until the end. minObj can't be empty
// though, so if it is, we make a minKey object.
if (minObj.isEmpty()) {
BSONObjBuilder bob;
bob.appendMinKey("");
minObj = bob.obj();
}
else {
// Must strip off the field names to make an index key.
minObj = stripFieldNames(minObj);
}
if (!maxObj.isEmpty()) {
// Must strip off the field names to make an index key.
maxObj = stripFieldNames(maxObj);
}
QLOG() << "Max/min query using index " << params.indices[idxNo].toString() << endl;
// Make our scan and output.
QuerySolutionNode* solnRoot = QueryPlannerAccess::makeIndexScan(params.indices[idxNo],
query,
params,
minObj,
maxObj);
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query, params, solnRoot);
if (NULL != soln) {
out->push_back(soln);
}
return Status::OK();
}
for (size_t i = 0; i < relevantIndices.size(); ++i) {
QLOG() << "Relevant index " << i << " is " << relevantIndices[i].toString() << endl;
LOG(2) << "Relevant index " << i << " is " << relevantIndices[i].toString() << endl;
}
// Figure out how useful each index is to each predicate.
QueryPlannerIXSelect::rateIndices(query.root(), "", relevantIndices);
QueryPlannerIXSelect::stripInvalidAssignments(query.root(), relevantIndices);
// query.root() is now annotated with RelevantTag(s).
QLOG() << "Rated tree:" << endl << query.root()->toString();
// If there is a GEO_NEAR it must have an index it can use directly.
MatchExpression* gnNode = NULL;
if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR, &gnNode)) {
// No index for GEO_NEAR? No query.
RelevantTag* tag = static_cast<RelevantTag*>(gnNode->getTag());
if (0 == tag->first.size() && 0 == tag->notFirst.size()) {
QLOG() << "Unable to find index for $geoNear query." << endl;
// Don't leave tags on query tree.
query.root()->resetTag();
return Status(ErrorCodes::BadValue, "unable to find index for $geoNear query");
}
QLOG() << "Rated tree after geonear processing:" << query.root()->toString();
}
// Likewise, if there is a TEXT it must have an index it can use directly.
MatchExpression* textNode = NULL;
if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT, &textNode)) {
RelevantTag* tag = static_cast<RelevantTag*>(textNode->getTag());
// Exactly one text index required for TEXT. We need to check this explicitly because
// the text stage can't be built if no text index exists or there is an ambiguity as to
// which one to use.
size_t textIndexCount = 0;
for (size_t i = 0; i < params.indices.size(); i++) {
if (INDEX_TEXT == params.indices[i].type) {
textIndexCount++;
}
}
if (textIndexCount != 1) {
// Don't leave tags on query tree.
query.root()->resetTag();
return Status(ErrorCodes::BadValue, "need exactly one text index for $text query");
}
// Error if the text node is tagged with zero indices.
if (0 == tag->first.size() && 0 == tag->notFirst.size()) {
// Don't leave tags on query tree.
query.root()->resetTag();
return Status(ErrorCodes::BadValue,
"failed to use text index to satisfy $text query (if text index is "
"compound, are equality predicates given for all prefix fields?)");
}
// At this point, we know that there is only one text index and that the TEXT node is
// assigned to it.
invariant(1 == tag->first.size() + tag->notFirst.size());
QLOG() << "Rated tree after text processing:" << query.root()->toString();
}
// If we have any relevant indices, we try to create indexed plans.
if (0 < relevantIndices.size()) {
// The enumerator spits out trees tagged with IndexTag(s).
PlanEnumeratorParams enumParams;
enumParams.intersect = params.options & QueryPlannerParams::INDEX_INTERSECTION;
enumParams.root = query.root();
enumParams.indices = &relevantIndices;
PlanEnumerator isp(enumParams);
isp.init();
MatchExpression* rawTree;
while (isp.getNext(&rawTree) && (out->size() < params.maxIndexedSolutions)) {
QLOG() << "About to build solntree from tagged tree:" << endl
<< rawTree->toString();
// The tagged tree produced by the plan enumerator is not guaranteed
// to be canonically sorted. In order to be compatible with the cached
// data, sort the tagged tree according to CanonicalQuery ordering.
boost::scoped_ptr<MatchExpression> clone(rawTree->shallowClone());
CanonicalQuery::sortTree(clone.get());
PlanCacheIndexTree* cacheData;
Status indexTreeStatus = cacheDataFromTaggedTree(clone.get(), relevantIndices, &cacheData);
if (!indexTreeStatus.isOK()) {
QLOG() << "Query is not cachable: " << indexTreeStatus.reason() << endl;
}
auto_ptr<PlanCacheIndexTree> autoData(cacheData);
// This can fail if enumeration makes a mistake.
QuerySolutionNode* solnRoot =
QueryPlannerAccess::buildIndexedDataAccess(query, rawTree, false, relevantIndices);
if (NULL == solnRoot) { continue; }
QuerySolution* soln = QueryPlannerAnalysis::analyzeDataAccess(query,
params,
solnRoot);
if (NULL != soln) {
QLOG() << "Planner: adding solution:" << endl << soln->toString();
if (indexTreeStatus.isOK()) {
SolutionCacheData* scd = new SolutionCacheData();
scd->tree.reset(autoData.release());
soln->cacheData.reset(scd);
}
out->push_back(soln);
}
}
}
// Don't leave tags on query tree.
query.root()->resetTag();
QLOG() << "Planner: outputted " << out->size() << " indexed solutions.\n";
// Produce legible error message for failed OR planning with a TEXT child.
// TODO: support collection scan for non-TEXT children of OR.
if (out->size() == 0 && textNode != NULL &&
MatchExpression::OR == query.root()->matchType()) {
MatchExpression* root = query.root();
for (size_t i = 0; i < root->numChildren(); ++i) {
if (textNode == root->getChild(i)) {
return Status(ErrorCodes::BadValue,
"Failed to produce a solution for TEXT under OR - "
"other non-TEXT clauses under OR have to be indexed as well.");
}
}
}
// An index was hinted. If there are any solutions, they use the hinted index. If not, we
// scan the entire index to provide results and output that as our plan. This is the
// desired behavior when an index is hinted that is not relevant to the query.
if (!hintIndex.isEmpty()) {
if (0 == out->size()) {
QuerySolution* soln = buildWholeIXSoln(params.indices[hintIndexNumber],
query, params);
verify(NULL != soln);
QLOG() << "Planner: outputting soln that uses hinted index as scan." << endl;
out->push_back(soln);
}
return Status::OK();
}
// If a sort order is requested, there may be an index that provides it, even if that
// index is not over any predicates in the query.
//
if (!query.getParsed().getSort().isEmpty()
&& !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)
&& !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)) {
// See if we have a sort provided from an index already.
// This is implied by the presence of a non-blocking solution.
bool usingIndexToSort = false;
for (size_t i = 0; i < out->size(); ++i) {
QuerySolution* soln = (*out)[i];
if (!soln->hasBlockingStage) {
usingIndexToSort = true;
break;
}
}
if (!usingIndexToSort) {
for (size_t i = 0; i < params.indices.size(); ++i) {
const IndexEntry& index = params.indices[i];
// Only regular (non-plugin) indexes can be used to provide a sort.
if (index.type != INDEX_BTREE) {
continue;
}
// Only non-sparse indexes can be used to provide a sort.
if (index.sparse) {
continue;
}
// TODO: Sparse indexes can't normally provide a sort, because non-indexed
// documents could potentially be missing from the result set. However, if the
// query predicate can be used to guarantee that all documents to be returned
// are indexed, then the index should be able to provide the sort.
//
// For example:
// - Sparse index {a: 1, b: 1} should be able to provide a sort for
// find({b: 1}).sort({a: 1}). SERVER-13908.
// - Index {a: 1, b: "2dsphere"} (which is "geo-sparse", if
// 2dsphereIndexVersion=2) should be able to provide a sort for
// find({b: GEO}).sort({a:1}). SERVER-10801.
const BSONObj kp = LiteParsedQuery::normalizeSortOrder(index.keyPattern);
if (providesSort(query, kp)) {
QLOG() << "Planner: outputting soln that uses index to provide sort."
<< endl;
QuerySolution* soln = buildWholeIXSoln(params.indices[i],
query, params);
if (NULL != soln) {
PlanCacheIndexTree* indexTree = new PlanCacheIndexTree();
indexTree->setIndexEntry(params.indices[i]);
SolutionCacheData* scd = new SolutionCacheData();
scd->tree.reset(indexTree);
scd->solnType = SolutionCacheData::WHOLE_IXSCAN_SOLN;
scd->wholeIXSolnDir = 1;
soln->cacheData.reset(scd);
out->push_back(soln);
break;
}
}
if (providesSort(query, QueryPlannerCommon::reverseSortObj(kp))) {
QLOG() << "Planner: outputting soln that uses (reverse) index "
<< "to provide sort." << endl;
QuerySolution* soln = buildWholeIXSoln(params.indices[i], query,
params, -1);
if (NULL != soln) {
PlanCacheIndexTree* indexTree = new PlanCacheIndexTree();
indexTree->setIndexEntry(params.indices[i]);
SolutionCacheData* scd = new SolutionCacheData();
scd->tree.reset(indexTree);
scd->solnType = SolutionCacheData::WHOLE_IXSCAN_SOLN;
scd->wholeIXSolnDir = -1;
soln->cacheData.reset(scd);
out->push_back(soln);
break;
}
}
}
}
}
// geoNear and text queries *require* an index.
// Also, if a hint is specified it indicates that we MUST use it.
bool possibleToCollscan = !QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR)
&& !QueryPlannerCommon::hasNode(query.root(), MatchExpression::TEXT)
&& hintIndex.isEmpty();
// The caller can explicitly ask for a collscan.
bool collscanRequested = (params.options & QueryPlannerParams::INCLUDE_COLLSCAN);
// No indexed plans? We must provide a collscan if possible or else we can't run the query.
bool collscanNeeded = (0 == out->size() && canTableScan);
if (possibleToCollscan && (collscanRequested || collscanNeeded)) {
QuerySolution* collscan = buildCollscanSoln(query, false, params);
if (NULL != collscan) {
SolutionCacheData* scd = new SolutionCacheData();
scd->solnType = SolutionCacheData::COLLSCAN_SOLN;
collscan->cacheData.reset(scd);
out->push_back(collscan);
QLOG() << "Planner: outputting a collscan:" << endl
<< collscan->toString();
}
}
return Status::OK();
}
