Пример #1
0
TEST_F(TableTupleFilterTest, tableTupleFilterTest)
{
    static const int MARKER = 33;

    TempTable* table = getTempTable();
    TableTupleFilter tableFilter;
    tableFilter.init(table);

    int tuplePerBlock = table->getTuplesPerBlock();
    // make sure table spans more than one block
    ASSERT_TRUE(NUM_OF_TUPLES / tuplePerBlock > 1);

    TableTuple tuple = table->tempTuple();
    TableIterator iterator = table->iterator();

    // iterator over and mark every 5th tuple
    int counter = 0;
    std::multiset<int64_t> control_values;

    while(iterator.next(tuple)) {
        if (++counter % 5 == 0) {
            NValue nvalue = tuple.getNValue(1);
            int64_t value = ValuePeeker::peekBigInt(nvalue);
            control_values.insert(value);
            tableFilter.updateTuple(tuple, MARKER);
        }
    }

    TableTupleFilter_iter<MARKER> endItr = tableFilter.end<MARKER>();
    for (TableTupleFilter_iter<MARKER> itr = tableFilter.begin<MARKER>(); itr != endItr; ++itr) {
            uint64_t tupleAddr = tableFilter.getTupleAddress(*itr);
            tuple.move((char *)tupleAddr);
            ASSERT_TRUE(tuple.isActive());
            NValue nvalue = tuple.getNValue(1);
            int64_t value = ValuePeeker::peekBigInt(nvalue);

            ASSERT_FALSE(control_values.empty());
            auto it = control_values.find(value);
            ASSERT_NE(it, control_values.end());
            control_values.erase(it);
    }
    ASSERT_TRUE(control_values.empty());

}
Пример #2
0
bool NestLoopIndexExecutor::p_execute(const NValueArray &params)
{
    assert(dynamic_cast<NestLoopIndexPlanNode*>(m_abstractNode));
    NestLoopIndexPlanNode* node = static_cast<NestLoopIndexPlanNode*>(m_abstractNode);

    // output table must be a temp table
    assert(m_tmpOutputTable);
    // target table is a persistent table
    assert(dynamic_cast<PersistentTable*>(m_indexNode->getTargetTable()));
    PersistentTable* inner_table = static_cast<PersistentTable*>(m_indexNode->getTargetTable());

    TableIndex* index = inner_table->index(m_indexNode->getTargetIndexName());
    assert(index);
    IndexCursor indexCursor(index->getTupleSchema());

    //outer_table is the input table that have tuples to be iterated
    assert(node->getInputTableCount() == 1);
    Table* outer_table = node->getInputTable();
    assert(outer_table);
    VOLT_TRACE("executing NestLoopIndex with outer table: %s, inner table: %s",
               outer_table->debug().c_str(), inner_table->debug().c_str());

    //
    // Substitute parameter to SEARCH KEY Note that the expressions
    // will include TupleValueExpression even after this substitution
    //
    int num_of_searchkeys = static_cast <int> (m_indexNode->getSearchKeyExpressions().size());
    for (int ctr = 0; ctr < num_of_searchkeys; ctr++) {
        VOLT_TRACE("Search Key[%d]:\n%s",
                   ctr, m_indexNode->getSearchKeyExpressions()[ctr]->debug(true).c_str());
    }

    // end expression
    AbstractExpression* end_expression = m_indexNode->getEndExpression();
    if (end_expression) {
        VOLT_TRACE("End Expression:\n%s", end_expression->debug(true).c_str());
    }

    // post expression
    AbstractExpression* post_expression = m_indexNode->getPredicate();
    if (post_expression != NULL) {
        VOLT_TRACE("Post Expression:\n%s", post_expression->debug(true).c_str());
    }

    // initial expression
    AbstractExpression* initial_expression = m_indexNode->getInitialExpression();
    if (initial_expression != NULL) {
        VOLT_TRACE("Initial Expression:\n%s", initial_expression->debug(true).c_str());
    }

    // SKIP NULL EXPRESSION
    AbstractExpression* skipNullExpr = m_indexNode->getSkipNullPredicate();
    // For reverse scan edge case NULL values and forward scan underflow case.
    if (skipNullExpr != NULL) {
        VOLT_DEBUG("Skip NULL Expression:\n%s", skipNullExpr->debug(true).c_str());
    }

    // pre join expression
    AbstractExpression* prejoin_expression = node->getPreJoinPredicate();
    if (prejoin_expression != NULL) {
        VOLT_TRACE("Prejoin Expression:\n%s", prejoin_expression->debug(true).c_str());
    }

    // where expression
    AbstractExpression* where_expression = node->getWherePredicate();
    if (where_expression != NULL) {
        VOLT_TRACE("Where Expression:\n%s", where_expression->debug(true).c_str());
    }

    LimitPlanNode* limit_node = dynamic_cast<LimitPlanNode*>(node->getInlinePlanNode(PLAN_NODE_TYPE_LIMIT));
    int limit = CountingPostfilter::NO_LIMIT;
    int offset = CountingPostfilter::NO_OFFSET;
    if (limit_node) {
        limit_node->getLimitAndOffsetByReference(params, limit, offset);
    }
    // Init the postfilter
    CountingPostfilter postfilter(m_tmpOutputTable, where_expression, limit, offset);

    //
    // OUTER TABLE ITERATION
    //
    TableTuple outer_tuple(outer_table->schema());
    TableTuple inner_tuple(inner_table->schema());
    TableIterator outer_iterator = outer_table->iteratorDeletingAsWeGo();
    int num_of_outer_cols = outer_table->columnCount();
    assert (outer_tuple.sizeInValues() == outer_table->columnCount());
    assert (inner_tuple.sizeInValues() == inner_table->columnCount());
    const TableTuple &null_inner_tuple = m_null_inner_tuple.tuple();
    ProgressMonitorProxy pmp(m_engine->getExecutorContext(), this);

    // The table filter to keep track of inner tuples that don't match any of outer tuples for FULL joins
    TableTupleFilter innerTableFilter;
    if (m_joinType == JOIN_TYPE_FULL) {
        // Prepopulate the set with all inner tuples
        innerTableFilter.init(inner_table);
    }

    TableTuple join_tuple;
    // It's not immediately obvious here, so there's some subtlety to
    // note with respect to the schema of the join_tuple.
    //
    // The inner_tuple is used to represent the values from the inner
    // table in the case of the join predicate passing, and for left
    // outer joins, the null_tuple is used if there is no match.  Both
    // of these tuples include the complete schema of the table being
    // scanned.  The inner table is being scanned via an inlined scan
    // node, so there is no temp table corresponding to it.
    //
    // Predicates that are evaluated against the inner table should
    // therefore use the complete schema of the table being scanned.
    //
    // The join_tuple is the tuple that contains the values that we
    // actually want to put in the output of the join (or to aggregate
    // if there is an inlined agg plan node).  This tuple needs to
    // omit the unused columns from the inner table.  The inlined
    // index scan itself has an inlined project node that defines the
    // columns that should be output by the join, and omits those that
    // are not needed.  So the join_tuple contains the columns we're
    // using from the outer table, followed by the "projected" schema
    // for the inlined scan of the inner table.
    if (m_aggExec != NULL) {
        VOLT_TRACE("Init inline aggregate...");
        const TupleSchema * aggInputSchema = node->getTupleSchemaPreAgg();
        join_tuple = m_aggExec->p_execute_init(params, &pmp, aggInputSchema, m_tmpOutputTable, &postfilter);
    }
    else {
        join_tuple = m_tmpOutputTable->tempTuple();
    }

    VOLT_TRACE("<num_of_outer_cols>: %d\n", num_of_outer_cols);
    while (postfilter.isUnderLimit() && outer_iterator.next(outer_tuple)) {
        VOLT_TRACE("outer_tuple:%s",
                   outer_tuple.debug(outer_table->name()).c_str());
        pmp.countdownProgress();

        // Set the join tuple columns that originate solely from the outer tuple.
        // Must be outside the inner loop in case of the empty inner table.
        join_tuple.setNValues(0, outer_tuple, 0, num_of_outer_cols);

        // did this loop body find at least one match for this tuple?
        bool outerMatch = false;
        // For outer joins if outer tuple fails pre-join predicate
        // (join expression based on the outer table only)
        // it can't match any of inner tuples
        if (prejoin_expression == NULL || prejoin_expression->eval(&outer_tuple, NULL).isTrue()) {
            int activeNumOfSearchKeys = num_of_searchkeys;
            VOLT_TRACE ("<Nested Loop Index exec, WHILE-LOOP...> Number of searchKeys: %d \n", num_of_searchkeys);
            IndexLookupType localLookupType = m_lookupType;
            SortDirectionType localSortDirection = m_sortDirection;
            VOLT_TRACE("Lookup type: %d\n", m_lookupType);
            VOLT_TRACE("SortDirectionType: %d\n", m_sortDirection);

            // did setting the search key fail (usually due to overflow)
            bool keyException = false;
            //
            // Now use the outer table tuple to construct the search key
            // against the inner table
            //
            const TableTuple& index_values = m_indexValues.tuple();
            index_values.setAllNulls();
            for (int ctr = 0; ctr < activeNumOfSearchKeys; ctr++) {
                // in a normal index scan, params would be substituted here,
                // but this scan fills in params outside the loop
                NValue candidateValue = m_indexNode->getSearchKeyExpressions()[ctr]->eval(&outer_tuple, NULL);
                if (candidateValue.isNull()) {
                    // when any part of the search key is NULL, the result is false when it compares to anything.
                    // do early return optimization, our index comparator may not handle null comparison correctly.
                    keyException = true;
                    break;
                }
                try {
                    index_values.setNValue(ctr, candidateValue);
                }
                catch (const SQLException &e) {
                    // This next bit of logic handles underflow and overflow while
                    // setting up the search keys.
                    // e.g. TINYINT > 200 or INT <= 6000000000

                    // re-throw if not an overflow or underflow
                    // currently, it's expected to always be an overflow or underflow
                    if ((e.getInternalFlags() & (SQLException::TYPE_OVERFLOW | SQLException::TYPE_UNDERFLOW | SQLException::TYPE_VAR_LENGTH_MISMATCH)) == 0) {
                        throw e;
                    }

                    // handle the case where this is a comparison, rather than equality match
                    // comparison is the only place where the executor might return matching tuples
                    // e.g. TINYINT < 1000 should return all values
                    if ((localLookupType != INDEX_LOOKUP_TYPE_EQ) &&
                        (ctr == (activeNumOfSearchKeys - 1))) {

                        if (e.getInternalFlags() & SQLException::TYPE_OVERFLOW) {
                            if ((localLookupType == INDEX_LOOKUP_TYPE_GT) ||
                                (localLookupType == INDEX_LOOKUP_TYPE_GTE)) {

                                // gt or gte when key overflows breaks out
                                // and only returns for left-outer
                                keyException = true;
                                break; // the outer while loop
                            }
                            else {
                                // overflow of LT or LTE should be treated as LTE
                                // to issue an "initial" forward scan
                                localLookupType = INDEX_LOOKUP_TYPE_LTE;
                            }
                        }
                        if (e.getInternalFlags() & SQLException::TYPE_UNDERFLOW) {
                            if ((localLookupType == INDEX_LOOKUP_TYPE_LT) ||
                                (localLookupType == INDEX_LOOKUP_TYPE_LTE)) {
                                // overflow of LT or LTE should be treated as LTE
                                // to issue an "initial" forward scans
                                localLookupType = INDEX_LOOKUP_TYPE_LTE;
                            }
                            else {
                                // don't allow GTE because it breaks null handling
                                localLookupType = INDEX_LOOKUP_TYPE_GT;
                            }
                        }
                        if (e.getInternalFlags() & SQLException::TYPE_VAR_LENGTH_MISMATCH) {
                            // shrink the search key and add the updated key to search key table tuple
                            index_values.shrinkAndSetNValue(ctr, candidateValue);
                            // search will be performed on shrinked key, so update lookup operation
                            // to account for it
                            switch (localLookupType) {
                                case INDEX_LOOKUP_TYPE_LT:
                                case INDEX_LOOKUP_TYPE_LTE:
                                    localLookupType = INDEX_LOOKUP_TYPE_LTE;
                                    break;
                                case INDEX_LOOKUP_TYPE_GT:
                                case INDEX_LOOKUP_TYPE_GTE:
                                    localLookupType = INDEX_LOOKUP_TYPE_GT;
                                    break;
                                default:
                                    assert(!"IndexScanExecutor::p_execute - can't index on not equals");
                                    return false;
                            }
                        }

                        // if here, means all tuples with the previous searchkey
                        // columns need to be scaned.
                        activeNumOfSearchKeys--;
                        if (localSortDirection == SORT_DIRECTION_TYPE_INVALID) {
                            localSortDirection = SORT_DIRECTION_TYPE_ASC;
                        }
                    }
                    // if a EQ comparison is out of range, then the tuple from
                    // the outer loop returns no matches (except left-outer)
                    else {
                        keyException = true;
                    }
                    break;
                } // End catch block for under- or overflow when setting index key
            } // End for each active search key
            VOLT_TRACE("Searching %s", index_values.debug("").c_str());

            // if a search value didn't fit into the targeted index key, skip this key
            if (!keyException) {
                //
                // Our index scan on the inner table is going to have three parts:
                //  (1) Lookup tuples using the search key
                //
                //  (2) For each tuple that comes back, check whether the
                //      end_expression is false.  If it is, then we stop
                //      scanning. Otherwise...
                //
                //  (3) Check whether the tuple satisfies the post expression.
                //      If it does, then add it to the output table
                //
                // Use our search key to prime the index iterator
                // The loop through each tuple given to us by the iterator
                //
                // Essentially cut and pasted this if ladder from
                // index scan executor
                if (num_of_searchkeys > 0) {
                    if (localLookupType == INDEX_LOOKUP_TYPE_EQ) {
                        index->moveToKey(&index_values, indexCursor);
                    }
                    else if (localLookupType == INDEX_LOOKUP_TYPE_GT) {
                        index->moveToGreaterThanKey(&index_values, indexCursor);
                    }
                    else if (localLookupType == INDEX_LOOKUP_TYPE_GTE) {
                        index->moveToKeyOrGreater(&index_values, indexCursor);
                    }
                    else if (localLookupType == INDEX_LOOKUP_TYPE_LT) {
                        index->moveToLessThanKey(&index_values, indexCursor);
                    }
                    else if (localLookupType == INDEX_LOOKUP_TYPE_LTE) {
                        // find the entry whose key is greater than search key,
                        // do a forward scan using initialExpr to find the correct
                        // start point to do reverse scan
                        bool isEnd = index->moveToGreaterThanKey(&index_values, indexCursor);
                        if (isEnd) {
                            index->moveToEnd(false, indexCursor);
                        }
                        else {
                            while (!(inner_tuple = index->nextValue(indexCursor)).isNullTuple()) {
                                pmp.countdownProgress();
                                if (initial_expression != NULL && !initial_expression->eval(&outer_tuple, &inner_tuple).isTrue()) {
                                    // just passed the first failed entry, so move 2 backward
                                    index->moveToBeforePriorEntry(indexCursor);
                                    break;
                                }
                            }
                            if (inner_tuple.isNullTuple()) {
                                index->moveToEnd(false, indexCursor);
                            }
                        }
                    }
                    else if (localLookupType == INDEX_LOOKUP_TYPE_GEO_CONTAINS) {
                        index->moveToCoveringCell(&index_values, indexCursor);
                    }
                    else {
                        return false;
                    }
                }
                else {
                    bool toStartActually = (localSortDirection != SORT_DIRECTION_TYPE_DESC);
                    index->moveToEnd(toStartActually, indexCursor);
                }

                AbstractExpression* skipNullExprIteration = skipNullExpr;

                while (postfilter.isUnderLimit() &&
                       IndexScanExecutor::getNextTuple(localLookupType,
                                                       &inner_tuple,
                                                       index,
                                                       &indexCursor,
                                                       num_of_searchkeys)) {
                    if (inner_tuple.isPendingDelete()) {
                        continue;
                    }
                    VOLT_TRACE("inner_tuple:%s",
                               inner_tuple.debug(inner_table->name()).c_str());
                    pmp.countdownProgress();

                    //
                    // First check to eliminate the null index rows for UNDERFLOW case only
                    //
                    if (skipNullExprIteration != NULL) {
                        if (skipNullExprIteration->eval(&outer_tuple, &inner_tuple).isTrue()) {
                            VOLT_DEBUG("Index scan: find out null rows or columns.");
                            continue;
                        }
                        skipNullExprIteration = NULL;
                    }

                    //
                    // First check whether the end_expression is now false
                    //
                    if (end_expression != NULL &&
                        !end_expression->eval(&outer_tuple, &inner_tuple).isTrue()) {
                        VOLT_TRACE("End Expression evaluated to false, stopping scan\n");
                        break;
                    }
                    //
                    // Then apply our post-predicate to do further filtering
                    //
                    if (post_expression == NULL ||
                        post_expression->eval(&outer_tuple, &inner_tuple).isTrue()) {
                        outerMatch = true;
                        // The inner tuple passed the join conditions
                        if (m_joinType == JOIN_TYPE_FULL) {
                            // Mark inner tuple as matched
                            innerTableFilter.updateTuple(inner_tuple, MATCHED_TUPLE);
                        }
                        // Still need to pass where filtering
                        if (postfilter.eval(&outer_tuple, &inner_tuple)) {
                            //
                            // Try to put the tuple into our output table
                            // Append the inner values to the end of our join tuple
                            //
                            for (int col_ctr = num_of_outer_cols;
                                 col_ctr < join_tuple.sizeInValues();
                                 ++col_ctr) {
                                join_tuple.setNValue(col_ctr,
                                          m_outputExpressions[col_ctr]->eval(&outer_tuple, &inner_tuple));
                            }
                            VOLT_TRACE("join_tuple tuple: %s",
                                       join_tuple.debug(m_tmpOutputTable->name()).c_str());
                            VOLT_TRACE("MATCH: %s",
                                   join_tuple.debug(m_tmpOutputTable->name()).c_str());
                            outputTuple(postfilter, join_tuple, pmp);
                        }
                    }
                } // END INNER WHILE LOOP
            } // END IF INDEX KEY EXCEPTION CONDITION
        } // END IF PRE JOIN CONDITION

        //
        // Left/Full Outer Join
        //
        if (m_joinType != JOIN_TYPE_INNER && !outerMatch && postfilter.isUnderLimit())
        {
            // Still needs to pass the filter
            if (postfilter.eval(&outer_tuple, &null_inner_tuple)) {
                // Matched! Complete the joined tuple with null inner column values.
                for (int col_ctr = num_of_outer_cols;
                     col_ctr < join_tuple.sizeInValues();
                     ++col_ctr) {
                    join_tuple.setNValue(col_ctr,
                            m_outputExpressions[col_ctr]->eval(&outer_tuple, &null_inner_tuple));
                }
                outputTuple(postfilter, join_tuple, pmp);
            }
        }
    } // END OUTER WHILE LOOP

    //
    // FULL Outer Join. Iterate over the unmatched inner tuples
    //
    if (m_joinType == JOIN_TYPE_FULL && postfilter.isUnderLimit()) {
        // Preset outer columns to null
        const TableTuple& null_outer_tuple = m_null_outer_tuple.tuple();
        join_tuple.setNValues(0, null_outer_tuple, 0, num_of_outer_cols);

        TableTupleFilter_iter<UNMATCHED_TUPLE> endItr = innerTableFilter.end<UNMATCHED_TUPLE>();
        for (TableTupleFilter_iter<UNMATCHED_TUPLE> itr = innerTableFilter.begin<UNMATCHED_TUPLE>();
                itr != endItr && postfilter.isUnderLimit(); ++itr) {
            // Restore the tuple value
            uint64_t tupleAddr = innerTableFilter.getTupleAddress(*itr);
            inner_tuple.move((char *)tupleAddr);
            // Still needs to pass the filter
            assert(inner_tuple.isActive());
            if (postfilter.eval(&null_outer_tuple, &inner_tuple)) {
                // Passed! Complete the joined tuple with the inner column values.
                for (int col_ctr = num_of_outer_cols;
                     col_ctr < join_tuple.sizeInValues();
                     ++col_ctr) {
                    join_tuple.setNValue(col_ctr,
                            m_outputExpressions[col_ctr]->eval(&null_outer_tuple, &inner_tuple));
                }
                outputTuple(postfilter, join_tuple, pmp);
            }
        }
    }

    if (m_aggExec != NULL) {
        m_aggExec->p_execute_finish();
    }

    VOLT_TRACE ("result table:\n %s", m_tmpOutputTable->debug().c_str());
    VOLT_TRACE("Finished NestLoopIndex");

    cleanupInputTempTable(inner_table);
    cleanupInputTempTable(outer_table);

    return (true);
}