/** * @brief Creates logical tiles from the two input logical tiles after applying * join predicate. * @return true on success, false otherwise. * * ExecutorContext is set when executing IN+NestLoop. For example: * select * from Foo1 where age IN (select id from Foo2 where name='mike'); * Here: * "select id from Foo2 where name='mike'" is transformed as left child. * "select * from Foo1 where age " is the right child. * "IN" is transformed as a execute context, in NestLoop * We put the results of left child in executor_context using NestLoop, and the * right child can execute using this context. Otherwise, the right child can't * execute. And there is no predicate_ for IN+NestLoop * * For now, we only set this context for IN operator. Normally, the right child * has a complete query that can execute without the context, and we use predicate_ * to join the left and right result. * */ bool NestedLoopJoinExecutor::DExecute() { LOG_INFO("********** Nested Loop %s Join executor :: 2 children ", GetJoinTypeString()); // Loop until we have non-empty result tile or exit for (;;) { // Build outer join output when done if (left_child_done_ && right_child_done_) { return BuildOuterJoinOutput(); } //===------------------------------------------------------------------===// // Pick left and right tiles //===------------------------------------------------------------------===// LogicalTile *left_tile = nullptr; LogicalTile *right_tile = nullptr; bool advance_right_child = false; // If we have already retrieved all left child's results in buffer if (left_child_done_ == true) { LOG_TRACE("Advance the left buffer iterator."); assert(!right_result_tiles_.empty()); left_result_itr_++; if (left_result_itr_ >= left_result_tiles_.size()) { advance_right_child = true; left_result_itr_ = 0; } } // Otherwise, we must attempt to execute the left child else { // Left child is finished, no more tiles if (children_[0]->Execute() == false) { LOG_TRACE("Left child is exhausted."); left_child_done_ = true; left_result_itr_ = 0; advance_right_child = true; } // Buffer the left child's result else { LOG_TRACE("Retrieve a new tile from left child"); BufferLeftTile(children_[0]->GetOutput()); left_result_itr_ = left_result_tiles_.size() - 1; } } if (advance_right_child == true || right_result_tiles_.empty()) { // return if right tile is empty if (right_child_done_ && right_result_tiles_.empty()) { return BuildOuterJoinOutput(); } assert(left_result_itr_ == 0); // Right child is finished, no more tiles if (children_[1]->Execute() == false) { LOG_TRACE("Right child is exhausted. Returning false."); // Right child exhausted. // Release cur Right tile. Clear right child's result buffer and return. right_child_done_ = true; return BuildOuterJoinOutput(); } // Buffer the Right child's result else { LOG_TRACE("Advance the Right child."); BufferRightTile(children_[1]->GetOutput()); // return if left tile is empty if (left_child_done_ && left_result_tiles_.empty()) { return BuildOuterJoinOutput(); } } } right_tile = right_result_tiles_.back().get(); left_tile = left_result_tiles_[left_result_itr_].get(); //===------------------------------------------------------------------===// // Build Join Tile //===------------------------------------------------------------------===// // Build output logical tile auto output_tile = BuildOutputLogicalTile(left_tile, right_tile); // Build position lists LogicalTile::PositionListsBuilder pos_lists_builder(left_tile, right_tile); // Go over every pair of tuples in left and right logical tiles for (auto right_tile_row_itr : *right_tile) { bool has_left_match = false; for (auto left_tile_row_itr : *left_tile) { // Join predicate exists if (predicate_ != nullptr) { expression::ContainerTuple<executor::LogicalTile> left_tuple( left_tile, left_tile_row_itr); expression::ContainerTuple<executor::LogicalTile> right_tuple( right_tile, right_tile_row_itr); // Join predicate is false. Skip pair and continue. if (predicate_->Evaluate(&left_tuple, &right_tuple, executor_context_) .IsFalse()) { continue; } } RecordMatchedLeftRow(left_result_itr_, left_tile_row_itr); // For Left and Full Outer Join has_left_match = true; // Insert a tuple into the output logical tile // First, copy the elements in left logical tile's tuple pos_lists_builder.AddRow(left_tile_row_itr, right_tile_row_itr); } // Inner loop of NLJ // For Right and Full Outer Join if (has_left_match) { RecordMatchedRightRow(right_result_tiles_.size() - 1, right_tile_row_itr); } } // Outer loop of NLJ // Check if we have any join tuples. if (pos_lists_builder.Size() > 0) { output_tile->SetPositionListsAndVisibility(pos_lists_builder.Release()); SetOutput(output_tile.release()); return true; } LOG_TRACE("This pair produces empty join result. Continue the loop."); } // end the very beginning for loop }
/** * @brief Creates logical tiles from the two input logical tiles after applying * join predicate. * @return true on success, false otherwise. */ bool HashJoinExecutor::DExecute() { LOG_TRACE("********** Hash Join executor :: 2 children \n"); // Loop until we have non-empty result tile or exit for (;;) { // Check if we have any buffered output tiles if (buffered_output_tiles.empty() == false) { auto output_tile = buffered_output_tiles.front(); SetOutput(output_tile); buffered_output_tiles.pop_front(); return true; } // Build outer join output when done if (left_child_done_ == true) { return BuildOuterJoinOutput(); } //===------------------------------------------------------------------===// // Pick right and left tiles //===------------------------------------------------------------------===// // Get all the tiles from RIGHT child if (right_child_done_ == false) { while (children_[1]->Execute()) { BufferRightTile(children_[1]->GetOutput()); } right_child_done_ = true; } // Get next tile from LEFT child if (children_[0]->Execute() == false) { LOG_TRACE("Did not get left tile \n"); left_child_done_ = true; continue; } BufferLeftTile(children_[0]->GetOutput()); LOG_TRACE("Got left tile \n"); if (right_result_tiles_.size() == 0) { LOG_TRACE("Did not get any right tiles \n"); return BuildOuterJoinOutput(); } LogicalTile *left_tile = left_result_tiles_.back().get(); //===------------------------------------------------------------------===// // Build Join Tile //===------------------------------------------------------------------===// // Get the hash table from the hash executor auto &hash_table = hash_executor_->GetHashTable(); auto &hashed_col_ids = hash_executor_->GetHashKeyIds(); oid_t prev_tile = INVALID_OID; std::unique_ptr<LogicalTile> output_tile; LogicalTile::PositionListsBuilder pos_lists_builder; // Go over the left tile for (auto left_tile_itr : *left_tile) { const expression::ContainerTuple<executor::LogicalTile> left_tuple( left_tile, left_tile_itr, &hashed_col_ids); // Find matching tuples in the hash table built on top of the right table auto right_tuples = hash_table.find(left_tuple); if (right_tuples != hash_table.end()) { // Not yet supported due to assertion in gettomg right_tuples->first // if (predicate_ != nullptr) { // auto eval = predicate_->Evaluate(&left_tuple, &right_tuples->first, // executor_context_); // if (eval.IsFalse()) // continue; // } RecordMatchedLeftRow(left_result_tiles_.size() - 1, left_tile_itr); // Go over the matching right tuples for (auto &location : right_tuples->second) { // Check if we got a new right tile itr if (prev_tile != location.first) { // Check if we have any join tuples if (pos_lists_builder.Size() > 0) { LOG_TRACE("Join tile size : %lu \n", pos_lists_builder.Size()); output_tile->SetPositionListsAndVisibility( pos_lists_builder.Release()); buffered_output_tiles.push_back(output_tile.release()); } // Get the logical tile from right child LogicalTile *right_tile = right_result_tiles_[location.first].get(); // Build output logical tile output_tile = BuildOutputLogicalTile(left_tile, right_tile); // Build position lists pos_lists_builder = LogicalTile::PositionListsBuilder(left_tile, right_tile); pos_lists_builder.SetRightSource( &right_result_tiles_[location.first]->GetPositionLists()); } // Add join tuple pos_lists_builder.AddRow(left_tile_itr, location.second); RecordMatchedRightRow(location.first, location.second); // Cache prev logical tile itr prev_tile = location.first; } } } // Check if we have any join tuples if (pos_lists_builder.Size() > 0) { LOG_TRACE("Join tile size : %lu \n", pos_lists_builder.Size()); output_tile->SetPositionListsAndVisibility(pos_lists_builder.Release()); buffered_output_tiles.push_back(output_tile.release()); } // Check if we have any buffered output tiles if (buffered_output_tiles.empty() == false) { auto output_tile = buffered_output_tiles.front(); SetOutput(output_tile); buffered_output_tiles.pop_front(); return true; } else { // Try again continue; } } }
/** * @brief Creates logical tiles from the two input logical tiles after applying * join predicate. * @return true on success, false otherwise. */ bool HashJoinExecutor::DExecute() { LOG_INFO("********** Hash Join %s Join executor :: 2 children \n", GetJoinTypeString()); // If there is still something in the buffer, we return the first tile in it if (buffered_output_tiles.size() > 0) { SetOutput(buffered_output_tiles[0]); buffered_output_tiles.pop_front(); return true; } //===--------------------------------------------------------------------===// // Pick all right tiles //===--------------------------------------------------------------------===// if (right_child_done_ != true) { // build hash table on right tiles while (hash_executor_->Execute() == true) { BufferRightTile(hash_executor_->GetOutput()); } right_child_done_ = true; } if (right_result_tiles_.empty()) { assert(left_result_tiles_.empty()); LOG_TRACE("Right child returned nothing. Exit."); return false; } // Get the hash table from the hash executor HashExecutor::HashMapType &hash_table_ = hash_executor_->GetHashTable(); // Get the address of column_ids auto col_ids_address = &(hash_executor_->GetHashKeyIds()); // Loop until we have non-empty result join logical tile or exit for (;;) { // Build outer join output when done if (left_child_done_) { assert(right_child_done_ == true); return BuildOuterJoinOutput(); } //===--------------------------------------------------------------------===// // Pick next left tiles //===--------------------------------------------------------------------===// LogicalTile *left_tile = nullptr; // Left child is finished, no more tiles if (children_[0]->Execute() == false) { LOG_TRACE("Left child is exhausted. Returning false."); // Left child exhausted. // Release cur left tile. Clear right child's result buffer and return. assert(right_result_tiles_.size() > 0); left_child_done_ = true; // hash_table_.clear(); return BuildOuterJoinOutput(); } // Buffer the left child's result else { LOG_TRACE("Advance the left child."); BufferLeftTile(children_[0]->GetOutput()); } left_tile = left_result_tiles_.back().get(); //===--------------------------------------------------------------------===// // Build Join Tile //===--------------------------------------------------------------------===// std::unordered_map<size_t, std::unique_ptr<LogicalTile>> output_tile_map; std::unordered_map<size_t, LogicalTile::PositionListsBuilder> pos_lists_builder_map; // Go over the left logical tile for (auto left_tile_row_itr : *left_tile) { auto target_key = HashExecutor::HashMapType::key_type( left_tile, left_tile_row_itr, col_ids_address); auto got = hash_table_.find(target_key); // There is no corresponding right tuples. Skip this left tuple and // continue. if (got == hash_table_.end()) { continue; } // Go over the matching right tuples // Key : container tuple with a subset of tuple attributes // Value : < child_tile offset, tuple offset > auto right_matched_tuples = got->second; for (auto value : right_matched_tuples) { // For Right and Full Outer Join RecordMatchedRightRow(value.first, value.second); // If this left-right pair has not been recorded if (output_tile_map.find(value.first) == output_tile_map.end()) { // we have to firstly update the right_tile, // because this right tuple can belong to a tile which is // different from the one we use to initialize our pos_lists_builder LogicalTile *current_right_tile = right_result_tiles_[value.first].get(); // Build and record output join logical tile output_tile_map[value.first] = BuildOutputLogicalTile(left_tile, current_right_tile); // Build and record corresponding position lists LogicalTile::PositionListsBuilder pos_lists_builder( left_tile, current_right_tile); pos_lists_builder_map[value.first] = pos_lists_builder; } // Insert a tuple into the correct output logical tile in the hashmap pos_lists_builder_map[value.first].AddRow(left_tile_row_itr, value.second); } // For Left and Full Outer Join // if we get some matched right tuples for this left tuple, // we remove it from the no_matching_left_row_sets_ RecordMatchedLeftRow(left_result_tiles_.size() - 1, left_tile_row_itr); } // fill the buffered_output_tiles with values in hashmap for (auto iter = output_tile_map.begin(); iter != output_tile_map.end(); ++iter) { iter->second->SetPositionListsAndVisibility( pos_lists_builder_map[iter->first].Release()); buffered_output_tiles.push_back(iter->second.release()); } // Check if we have any join tuples if (buffered_output_tiles.size() > 0) { SetOutput(buffered_output_tiles[0]); buffered_output_tiles.pop_front(); return true; } LOG_TRACE("This left tile produces empty join result. Continue the loop."); } }
/** * @brief Creates logical tiles from the two input logical tiles after applying * join predicate. * @return true on success, false otherwise. */ bool MergeJoinExecutor::DExecute() { LOG_INFO( "********** Merge Join executor :: 2 children " "left:: start: %lu, end: %lu, done: %d " "right:: start: %lu, end: %lu, done: %d", left_start_row, left_end_row, left_child_done_, right_start_row, right_end_row, right_child_done_); // Build outer join output when done if (right_child_done_ && left_child_done_) { return BuildOuterJoinOutput(); } //===--------------------------------------------------------------------===// // Pick right and left tiles //===--------------------------------------------------------------------===// // Try to get next tile from RIGHT child if (((right_child_done_ == false) && (right_start_row == right_end_row)) || (left_child_done_ == true)) { if (children_[1]->Execute() == false) { LOG_TRACE("Did not get right tile "); right_child_done_ = true; // Try again return DExecute(); } LOG_TRACE("Got right tile "); auto right_tile = children_[1]->GetOutput(); BufferRightTile(right_tile); right_start_row = 0; right_end_row = Advance(right_tile, right_start_row, false); LOG_TRACE("size of right tiles: %lu", right_result_tiles_.size()); } // Try to get next tile from LEFT child if (((left_child_done_ == false) && (left_start_row == left_end_row)) || (right_child_done_ == true)) { if (children_[0]->Execute() == false) { LOG_TRACE("Did not get left tile "); left_child_done_ = true; // Try again return DExecute(); } LOG_TRACE("Got left tile "); auto left_tile = children_[0]->GetOutput(); BufferLeftTile(left_tile); left_start_row = 0; left_end_row = Advance(left_tile, left_start_row, true); LOG_TRACE("size of left tiles: %lu", left_result_tiles_.size()); } // Check if we have logical tiles to process if(left_result_tiles_.empty() || right_result_tiles_.empty()) { return false; } LogicalTile *left_tile = left_result_tiles_.back().get(); LogicalTile *right_tile = right_result_tiles_.back().get(); //===--------------------------------------------------------------------===// // Build Join Tile //===--------------------------------------------------------------------===// // Build output logical tile auto output_tile = BuildOutputLogicalTile(left_tile, right_tile); // Build position lists LogicalTile::PositionListsBuilder pos_lists_builder(left_tile, right_tile); while ((left_end_row > left_start_row) && (right_end_row > right_start_row)) { expression::ContainerTuple<executor::LogicalTile> left_tuple( left_tile, left_start_row); expression::ContainerTuple<executor::LogicalTile> right_tuple( right_tile, right_start_row); bool not_matching_tuple_pair = false; // Evaluate and compare the join clauses for (auto &clause : *join_clauses_) { auto left_value = clause.left_->Evaluate(&left_tuple, &right_tuple, nullptr); auto right_value = clause.right_->Evaluate(&left_tuple, &right_tuple, nullptr); // Compare the values int comparison = left_value.Compare(right_value); // Left key < Right key, advance left if (comparison < 0) { LOG_TRACE("left < right, advance left "); left_start_row = left_end_row; left_end_row = Advance(left_tile, left_start_row, true); not_matching_tuple_pair = true; break; } // Left key > Right key, advance right else if (comparison > 0) { LOG_TRACE("left > right, advance right "); right_start_row = right_end_row; right_end_row = Advance(right_tile, right_start_row, false); not_matching_tuple_pair = true; break; } // Left key == Right key, go and check next join clause } // Atleast one of the join clauses don't match // One of the tile has been advanced if (not_matching_tuple_pair) { continue; } // Join clauses matched, try to match predicate LOG_TRACE("one pair of tuples matches join clause "); // Join predicate exists if (predicate_ != nullptr) { if (predicate_->Evaluate(&left_tuple, &right_tuple, executor_context_) .IsFalse()) { // Join predicate is false. Advance both. left_start_row = left_end_row; left_end_row = Advance(left_tile, left_start_row, true); right_start_row = right_end_row; right_end_row = Advance(right_tile, right_start_row, false); } } // Sub tile matched, do a Cartesian product // Go over every pair of tuples in left and right logical tiles for (size_t left_tile_row_itr = left_start_row; left_tile_row_itr < left_end_row; left_tile_row_itr++) { for (size_t right_tile_row_itr = right_start_row; right_tile_row_itr < right_end_row; right_tile_row_itr++) { // Insert a tuple into the output logical tile pos_lists_builder.AddRow(left_tile_row_itr, right_tile_row_itr); RecordMatchedLeftRow(left_result_tiles_.size() - 1, left_tile_row_itr); RecordMatchedRightRow(right_result_tiles_.size() - 1, right_tile_row_itr); } } // Then, advance both left_start_row = left_end_row; left_end_row = Advance(left_tile, left_start_row, true); right_start_row = right_end_row; right_end_row = Advance(right_tile, right_start_row, false); } // Check if we have any join tuples. if (pos_lists_builder.Size() > 0) { output_tile->SetPositionListsAndVisibility(pos_lists_builder.Release()); SetOutput(output_tile.release()); return true; } // Try again else { // If we are out of any more pairs of child tiles to examine, // then we will return false earlier in this function // So, no need to return false here DExecute(); } return true; }
/** * @brief Creates logical tiles from the two input logical tiles after applying * join predicate. * @return true on success, false otherwise. */ bool HashJoinExecutor::DExecute() { // build hash map for right table if (!right_child_done_) { while (hash_executor_->Execute() == true) BufferRightTile(children_[1]->GetOutput()); right_child_done_ = true; } for (;;) { // left child & right child all done if (left_child_done_ && right_child_done_) { return BuildOuterJoinOutput(); } // if there is remaining pairs in buffer, release one at a time. if (!buffered_output_tiles.empty()) { // just hand in one. SetOutput(buffered_output_tiles.front()); buffered_output_tiles.pop_front(); return true; } // traverse every left child tile if (children_[0]->Execute()) { BufferLeftTile(children_[0]->GetOutput()); LogicalTile *left_tile = left_result_tiles_.back().get(); // traverse every tuple in curt left tile for (auto left_tile_row_itr : *left_tile) { auto hash = HashExecutor::HashMapType::key_type( left_tile, left_tile_row_itr, &hash_executor_->GetHashKeyIds()); auto hash_result = hash_executor_->GetHashTable().find(hash); if (hash_result != hash_executor_->GetHashTable().end()) { RecordMatchedLeftRow(left_logical_tile_itr_, left_tile_row_itr); // traverse right set for (auto iter = hash_result->second.begin(); iter != hash_result->second.end(); ++iter) { auto tile_index = iter->first; auto tuple_index = iter->second; RecordMatchedRightRow(tile_index, tuple_index); LogicalTile *right_tile = right_result_tiles_[tile_index].get(); LogicalTile::PositionListsBuilder pos_lists_builder(left_tile, right_tile); pos_lists_builder.AddRow(left_tile_row_itr, tuple_index); auto output_tile = BuildOutputLogicalTile(left_tile, right_tile); output_tile->SetPositionListsAndVisibility( pos_lists_builder.Release()); buffered_output_tiles.emplace_back(output_tile.release()); } // end of traversing right set } // end of if match } // end of traversal of curt left_tile // Release at most one pair. // PS: This should be done after traversing all the tuples in curt tile left_logical_tile_itr_++; if (!buffered_output_tiles.empty()) { // release one at a time SetOutput(buffered_output_tiles.front()); buffered_output_tiles.pop_front(); return true; } } // end of still have left tile // All left tiles are exhausted. else { left_child_done_ = true; return BuildOuterJoinOutput(); } } // end of infinite loop // never should go here return false; } // end of DExecute
/** * @brief Creates logical tiles from the two input logical tiles after applying * join predicate. * @return true on success, false otherwise. */ bool HashJoinExecutor::DExecute() { LOG_INFO("Hash Join Executor"); // Loop until we have non-empty result join logical tile or exit while (true) { // if (!buffered_output_tiles.empty()) { if (!result.empty()) { auto* output_tile = result.back(); result.pop_back(); SetOutput(output_tile); return true; } // Build outer join output when done if (left_child_done_ && right_child_done_) { return BuildOuterJoinOutput(); } //===--------------------------------------------------------------------===// // Pick right and left tiles //===--------------------------------------------------------------------===// // Get all the logical tiles from RIGHT child if (!right_child_done_) { while (children_[1]->Execute()) { BufferRightTile(children_[1]->GetOutput()); } right_child_done_ = true; LOG_INFO("Hash Join Executor: Got all %lu right tiles.", right_result_tiles_.size()); } // Get next logical tile from LEFT child if (children_[0]->Execute()) { BufferLeftTile(children_[0]->GetOutput()); LOG_INFO("Hash Join Executor: Got left tile %p.", left_result_tiles_.back().get()); } else { // Left input is exhausted, loop around left_child_done_ = true; return BuildOuterJoinOutput(); } if (right_result_tiles_.empty()) { /// No right children, a hash lookup would be empty. Continue ... continue; } //===--------------------------------------------------------------------===// // Build Join Tile //===--------------------------------------------------------------------===// LogicalTile* left_tile = left_result_tiles_.back().get(); std::unordered_map<size_t, std::unique_ptr<LogicalTile::PositionListsBuilder>> right_matches; // Get the hash table from the hash executor auto& hash_table = hash_executor_->GetHashTable(); auto& hash_columns = hash_executor_->GetHashKeyIds(); for (oid_t left_tid : *left_tile) { /// Create key and probe hash table HashExecutor::HashMapType::key_type key(left_tile, left_tid, &hash_columns); const auto& iter = hash_table.find(key); if (iter == hash_table.end()) { continue; } auto& matches = iter->second; for (auto& match : matches) { auto right_tile_index = match.first; auto* right_tile = right_result_tiles_[right_tile_index].get(); auto right_tid = match.second; RecordMatchedRightRow(right_tile_index, right_tid); RecordMatchedLeftRow(left_result_tiles_.size() - 1, left_tid); const auto& pos_match_iter = right_matches.find(right_tile_index); if (pos_match_iter == right_matches.end()) { std::unique_ptr<LogicalTile::PositionListsBuilder> builder{ new LogicalTile::PositionListsBuilder(left_tile, right_tile)}; right_matches.insert( std::make_pair(right_tile_index, std::move(builder))); } right_matches[right_tile_index]->AddRow(left_tid, right_tid); } } // Create a new logical tile for every grouped match in matches for (auto& iter : right_matches) { auto output_tile = BuildOutputLogicalTile( left_tile, right_result_tiles_[iter.first].get()); auto& pos_lists_builder = iter.second; output_tile->SetPositionListsAndVisibility(pos_lists_builder->Release()); result.push_back(output_tile.release()); } } }