/**
* @brief Creates logical tiles from the two input logical tiles after applying
* join predicate.
* @return true on success, false otherwise.
*
* ExecutorContext is set when executing IN+NestLoop. For example:
* select * from Foo1 where age IN (select id from Foo2 where name='mike');
* Here:
* "select id from Foo2 where name='mike'" is transformed as left child.
* "select * from Foo1 where age " is the right child.
* "IN" is transformed as a execute context, in NestLoop
* We put the results of left child in executor_context using NestLoop, and the
* right child can execute using this context. Otherwise, the right child can't
* execute. And there is no predicate_ for IN+NestLoop
*
* For now, we only set this context for IN operator. Normally, the right child
* has a complete query that can execute without the context, and we use predicate_
* to join the left and right result.
*
*/
bool NestedLoopJoinExecutor::DExecute() {
LOG_INFO("********** Nested Loop %s Join executor :: 2 children ",
GetJoinTypeString());
// Loop until we have non-empty result tile or exit
for (;;) {
// Build outer join output when done
if (left_child_done_ && right_child_done_) {
return BuildOuterJoinOutput();
}
//===------------------------------------------------------------------===//
// Pick left and right tiles
//===------------------------------------------------------------------===//
LogicalTile *left_tile = nullptr;
LogicalTile *right_tile = nullptr;
bool advance_right_child = false;
// If we have already retrieved all left child's results in buffer
if (left_child_done_ == true) {
LOG_TRACE("Advance the left buffer iterator.");
assert(!right_result_tiles_.empty());
left_result_itr_++;
if (left_result_itr_ >= left_result_tiles_.size()) {
advance_right_child = true;
left_result_itr_ = 0;
}
}
// Otherwise, we must attempt to execute the left child
else {
// Left child is finished, no more tiles
if (children_[0]->Execute() == false) {
LOG_TRACE("Left child is exhausted.");
left_child_done_ = true;
left_result_itr_ = 0;
advance_right_child = true;
}
// Buffer the left child's result
else {
LOG_TRACE("Retrieve a new tile from left child");
BufferLeftTile(children_[0]->GetOutput());
left_result_itr_ = left_result_tiles_.size() - 1;
}
}
if (advance_right_child == true || right_result_tiles_.empty()) {
// return if right tile is empty
if (right_child_done_ && right_result_tiles_.empty()) {
return BuildOuterJoinOutput();
}
assert(left_result_itr_ == 0);
// Right child is finished, no more tiles
if (children_[1]->Execute() == false) {
LOG_TRACE("Right child is exhausted. Returning false.");
// Right child exhausted.
// Release cur Right tile. Clear right child's result buffer and return.
right_child_done_ = true;
return BuildOuterJoinOutput();
}
// Buffer the Right child's result
else {
LOG_TRACE("Advance the Right child.");
BufferRightTile(children_[1]->GetOutput());
// return if left tile is empty
if (left_child_done_ && left_result_tiles_.empty()) {
return BuildOuterJoinOutput();
}
}
}
right_tile = right_result_tiles_.back().get();
left_tile = left_result_tiles_[left_result_itr_].get();
//===------------------------------------------------------------------===//
// Build Join Tile
//===------------------------------------------------------------------===//
// Build output logical tile
auto output_tile = BuildOutputLogicalTile(left_tile, right_tile);
// Build position lists
LogicalTile::PositionListsBuilder pos_lists_builder(left_tile, right_tile);
// Go over every pair of tuples in left and right logical tiles
for (auto right_tile_row_itr : *right_tile) {
bool has_left_match = false;
for (auto left_tile_row_itr : *left_tile) {
// Join predicate exists
if (predicate_ != nullptr) {
expression::ContainerTuple<executor::LogicalTile> left_tuple(
left_tile, left_tile_row_itr);
expression::ContainerTuple<executor::LogicalTile> right_tuple(
right_tile, right_tile_row_itr);
// Join predicate is false. Skip pair and continue.
if (predicate_->Evaluate(&left_tuple, &right_tuple, executor_context_)
.IsFalse()) {
continue;
}
}
RecordMatchedLeftRow(left_result_itr_, left_tile_row_itr);
// For Left and Full Outer Join
has_left_match = true;
// Insert a tuple into the output logical tile
// First, copy the elements in left logical tile's tuple
pos_lists_builder.AddRow(left_tile_row_itr, right_tile_row_itr);
} // Inner loop of NLJ
// For Right and Full Outer Join
if (has_left_match) {
RecordMatchedRightRow(right_result_tiles_.size() - 1,
right_tile_row_itr);
}
} // Outer loop of NLJ
// Check if we have any join tuples.
if (pos_lists_builder.Size() > 0) {
output_tile->SetPositionListsAndVisibility(pos_lists_builder.Release());
SetOutput(output_tile.release());
return true;
}
LOG_TRACE("This pair produces empty join result. Continue the loop.");
} // end the very beginning for loop
}
/**
* @brief Creates logical tiles from the two input logical tiles after applying
* join predicate.
* @return true on success, false otherwise.
*/
bool HashJoinExecutor::DExecute() {
LOG_TRACE("********** Hash Join executor :: 2 children \n");
// Loop until we have non-empty result tile or exit
for (;;) {
// Check if we have any buffered output tiles
if (buffered_output_tiles.empty() == false) {
auto output_tile = buffered_output_tiles.front();
SetOutput(output_tile);
buffered_output_tiles.pop_front();
return true;
}
// Build outer join output when done
if (left_child_done_ == true) {
return BuildOuterJoinOutput();
}
//===------------------------------------------------------------------===//
// Pick right and left tiles
//===------------------------------------------------------------------===//
// Get all the tiles from RIGHT child
if (right_child_done_ == false) {
while (children_[1]->Execute()) {
BufferRightTile(children_[1]->GetOutput());
}
right_child_done_ = true;
}
// Get next tile from LEFT child
if (children_[0]->Execute() == false) {
LOG_TRACE("Did not get left tile \n");
left_child_done_ = true;
continue;
}
BufferLeftTile(children_[0]->GetOutput());
LOG_TRACE("Got left tile \n");
if (right_result_tiles_.size() == 0) {
LOG_TRACE("Did not get any right tiles \n");
return BuildOuterJoinOutput();
}
LogicalTile *left_tile = left_result_tiles_.back().get();
//===------------------------------------------------------------------===//
// Build Join Tile
//===------------------------------------------------------------------===//
// Get the hash table from the hash executor
auto &hash_table = hash_executor_->GetHashTable();
auto &hashed_col_ids = hash_executor_->GetHashKeyIds();
oid_t prev_tile = INVALID_OID;
std::unique_ptr<LogicalTile> output_tile;
LogicalTile::PositionListsBuilder pos_lists_builder;
// Go over the left tile
for (auto left_tile_itr : *left_tile) {
const expression::ContainerTuple<executor::LogicalTile> left_tuple(
left_tile, left_tile_itr, &hashed_col_ids);
// Find matching tuples in the hash table built on top of the right table
auto right_tuples = hash_table.find(left_tuple);
if (right_tuples != hash_table.end()) {
// Not yet supported due to assertion in gettomg right_tuples->first
// if (predicate_ != nullptr) {
// auto eval = predicate_->Evaluate(&left_tuple, &right_tuples->first,
// executor_context_);
// if (eval.IsFalse())
// continue;
// }
RecordMatchedLeftRow(left_result_tiles_.size() - 1, left_tile_itr);
// Go over the matching right tuples
for (auto &location : right_tuples->second) {
// Check if we got a new right tile itr
if (prev_tile != location.first) {
// Check if we have any join tuples
if (pos_lists_builder.Size() > 0) {
LOG_TRACE("Join tile size : %lu \n", pos_lists_builder.Size());
output_tile->SetPositionListsAndVisibility(
pos_lists_builder.Release());
buffered_output_tiles.push_back(output_tile.release());
}
// Get the logical tile from right child
LogicalTile *right_tile = right_result_tiles_[location.first].get();
// Build output logical tile
output_tile = BuildOutputLogicalTile(left_tile, right_tile);
// Build position lists
pos_lists_builder =
LogicalTile::PositionListsBuilder(left_tile, right_tile);
pos_lists_builder.SetRightSource(
&right_result_tiles_[location.first]->GetPositionLists());
}
// Add join tuple
pos_lists_builder.AddRow(left_tile_itr, location.second);
RecordMatchedRightRow(location.first, location.second);
// Cache prev logical tile itr
prev_tile = location.first;
}
}
}
// Check if we have any join tuples
if (pos_lists_builder.Size() > 0) {
LOG_TRACE("Join tile size : %lu \n", pos_lists_builder.Size());
output_tile->SetPositionListsAndVisibility(pos_lists_builder.Release());
buffered_output_tiles.push_back(output_tile.release());
}
// Check if we have any buffered output tiles
if (buffered_output_tiles.empty() == false) {
auto output_tile = buffered_output_tiles.front();
SetOutput(output_tile);
buffered_output_tiles.pop_front();
return true;
} else {
// Try again
continue;
}
}
}
/**
* @brief Creates logical tiles from the two input logical tiles after applying
* join predicate.
* @return true on success, false otherwise.
*/
bool HashJoinExecutor::DExecute() {
LOG_INFO("********** Hash Join %s Join executor :: 2 children \n",
GetJoinTypeString());
// If there is still something in the buffer, we return the first tile in it
if (buffered_output_tiles.size() > 0) {
SetOutput(buffered_output_tiles[0]);
buffered_output_tiles.pop_front();
return true;
}
//===--------------------------------------------------------------------===//
// Pick all right tiles
//===--------------------------------------------------------------------===//
if (right_child_done_ != true) {
// build hash table on right tiles
while (hash_executor_->Execute() == true) {
BufferRightTile(hash_executor_->GetOutput());
}
right_child_done_ = true;
}
if (right_result_tiles_.empty()) {
assert(left_result_tiles_.empty());
LOG_TRACE("Right child returned nothing. Exit.");
return false;
}
// Get the hash table from the hash executor
HashExecutor::HashMapType &hash_table_ = hash_executor_->GetHashTable();
// Get the address of column_ids
auto col_ids_address = &(hash_executor_->GetHashKeyIds());
// Loop until we have non-empty result join logical tile or exit
for (;;) {
// Build outer join output when done
if (left_child_done_) {
assert(right_child_done_ == true);
return BuildOuterJoinOutput();
}
//===--------------------------------------------------------------------===//
// Pick next left tiles
//===--------------------------------------------------------------------===//
LogicalTile *left_tile = nullptr;
// Left child is finished, no more tiles
if (children_[0]->Execute() == false) {
LOG_TRACE("Left child is exhausted. Returning false.");
// Left child exhausted.
// Release cur left tile. Clear right child's result buffer and return.
assert(right_result_tiles_.size() > 0);
left_child_done_ = true;
// hash_table_.clear();
return BuildOuterJoinOutput();
}
// Buffer the left child's result
else {
LOG_TRACE("Advance the left child.");
BufferLeftTile(children_[0]->GetOutput());
}
left_tile = left_result_tiles_.back().get();
//===--------------------------------------------------------------------===//
// Build Join Tile
//===--------------------------------------------------------------------===//
std::unordered_map<size_t, std::unique_ptr<LogicalTile>> output_tile_map;
std::unordered_map<size_t, LogicalTile::PositionListsBuilder>
pos_lists_builder_map;
// Go over the left logical tile
for (auto left_tile_row_itr : *left_tile) {
auto target_key = HashExecutor::HashMapType::key_type(
left_tile, left_tile_row_itr, col_ids_address);
auto got = hash_table_.find(target_key);
// There is no corresponding right tuples. Skip this left tuple and
// continue.
if (got == hash_table_.end()) {
continue;
}
// Go over the matching right tuples
// Key : container tuple with a subset of tuple attributes
// Value : < child_tile offset, tuple offset >
auto right_matched_tuples = got->second;
for (auto value : right_matched_tuples) {
// For Right and Full Outer Join
RecordMatchedRightRow(value.first, value.second);
// If this left-right pair has not been recorded
if (output_tile_map.find(value.first) == output_tile_map.end()) {
// we have to firstly update the right_tile,
// because this right tuple can belong to a tile which is
// different from the one we use to initialize our pos_lists_builder
LogicalTile *current_right_tile =
right_result_tiles_[value.first].get();
// Build and record output join logical tile
output_tile_map[value.first] =
BuildOutputLogicalTile(left_tile, current_right_tile);
// Build and record corresponding position lists
LogicalTile::PositionListsBuilder pos_lists_builder(
left_tile, current_right_tile);
pos_lists_builder_map[value.first] = pos_lists_builder;
}
// Insert a tuple into the correct output logical tile in the hashmap
pos_lists_builder_map[value.first].AddRow(left_tile_row_itr,
value.second);
}
// For Left and Full Outer Join
// if we get some matched right tuples for this left tuple,
// we remove it from the no_matching_left_row_sets_
RecordMatchedLeftRow(left_result_tiles_.size() - 1, left_tile_row_itr);
}
// fill the buffered_output_tiles with values in hashmap
for (auto iter = output_tile_map.begin(); iter != output_tile_map.end();
++iter) {
iter->second->SetPositionListsAndVisibility(
pos_lists_builder_map[iter->first].Release());
buffered_output_tiles.push_back(iter->second.release());
}
// Check if we have any join tuples
if (buffered_output_tiles.size() > 0) {
SetOutput(buffered_output_tiles[0]);
buffered_output_tiles.pop_front();
return true;
}
LOG_TRACE("This left tile produces empty join result. Continue the loop.");
}
}
/**
* @brief Creates logical tiles from the two input logical tiles after applying
* join predicate.
* @return true on success, false otherwise.
*/
bool MergeJoinExecutor::DExecute() {
LOG_INFO(
"********** Merge Join executor :: 2 children "
"left:: start: %lu, end: %lu, done: %d "
"right:: start: %lu, end: %lu, done: %d",
left_start_row, left_end_row, left_child_done_, right_start_row,
right_end_row, right_child_done_);
// Build outer join output when done
if (right_child_done_ && left_child_done_) {
return BuildOuterJoinOutput();
}
//===--------------------------------------------------------------------===//
// Pick right and left tiles
//===--------------------------------------------------------------------===//
// Try to get next tile from RIGHT child
if (((right_child_done_ == false) && (right_start_row == right_end_row)) ||
(left_child_done_ == true)) {
if (children_[1]->Execute() == false) {
LOG_TRACE("Did not get right tile ");
right_child_done_ = true;
// Try again
return DExecute();
}
LOG_TRACE("Got right tile ");
auto right_tile = children_[1]->GetOutput();
BufferRightTile(right_tile);
right_start_row = 0;
right_end_row = Advance(right_tile, right_start_row, false);
LOG_TRACE("size of right tiles: %lu", right_result_tiles_.size());
}
// Try to get next tile from LEFT child
if (((left_child_done_ == false) && (left_start_row == left_end_row)) ||
(right_child_done_ == true)) {
if (children_[0]->Execute() == false) {
LOG_TRACE("Did not get left tile ");
left_child_done_ = true;
// Try again
return DExecute();
}
LOG_TRACE("Got left tile ");
auto left_tile = children_[0]->GetOutput();
BufferLeftTile(left_tile);
left_start_row = 0;
left_end_row = Advance(left_tile, left_start_row, true);
LOG_TRACE("size of left tiles: %lu", left_result_tiles_.size());
}
// Check if we have logical tiles to process
if(left_result_tiles_.empty() || right_result_tiles_.empty()) {
return false;
}
LogicalTile *left_tile = left_result_tiles_.back().get();
LogicalTile *right_tile = right_result_tiles_.back().get();
//===--------------------------------------------------------------------===//
// Build Join Tile
//===--------------------------------------------------------------------===//
// Build output logical tile
auto output_tile = BuildOutputLogicalTile(left_tile, right_tile);
// Build position lists
LogicalTile::PositionListsBuilder pos_lists_builder(left_tile, right_tile);
while ((left_end_row > left_start_row) && (right_end_row > right_start_row)) {
expression::ContainerTuple<executor::LogicalTile> left_tuple(
left_tile, left_start_row);
expression::ContainerTuple<executor::LogicalTile> right_tuple(
right_tile, right_start_row);
bool not_matching_tuple_pair = false;
// Evaluate and compare the join clauses
for (auto &clause : *join_clauses_) {
auto left_value =
clause.left_->Evaluate(&left_tuple, &right_tuple, nullptr);
auto right_value =
clause.right_->Evaluate(&left_tuple, &right_tuple, nullptr);
// Compare the values
int comparison = left_value.Compare(right_value);
// Left key < Right key, advance left
if (comparison < 0) {
LOG_TRACE("left < right, advance left ");
left_start_row = left_end_row;
left_end_row = Advance(left_tile, left_start_row, true);
not_matching_tuple_pair = true;
break;
}
// Left key > Right key, advance right
else if (comparison > 0) {
LOG_TRACE("left > right, advance right ");
right_start_row = right_end_row;
right_end_row = Advance(right_tile, right_start_row, false);
not_matching_tuple_pair = true;
break;
}
// Left key == Right key, go and check next join clause
}
// Atleast one of the join clauses don't match
// One of the tile has been advanced
if (not_matching_tuple_pair) {
continue;
}
// Join clauses matched, try to match predicate
LOG_TRACE("one pair of tuples matches join clause ");
// Join predicate exists
if (predicate_ != nullptr) {
if (predicate_->Evaluate(&left_tuple, &right_tuple, executor_context_)
.IsFalse()) {
// Join predicate is false. Advance both.
left_start_row = left_end_row;
left_end_row = Advance(left_tile, left_start_row, true);
right_start_row = right_end_row;
right_end_row = Advance(right_tile, right_start_row, false);
}
}
// Sub tile matched, do a Cartesian product
// Go over every pair of tuples in left and right logical tiles
for (size_t left_tile_row_itr = left_start_row;
left_tile_row_itr < left_end_row; left_tile_row_itr++) {
for (size_t right_tile_row_itr = right_start_row;
right_tile_row_itr < right_end_row; right_tile_row_itr++) {
// Insert a tuple into the output logical tile
pos_lists_builder.AddRow(left_tile_row_itr, right_tile_row_itr);
RecordMatchedLeftRow(left_result_tiles_.size() - 1, left_tile_row_itr);
RecordMatchedRightRow(right_result_tiles_.size() - 1,
right_tile_row_itr);
}
}
// Then, advance both
left_start_row = left_end_row;
left_end_row = Advance(left_tile, left_start_row, true);
right_start_row = right_end_row;
right_end_row = Advance(right_tile, right_start_row, false);
}
// Check if we have any join tuples.
if (pos_lists_builder.Size() > 0) {
output_tile->SetPositionListsAndVisibility(pos_lists_builder.Release());
SetOutput(output_tile.release());
return true;
}
// Try again
else {
// If we are out of any more pairs of child tiles to examine,
// then we will return false earlier in this function
// So, no need to return false here
DExecute();
}
return true;
}
/**
* @brief Creates logical tiles from the two input logical tiles after applying
* join predicate.
* @return true on success, false otherwise.
*/
bool HashJoinExecutor::DExecute() {
// build hash map for right table
if (!right_child_done_) {
while (hash_executor_->Execute() == true)
BufferRightTile(children_[1]->GetOutput());
right_child_done_ = true;
}
for (;;) {
// left child & right child all done
if (left_child_done_ && right_child_done_) {
return BuildOuterJoinOutput();
}
// if there is remaining pairs in buffer, release one at a time.
if (!buffered_output_tiles.empty()) {
// just hand in one.
SetOutput(buffered_output_tiles.front());
buffered_output_tiles.pop_front();
return true;
}
// traverse every left child tile
if (children_[0]->Execute()) {
BufferLeftTile(children_[0]->GetOutput());
LogicalTile *left_tile = left_result_tiles_.back().get();
// traverse every tuple in curt left tile
for (auto left_tile_row_itr : *left_tile) {
auto hash = HashExecutor::HashMapType::key_type(
left_tile, left_tile_row_itr, &hash_executor_->GetHashKeyIds());
auto hash_result = hash_executor_->GetHashTable().find(hash);
if (hash_result != hash_executor_->GetHashTable().end()) {
RecordMatchedLeftRow(left_logical_tile_itr_, left_tile_row_itr);
// traverse right set
for (auto iter = hash_result->second.begin();
iter != hash_result->second.end(); ++iter) {
auto tile_index = iter->first;
auto tuple_index = iter->second;
RecordMatchedRightRow(tile_index, tuple_index);
LogicalTile *right_tile = right_result_tiles_[tile_index].get();
LogicalTile::PositionListsBuilder pos_lists_builder(left_tile,
right_tile);
pos_lists_builder.AddRow(left_tile_row_itr, tuple_index);
auto output_tile = BuildOutputLogicalTile(left_tile, right_tile);
output_tile->SetPositionListsAndVisibility(
pos_lists_builder.Release());
buffered_output_tiles.emplace_back(output_tile.release());
} // end of traversing right set
} // end of if match
} // end of traversal of curt left_tile
// Release at most one pair.
// PS: This should be done after traversing all the tuples in curt tile
left_logical_tile_itr_++;
if (!buffered_output_tiles.empty()) {
// release one at a time
SetOutput(buffered_output_tiles.front());
buffered_output_tiles.pop_front();
return true;
}
} // end of still have left tile
// All left tiles are exhausted.
else {
left_child_done_ = true;
return BuildOuterJoinOutput();
}
} // end of infinite loop
// never should go here
return false;
} // end of DExecute
/**
* @brief Creates logical tiles from the two input logical tiles after applying
* join predicate.
* @return true on success, false otherwise.
*/
bool HashJoinExecutor::DExecute() {
LOG_INFO("Hash Join Executor");
// Loop until we have non-empty result join logical tile or exit
while (true) {
// if (!buffered_output_tiles.empty()) {
if (!result.empty()) {
auto* output_tile = result.back();
result.pop_back();
SetOutput(output_tile);
return true;
}
// Build outer join output when done
if (left_child_done_ && right_child_done_) {
return BuildOuterJoinOutput();
}
//===--------------------------------------------------------------------===//
// Pick right and left tiles
//===--------------------------------------------------------------------===//
// Get all the logical tiles from RIGHT child
if (!right_child_done_) {
while (children_[1]->Execute()) {
BufferRightTile(children_[1]->GetOutput());
}
right_child_done_ = true;
LOG_INFO("Hash Join Executor: Got all %lu right tiles.",
right_result_tiles_.size());
}
// Get next logical tile from LEFT child
if (children_[0]->Execute()) {
BufferLeftTile(children_[0]->GetOutput());
LOG_INFO("Hash Join Executor: Got left tile %p.",
left_result_tiles_.back().get());
} else {
// Left input is exhausted, loop around
left_child_done_ = true;
return BuildOuterJoinOutput();
}
if (right_result_tiles_.empty()) {
/// No right children, a hash lookup would be empty. Continue ...
continue;
}
//===--------------------------------------------------------------------===//
// Build Join Tile
//===--------------------------------------------------------------------===//
LogicalTile* left_tile = left_result_tiles_.back().get();
std::unordered_map<size_t,
std::unique_ptr<LogicalTile::PositionListsBuilder>>
right_matches;
// Get the hash table from the hash executor
auto& hash_table = hash_executor_->GetHashTable();
auto& hash_columns = hash_executor_->GetHashKeyIds();
for (oid_t left_tid : *left_tile) {
/// Create key and probe hash table
HashExecutor::HashMapType::key_type key(left_tile, left_tid,
&hash_columns);
const auto& iter = hash_table.find(key);
if (iter == hash_table.end()) {
continue;
}
auto& matches = iter->second;
for (auto& match : matches) {
auto right_tile_index = match.first;
auto* right_tile = right_result_tiles_[right_tile_index].get();
auto right_tid = match.second;
RecordMatchedRightRow(right_tile_index, right_tid);
RecordMatchedLeftRow(left_result_tiles_.size() - 1, left_tid);
const auto& pos_match_iter = right_matches.find(right_tile_index);
if (pos_match_iter == right_matches.end()) {
std::unique_ptr<LogicalTile::PositionListsBuilder> builder{
new LogicalTile::PositionListsBuilder(left_tile, right_tile)};
right_matches.insert(
std::make_pair(right_tile_index, std::move(builder)));
}
right_matches[right_tile_index]->AddRow(left_tid, right_tid);
}
}
// Create a new logical tile for every grouped match in matches
for (auto& iter : right_matches) {
auto output_tile = BuildOutputLogicalTile(
left_tile, right_result_tiles_[iter.first].get());
auto& pos_lists_builder = iter.second;
output_tile->SetPositionListsAndVisibility(pos_lists_builder->Release());
result.push_back(output_tile.release());
}
}
}