void BuildIndex(std::shared_ptr<index::Index> index,
storage::DataTable *table) {
auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance();
auto txn = txn_manager.BeginTransaction();
oid_t start_tile_group_count = START_OID;
oid_t table_tile_group_count = table->GetTileGroupCount();
while (start_tile_group_count < table_tile_group_count) {
auto tile_group = table->GetTileGroup(start_tile_group_count++);
auto column_count = table->GetSchema()->GetColumnCount();
oid_t active_tuple_count = tile_group->GetNextTupleSlot();
for (oid_t tuple_id = 0; tuple_id < active_tuple_count; tuple_id++) {
std::unique_ptr<storage::Tuple> tuple_ptr(
new storage::Tuple(table->GetSchema(), true));
CopyTuple(tuple_id, tuple_ptr.get(), tile_group.get(), column_count);
ItemPointer location(tile_group->GetTileGroupId(), tuple_id);
ItemPointer *index_entry_ptr = nullptr;
table->InsertInIndexes(tuple_ptr.get(), location, txn, &index_entry_ptr);
}
index->IncrementIndexedTileGroupOffset();
}
txn_manager.CommitTransaction(txn);
}
const std::string DataTable::GetInfo() const {
std::ostringstream os;
// os << "=====================================================\n";
// os << "TABLE :\n";
oid_t tile_group_count = GetTileGroupCount();
// os << "Tile Group Count : " << tile_group_count << "\n";
oid_t tuple_count = 0;
oid_t table_id = 0;
for (oid_t tile_group_itr = 0; tile_group_itr < tile_group_count;
tile_group_itr++) {
auto tile_group = GetTileGroup(tile_group_itr);
table_id = tile_group->GetTableId();
auto tile_tuple_count = tile_group->GetNextTupleSlot();
// os << "Tile Group Id : " << tile_group_itr
// << " Tuple Count : " << tile_tuple_count << "\n";
// os << (*tile_group);
tuple_count += tile_tuple_count;
}
os << "Table " << table_id << " Tuple Count :: " << tuple_count << "\n";
// os << "=====================================================\n";
return os.str();
}
void TileGroup::SetValue(type::Value &value, oid_t tuple_id,
oid_t column_id) {
PL_ASSERT(tuple_id < GetNextTupleSlot());
oid_t tile_column_id, tile_offset;
LocateTileAndColumn(column_id, tile_offset, tile_column_id);
GetTile(tile_offset)->SetValue(value, tuple_id, tile_column_id);
}
void IndexTuner::BuildIndex(storage::DataTable* table,
std::shared_ptr<index::Index> index) {
auto table_schema = table->GetSchema();
auto index_tile_group_offset = index->GetIndexedTileGroupOff();
auto table_tile_group_count = table->GetTileGroupCount();
oid_t tile_groups_indexed = 0;
auto index_schema = index->GetKeySchema();
auto indexed_columns = index_schema->GetIndexedColumns();
std::unique_ptr<storage::Tuple> key(new storage::Tuple(index_schema, true));
while (index_tile_group_offset < table_tile_group_count &&
(tile_groups_indexed < max_tile_groups_indexed)) {
std::unique_ptr<storage::Tuple> tuple_ptr(
new storage::Tuple(table_schema, true));
auto tile_group = table->GetTileGroup(index_tile_group_offset);
auto tile_group_id = tile_group->GetTileGroupId();
oid_t active_tuple_count = tile_group->GetNextTupleSlot();
for (oid_t tuple_id = 0; tuple_id < active_tuple_count; tuple_id++) {
// Copy over the tuple
tile_group->CopyTuple(tuple_id, tuple_ptr.get());
// Set the location
ItemPointer location(tile_group_id, tuple_id);
// Set the key
key->SetFromTuple(tuple_ptr.get(), indexed_columns, index->GetPool());
// Insert in specific index
// index->InsertEntry(key.get(), location);
}
// Update indexed tile group offset (set of tgs indexed)
index->IncrementIndexedTileGroupOffset();
// Sleep a bit
// std::this_thread::sleep_for(std::chrono::microseconds(sleep_duration));
index_tile_group_offset++;
tile_groups_indexed++;
}
}
// count number of expired versions.
int GarbageNum(storage::DataTable *table) {
auto table_tile_group_count_ = table->GetTileGroupCount();
auto current_tile_group_offset_ = START_OID;
int old_num = 0;
while (current_tile_group_offset_ < table_tile_group_count_) {
auto tile_group = table->GetTileGroup(current_tile_group_offset_++);
auto tile_group_header = tile_group->GetHeader();
oid_t active_tuple_count = tile_group->GetNextTupleSlot();
for (oid_t tuple_id = 0; tuple_id < active_tuple_count; tuple_id++) {
auto tuple_txn_id = tile_group_header->GetTransactionId(tuple_id);
auto tuple_end_cid = tile_group_header->GetEndCommitId(tuple_id);
if (tuple_txn_id == INITIAL_TXN_ID && tuple_end_cid != MAX_CID) {
old_num++;
}
}
}
LOG_INFO("old version num = %d", old_num);
return old_num;
}
/**
* @brief Creates logical tile from tile group and applies scan predicate.
* @return true on success, false otherwise.
*/
bool SeqScanExecutor::DExecute() {
// Scanning over a logical tile.
if (children_.size() == 1) {
// FIXME Check all requirements for children_.size() == 0 case.
LOG_TRACE("Seq Scan executor :: 1 child ");
PL_ASSERT(target_table_ == nullptr);
PL_ASSERT(column_ids_.size() == 0);
while (children_[0]->Execute()) {
std::unique_ptr<LogicalTile> tile(children_[0]->GetOutput());
if (predicate_ != nullptr) {
// Invalidate tuples that don't satisfy the predicate.
for (oid_t tuple_id : *tile) {
expression::ContainerTuple<LogicalTile> tuple(tile.get(), tuple_id);
if (predicate_->Evaluate(&tuple, nullptr, executor_context_)
.IsFalse()) {
tile->RemoveVisibility(tuple_id);
}
}
}
if (0 == tile->GetTupleCount()) { // Avoid returning empty tiles
continue;
}
/* Hopefully we needn't do projections here */
SetOutput(tile.release());
return true;
}
return false;
}
// Scanning a table
else if (children_.size() == 0) {
LOG_TRACE("Seq Scan executor :: 0 child ");
PL_ASSERT(target_table_ != nullptr);
PL_ASSERT(column_ids_.size() > 0);
// Force to use occ txn manager if dirty read is forbidden
concurrency::TransactionManager &transaction_manager =
concurrency::TransactionManagerFactory::GetInstance();
// LOG_TRACE("Number of tuples: %f",
// target_table_->GetIndex(0)->GetNumberOfTuples());
// Retrieve next tile group.
while (current_tile_group_offset_ < table_tile_group_count_) {
auto tile_group =
target_table_->GetTileGroup(current_tile_group_offset_++);
auto tile_group_header = tile_group->GetHeader();
oid_t active_tuple_count = tile_group->GetNextTupleSlot();
// Construct position list by looping through tile group
// and applying the predicate.
std::vector<oid_t> position_list;
for (oid_t tuple_id = 0; tuple_id < active_tuple_count; tuple_id++) {
ItemPointer location(tile_group->GetTileGroupId(), tuple_id);
// check transaction visibility
if (transaction_manager.IsVisible(tile_group_header, tuple_id)) {
// if the tuple is visible, then perform predicate evaluation.
if (predicate_ == nullptr) {
position_list.push_back(tuple_id);
auto res = transaction_manager.PerformRead(location);
if (!res) {
transaction_manager.SetTransactionResult(RESULT_FAILURE);
return res;
}
} else {
expression::ContainerTuple<storage::TileGroup> tuple(
tile_group.get(), tuple_id);
auto eval = predicate_->Evaluate(&tuple, nullptr, executor_context_)
.IsTrue();
if (eval == true) {
position_list.push_back(tuple_id);
auto res = transaction_manager.PerformRead(location);
if (!res) {
transaction_manager.SetTransactionResult(RESULT_FAILURE);
return res;
}
}
}
}
}
// Don't return empty tiles
if (position_list.size() == 0) {
continue;
}
// Construct logical tile.
std::unique_ptr<LogicalTile> logical_tile(LogicalTileFactory::GetTile());
logical_tile->AddColumns(tile_group, column_ids_);
logical_tile->AddPositionList(std::move(position_list));
SetOutput(logical_tile.release());
return true;
}
}
return false;
}
Value TileGroup::GetValue(oid_t tuple_id, oid_t column_id) {
PL_ASSERT(tuple_id < GetNextTupleSlot());
oid_t tile_column_id, tile_offset;
LocateTileAndColumn(column_id, tile_offset, tile_column_id);
return GetTile(tile_offset)->GetValue(tuple_id, tile_column_id);
}
void TileGroupHeader::PrintVisibility(txn_id_t txn_id, cid_t at_cid) {
oid_t active_tuple_slots = GetNextTupleSlot();
std::stringstream os;
os << "\t-----------------------------------------------------------\n";
for (oid_t header_itr = 0; header_itr < active_tuple_slots; header_itr++) {
bool own = (txn_id == GetTransactionId(header_itr));
bool activated = (at_cid >= GetBeginCommitId(header_itr));
bool invalidated = (at_cid >= GetEndCommitId(header_itr));
txn_id_t txn_id = GetTransactionId(header_itr);
cid_t beg_commit_id = GetBeginCommitId(header_itr);
cid_t end_commit_id = GetEndCommitId(header_itr);
bool insert_commit = GetInsertCommit(header_itr);
bool delete_commit = GetDeleteCommit(header_itr);
int width = 10;
os << "\tslot :: " << std::setw(width) << header_itr;
os << " txn id : ";
if (txn_id == MAX_TXN_ID)
os << std::setw(width) << "MAX_TXN_ID";
else
os << std::setw(width) << txn_id;
os << " beg cid : ";
if (beg_commit_id == MAX_CID)
os << std::setw(width) << "MAX_CID";
else
os << std::setw(width) << beg_commit_id;
os << " end cid : ";
if (end_commit_id == MAX_CID)
os << std::setw(width) << "MAX_CID";
else
os << std::setw(width) << end_commit_id;
os << " insert commit : ";
if (insert_commit == true)
os << "O";
else
os << "X";
os << " delete commit : ";
if (delete_commit == true)
os << "O";
else
os << "X";
peloton::ItemPointer location = GetPrevItemPointer(header_itr);
os << " prev : "
<< "[ " << location.block << " , " << location.offset << " ]"; //<<
os << " own : " << own;
os << " activated : " << activated;
os << " invalidated : " << invalidated << " ";
// Visible iff past Insert || Own Insert
if ((!own && activated && !invalidated) ||
(own && !activated && !invalidated))
os << "\t\t[ true ]\n";
else
os << "\t\t[ false ]\n";
}
os << "\t-----------------------------------------------------------\n";
std::cout << os.str().c_str();
}
bool HybridScanExecutor::SeqScanUtil() {
assert(children_.size() == 0);
// LOG_INFO("Hybrid executor, Seq Scan :: 0 child");
assert(table_ != nullptr);
assert(column_ids_.size() > 0);
auto &transaction_manager =
concurrency::TransactionManagerFactory::GetInstance();
// Retrieve next tile group.
while (current_tile_group_offset_ < table_tile_group_count_) {
auto tile_group =
table_->GetTileGroup(current_tile_group_offset_++);
auto tile_group_header = tile_group->GetHeader();
oid_t active_tuple_count = tile_group->GetNextTupleSlot();
// Construct position list by looping through tile group
// and applying the predicate.
oid_t upper_bound_block = 0;
if (item_pointers_.size() > 0) {
auto reverse_iter = item_pointers_.rbegin();
upper_bound_block = reverse_iter->block;
}
std::vector<oid_t> position_list;
for (oid_t tuple_id = 0; tuple_id < active_tuple_count; tuple_id++) {
ItemPointer location(tile_group->GetTileGroupId(), tuple_id);
if (type_ == planner::HYBRID &&
item_pointers_.size() > 0 &&
location.block <= upper_bound_block) {
if (item_pointers_.find(location) != item_pointers_.end()) {
continue;
}
}
// check transaction visibility
if (transaction_manager.IsVisible(tile_group_header, tuple_id)) {
// if the tuple is visible, then perform predicate evaluation.
if (predicate_ == nullptr) {
position_list.push_back(tuple_id);
} else {
expression::ContainerTuple<storage::TileGroup> tuple(
tile_group.get(), tuple_id);
auto eval = predicate_->Evaluate(&tuple, nullptr, executor_context_)
.IsTrue();
if (eval == true) {
position_list.push_back(tuple_id);
}
}
} else {
expression::ContainerTuple<storage::TileGroup> tuple(
tile_group.get(), tuple_id);
auto eval = predicate_->Evaluate(&tuple, nullptr, executor_context_)
.IsTrue();
if (eval == true) {
position_list.push_back(tuple_id);
auto res = transaction_manager.PerformRead(location);
if (!res) {
transaction_manager.SetTransactionResult(RESULT_FAILURE);
return res;
}
}
}
}
// Don't return empty tiles
if (position_list.size() == 0) {
continue;
}
// Construct logical tile.
std::unique_ptr<LogicalTile> logical_tile(LogicalTileFactory::GetTile());
logical_tile->AddColumns(tile_group, column_ids_);
logical_tile->AddPositionList(std::move(position_list));
LOG_INFO("Hybrid executor, Seq Scan :: Got a logical tile");
SetOutput(logical_tile.release());
return true;
}
return false;
}
/**
* @brief Creates logical tile from tile group and applies scan predicate.
* @return true on success, false otherwise.
*/
bool SeqScanExecutor::DExecute() {
// Scanning over a logical tile.
if (children_.size() == 1 &&
// There will be a child node on the create index scenario,
// but we don't want to use this execution flow
!(GetRawNode()->GetChildren().size() > 0 &&
GetRawNode()->GetChildren()[0].get()->GetPlanNodeType() ==
PlanNodeType::CREATE &&
((planner::CreatePlan *)GetRawNode()->GetChildren()[0].get())
->GetCreateType() == CreateType::INDEX)) {
// FIXME Check all requirements for children_.size() == 0 case.
LOG_TRACE("Seq Scan executor :: 1 child ");
PELOTON_ASSERT(target_table_ == nullptr);
PELOTON_ASSERT(column_ids_.size() == 0);
while (children_[0]->Execute()) {
std::unique_ptr<LogicalTile> tile(children_[0]->GetOutput());
if (predicate_ != nullptr) {
// Invalidate tuples that don't satisfy the predicate.
for (oid_t tuple_id : *tile) {
ContainerTuple<LogicalTile> tuple(tile.get(), tuple_id);
auto eval = predicate_->Evaluate(&tuple, nullptr, executor_context_);
if (eval.IsFalse()) {
// if (predicate_->Evaluate(&tuple, nullptr, executor_context_)
// .IsFalse()) {
tile->RemoveVisibility(tuple_id);
}
}
}
if (0 == tile->GetTupleCount()) { // Avoid returning empty tiles
continue;
}
/* Hopefully we needn't do projections here */
SetOutput(tile.release());
return true;
}
return false;
}
// Scanning a table
else if (children_.size() == 0 ||
// If we are creating an index, there will be a child
(children_.size() == 1 &&
// This check is only needed to pass seq_scan_test
// unless it is possible to add a executor child
// without a corresponding plan.
GetRawNode()->GetChildren().size() > 0 &&
// Check if the plan is what we actually expect.
GetRawNode()->GetChildren()[0].get()->GetPlanNodeType() ==
PlanNodeType::CREATE &&
// If it is, confirm it is for indexes
((planner::CreatePlan *)GetRawNode()->GetChildren()[0].get())
->GetCreateType() == CreateType::INDEX)) {
LOG_TRACE("Seq Scan executor :: 0 child ");
PELOTON_ASSERT(target_table_ != nullptr);
PELOTON_ASSERT(column_ids_.size() > 0);
if (children_.size() > 0 && !index_done_) {
children_[0]->Execute();
// This stops continuous executions due to
// a parent and avoids multiple creations
// of the same index.
index_done_ = true;
}
concurrency::TransactionManager &transaction_manager =
concurrency::TransactionManagerFactory::GetInstance();
bool acquire_owner = GetPlanNode<planner::AbstractScan>().IsForUpdate();
auto current_txn = executor_context_->GetTransaction();
// Retrieve next tile group.
while (current_tile_group_offset_ < table_tile_group_count_) {
auto tile_group =
target_table_->GetTileGroup(current_tile_group_offset_++);
auto tile_group_header = tile_group->GetHeader();
oid_t active_tuple_count = tile_group->GetNextTupleSlot();
// Construct position list by looping through tile group
// and applying the predicate.
std::vector<oid_t> position_list;
for (oid_t tuple_id = 0; tuple_id < active_tuple_count; tuple_id++) {
ItemPointer location(tile_group->GetTileGroupId(), tuple_id);
auto visibility = transaction_manager.IsVisible(
current_txn, tile_group_header, tuple_id);
// check transaction visibility
if (visibility == VisibilityType::OK) {
// if the tuple is visible, then perform predicate evaluation.
if (predicate_ == nullptr) {
position_list.push_back(tuple_id);
auto res = transaction_manager.PerformRead(current_txn, location,
acquire_owner);
if (!res) {
transaction_manager.SetTransactionResult(current_txn,
ResultType::FAILURE);
return res;
}
} else {
ContainerTuple<storage::TileGroup> tuple(tile_group.get(),
tuple_id);
LOG_TRACE("Evaluate predicate for a tuple");
auto eval =
predicate_->Evaluate(&tuple, nullptr, executor_context_);
LOG_TRACE("Evaluation result: %s", eval.GetInfo().c_str());
if (eval.IsTrue()) {
position_list.push_back(tuple_id);
auto res = transaction_manager.PerformRead(current_txn, location,
acquire_owner);
if (!res) {
transaction_manager.SetTransactionResult(current_txn,
ResultType::FAILURE);
return res;
} else {
LOG_TRACE("Sequential Scan Predicate Satisfied");
}
}
}
}
}
// Don't return empty tiles
if (position_list.size() == 0) {
continue;
}
// Construct logical tile.
std::unique_ptr<LogicalTile> logical_tile(LogicalTileFactory::GetTile());
logical_tile->AddColumns(tile_group, column_ids_);
logical_tile->AddPositionList(std::move(position_list));
LOG_TRACE("Information %s", logical_tile->GetInfo().c_str());
SetOutput(logical_tile.release());
return true;
}
}
return false;
}
/**
* @brief Creates logical tile from tile group and applies scan predicate.
* @return true on success, false otherwise.
*/
bool SeqScanExecutor::DExecute() {
// Scanning over a logical tile.
if (children_.size() == 1) {
// FIXME Check all requirements for children_.size() == 0 case.
LOG_TRACE("Seq Scan executor :: 1 child \n");
assert(target_table_ == nullptr);
assert(column_ids_.size() == 0);
while (children_[0]->Execute()) {
std::unique_ptr<LogicalTile> tile(children_[0]->GetOutput());
if (predicate_ != nullptr) {
// Invalidate tuples that don't satisfy the predicate.
for (oid_t tuple_id : *tile) {
expression::ContainerTuple<LogicalTile> tuple(tile.get(), tuple_id);
if (predicate_->Evaluate(&tuple, nullptr, executor_context_)
.IsFalse()) {
tile->RemoveVisibility(tuple_id);
}
}
}
if (0 == tile->GetTupleCount()) { // Avoid returning empty tiles
continue;
}
/* Hopefully we needn't do projections here */
SetOutput(tile.release());
return true;
}
return false;
}
// Scanning a table
else if (children_.size() == 0) {
LOG_TRACE("Seq Scan executor :: 0 child \n");
assert(target_table_ != nullptr);
assert(column_ids_.size() > 0);
// Retrieve next tile group.
while (current_tile_group_offset_ < table_tile_group_count_) {
auto tile_group =
target_table_->GetTileGroup(current_tile_group_offset_++);
storage::TileGroupHeader *tile_group_header = tile_group->GetHeader();
auto transaction_ = executor_context_->GetTransaction();
txn_id_t txn_id = transaction_->GetTransactionId();
cid_t commit_id = transaction_->GetLastCommitId();
oid_t active_tuple_count = tile_group->GetNextTupleSlot();
// Print tile group visibility
// tile_group_header->PrintVisibility(txn_id, commit_id);
// Construct logical tile.
std::unique_ptr<LogicalTile> logical_tile(LogicalTileFactory::GetTile());
logical_tile->AddColumns(tile_group, column_ids_);
// Construct position list by looping through tile group
// and applying the predicate.
std::vector<oid_t> position_list;
for (oid_t tuple_id = 0; tuple_id < active_tuple_count; tuple_id++) {
if (tile_group_header->IsVisible(tuple_id, txn_id, commit_id) ==
false) {
continue;
}
expression::ContainerTuple<storage::TileGroup> tuple(tile_group.get(),
tuple_id);
if (predicate_ == nullptr) {
position_list.push_back(tuple_id);
} else {
auto eval =
predicate_->Evaluate(&tuple, nullptr, executor_context_).IsTrue();
if (eval == true) position_list.push_back(tuple_id);
}
}
logical_tile->AddPositionList(std::move(position_list));
// Don't return empty tiles
if (0 == logical_tile->GetTupleCount()) {
continue;
}
SetOutput(logical_tile.release());
return true;
}
}
return false;
}
