// Do insert and create insert tuple log records std::vector<ItemPointer> InsertTuples( storage::DataTable* table, const std::vector<storage::Tuple*>& tuples, bool committed) { std::vector<ItemPointer> locations; auto& txn_manager = concurrency::TransactionManagerFactory::GetInstance(); for (auto tuple : tuples) { auto txn = txn_manager.BeginTransaction(); ItemPointer location = table->InsertTuple(tuple); if (location.block == INVALID_OID) { txn->SetResult(Result::RESULT_FAILURE); LOG_ERROR("Insert failed"); exit(EXIT_FAILURE); } txn->RecordInsert(location); locations.push_back(location); // Logging { auto& log_manager = logging::LogManager::GetInstance(); if (log_manager.IsInLoggingMode()) { auto logger = log_manager.GetBackendLogger(); auto record = logger->GetTupleRecord( LOGRECORD_TYPE_TUPLE_INSERT, txn->GetTransactionId(), table->GetOid(), location, INVALID_ITEMPOINTER, tuple, LOGGING_TESTS_DATABASE_OID); logger->Log(record); } } // commit or abort as required if (committed) { txn_manager.CommitTransaction(); } else { txn_manager.AbortTransaction(); } } return locations; }
void PessimisticTxnManager::PerformDelete(const oid_t &tile_group_id, const oid_t &tuple_id) { auto &manager = catalog::Manager::GetInstance(); auto tile_group_header = manager.GetTileGroup(tile_group_id)->GetHeader(); assert(tile_group_header->GetTransactionId(tuple_id) == current_txn->GetTransactionId()); assert(tile_group_header->GetBeginCommitId(tuple_id) == MAX_CID); assert(tile_group_header->GetEndCommitId(tuple_id) == MAX_CID); tile_group_header->SetEndCommitId(tuple_id, INVALID_CID); // Add the old tuple into the delete set auto old_location = tile_group_header->GetPrevItemPointer(tuple_id); if (old_location.IsNull() == false) { // delete an inserted version current_txn->RecordDelete(old_location.block, old_location.offset); } else { // if this version is newly inserted. current_txn->RecordDelete(tile_group_id, tuple_id); } }
// count number of expired versions. int GarbageNum(storage::DataTable *table) { auto table_tile_group_count_ = table->GetTileGroupCount(); auto current_tile_group_offset_ = START_OID; int old_num = 0; while (current_tile_group_offset_ < table_tile_group_count_) { auto tile_group = table->GetTileGroup(current_tile_group_offset_++); auto tile_group_header = tile_group->GetHeader(); oid_t active_tuple_count = tile_group->GetNextTupleSlot(); for (oid_t tuple_id = 0; tuple_id < active_tuple_count; tuple_id++) { auto tuple_txn_id = tile_group_header->GetTransactionId(tuple_id); auto tuple_end_cid = tile_group_header->GetEndCommitId(tuple_id); if (tuple_txn_id == INITIAL_TXN_ID && tuple_end_cid != MAX_CID) { old_num++; } } } LOG_INFO("old version num = %d", old_num); return old_num; }
void TimestampOrderingTransactionManager::PerformInsert( TransactionContext *const current_txn, const ItemPointer &location, ItemPointer *index_entry_ptr) { PL_ASSERT(current_txn->GetIsolationLevel() != IsolationLevelType::READ_ONLY); oid_t tile_group_id = location.block; oid_t tuple_id = location.offset; auto &manager = catalog::Manager::GetInstance(); auto tile_group_header = manager.GetTileGroup(tile_group_id)->GetHeader(); auto transaction_id = current_txn->GetTransactionId(); // check MVCC info // the tuple slot must be empty. PL_ASSERT(tile_group_header->GetTransactionId(tuple_id) == INVALID_TXN_ID); PL_ASSERT(tile_group_header->GetBeginCommitId(tuple_id) == MAX_CID); PL_ASSERT(tile_group_header->GetEndCommitId(tuple_id) == MAX_CID); tile_group_header->SetTransactionId(tuple_id, transaction_id); // no need to set next item pointer. // Add the new tuple into the insert set current_txn->RecordInsert(location); InitTupleReserved(tile_group_header, tuple_id); // Write down the head pointer's address in tile group header tile_group_header->SetIndirection(tuple_id, index_entry_ptr); // Increment table insert op stats if (static_cast<StatsType>(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { stats::BackendStatsContext::GetInstance()->IncrementTableInserts( location.block); } }
/** * @brief Populates the tiles in the given tile-group in a specific manner. * @param tile_group Tile-group to populate with values. * @param num_rows Number of tuples to insert. */ void ExecutorTestsUtil::PopulateTiles( std::shared_ptr<storage::TileGroup> tile_group, int num_rows) { // Create tuple schema from tile schemas. std::vector<catalog::Schema> &tile_schemas = tile_group->GetTileSchemas(); std::unique_ptr<catalog::Schema> schema( catalog::Schema::AppendSchemaList(tile_schemas)); // Ensure that the tile group is as expected. assert(schema->GetColumnCount() == 4); // Insert tuples into tile_group. auto &txn_manager = concurrency::TransactionManager::GetInstance(); const bool allocate = true; auto txn = txn_manager.BeginTransaction(); const txn_id_t txn_id = txn->GetTransactionId(); const cid_t commit_id = txn->GetCommitId(); auto testing_pool = TestingHarness::GetInstance().GetTestingPool(); for (int col_itr = 0; col_itr < num_rows; col_itr++) { storage::Tuple tuple(schema.get(), allocate); tuple.SetValue(0, ValueFactory::GetIntegerValue(PopulatedValue(col_itr, 0)), testing_pool); tuple.SetValue(1, ValueFactory::GetIntegerValue(PopulatedValue(col_itr, 1)), testing_pool); tuple.SetValue(2, ValueFactory::GetDoubleValue(PopulatedValue(col_itr, 2)), testing_pool); Value string_value = ValueFactory::GetStringValue( std::to_string(PopulatedValue(col_itr, 3))); tuple.SetValue(3, string_value, testing_pool); oid_t tuple_slot_id = tile_group->InsertTuple(txn_id, &tuple); tile_group->CommitInsertedTuple(tuple_slot_id, txn_id, commit_id); } txn_manager.CommitTransaction(); }
void DeleteTuple(storage::DataTable *table, oid_t id){ auto txn = peloton::concurrency::current_txn; std::unique_ptr<executor::ExecutorContext> context( new executor::ExecutorContext(txn)); LOG_INFO("Removing tuple with id %d from table %s", (int)id, table->GetName().c_str()); LOG_INFO("Transaction ID: %d", (int)txn->GetTransactionId()); // Delete planner::DeletePlan delete_node(table, false); executor::DeleteExecutor delete_executor(&delete_node, context.get()); // Predicate // WHERE id_in_table = id expression::TupleValueExpression *tup_val_exp = new expression::TupleValueExpression(VALUE_TYPE_INTEGER, 0, 0); expression::ConstantValueExpression *const_val_exp = new expression::ConstantValueExpression( ValueFactory::GetIntegerValue(id)); auto predicate = new expression::ComparisonExpression<expression::CmpEq>( EXPRESSION_TYPE_COMPARE_EQUAL, tup_val_exp, const_val_exp); // Seq scan std::vector<oid_t> column_ids = {0, 1}; std::unique_ptr<planner::SeqScanPlan> seq_scan_node( new planner::SeqScanPlan(table, predicate, column_ids)); executor::SeqScanExecutor seq_scan_executor(seq_scan_node.get(), context.get()); // Parent-Child relationship delete_node.AddChild(std::move(seq_scan_node)); delete_executor.AddChild(&seq_scan_executor); delete_executor.Init(); delete_executor.Execute(); }
bool IndexScanExecutor::ExecPrimaryIndexLookup() { PL_ASSERT(!done_); std::vector<ItemPointer *> tuple_location_ptrs; PL_ASSERT(index_->GetIndexType() == INDEX_CONSTRAINT_TYPE_PRIMARY_KEY); if (0 == key_column_ids_.size()) { index_->ScanAllKeys(tuple_location_ptrs); } else { index_->Scan(values_, key_column_ids_, expr_types_, SCAN_DIRECTION_TYPE_FORWARD, tuple_location_ptrs); } if (tuple_location_ptrs.size() == 0) return false; auto &transaction_manager = concurrency::TransactionManagerFactory::GetInstance(); std::map<oid_t, std::vector<oid_t>> visible_tuples; std::vector<ItemPointer> garbage_tuples; // for every tuple that is found in the index. for (auto tuple_location_ptr : tuple_location_ptrs) { ItemPointer tuple_location = *tuple_location_ptr; auto &manager = catalog::Manager::GetInstance(); auto tile_group = manager.GetTileGroup(tuple_location.block); auto tile_group_header = tile_group.get()->GetHeader(); size_t chain_length = 0; while (true) { ++chain_length; // if the tuple is visible. if (transaction_manager.IsVisible(tile_group_header, tuple_location.offset)) { LOG_TRACE("traverse chain length : %lu", chain_length); LOG_TRACE("perform read: %u, %u", tuple_location.block, tuple_location.offset); // perform predicate evaluation. if (predicate_ == nullptr) { visible_tuples[tuple_location.block].push_back(tuple_location.offset); auto res = transaction_manager.PerformRead(tuple_location); if (!res) { transaction_manager.SetTransactionResult(RESULT_FAILURE); return res; } } else { expression::ContainerTuple<storage::TileGroup> tuple( tile_group.get(), tuple_location.offset); auto eval = predicate_->Evaluate(&tuple, nullptr, executor_context_).IsTrue(); if (eval == true) { visible_tuples[tuple_location.block] .push_back(tuple_location.offset); auto res = transaction_manager.PerformRead(tuple_location); if (!res) { transaction_manager.SetTransactionResult(RESULT_FAILURE); return res; } } } break; } // if the tuple is not visible. else { ItemPointer old_item = tuple_location; cid_t old_end_cid = tile_group_header->GetEndCommitId(old_item.offset); tuple_location = tile_group_header->GetNextItemPointer(old_item.offset); // there must exist a visible version. // FIXME: currently, only speculative read transaction manager **may** see a null version // it's a potential bug if(tuple_location.IsNull()) { transaction_manager.SetTransactionResult(RESULT_FAILURE); // FIXME: this cause unnecessary abort when we have delete operations return false; } // FIXME: Is this always true? what if we have a deleted tuple? --jiexi PL_ASSERT(tuple_location.IsNull() == false); cid_t max_committed_cid = transaction_manager.GetMaxCommittedCid(); // check whether older version is garbage. if (old_end_cid <= max_committed_cid) { PL_ASSERT(tile_group_header->GetTransactionId(old_item.offset) == INITIAL_TXN_ID || tile_group_header->GetTransactionId(old_item.offset) == INVALID_TXN_ID); if (tile_group_header->SetAtomicTransactionId(old_item.offset, INVALID_TXN_ID) == true) { // atomically swap item pointer held in the index bucket. AtomicUpdateItemPointer(tuple_location_ptr, tuple_location); // currently, let's assume only primary index exists. // gc::GCManagerFactory::GetInstance().RecycleTupleSlot( // table_->GetOid(), old_item.block, old_item.offset, // transaction_manager.GetNextCommitId()); garbage_tuples.push_back(old_item); tile_group = manager.GetTileGroup(tuple_location.block); tile_group_header = tile_group.get()->GetHeader(); tile_group_header->SetPrevItemPointer(tuple_location.offset, INVALID_ITEMPOINTER); } else { tile_group = manager.GetTileGroup(tuple_location.block); tile_group_header = tile_group.get()->GetHeader(); } } else { tile_group = manager.GetTileGroup(tuple_location.block); tile_group_header = tile_group.get()->GetHeader(); } } } } // Add all garbage tuples to GC manager if(garbage_tuples.size() != 0) { cid_t garbage_timestamp = transaction_manager.GetNextCommitId(); for (auto garbage : garbage_tuples) { gc::GCManagerFactory::GetInstance().RecycleTupleSlot( table_->GetOid(), garbage.block, garbage.offset, garbage_timestamp); } } // Construct a logical tile for each block for (auto tuples : visible_tuples) { auto &manager = catalog::Manager::GetInstance(); auto tile_group = manager.GetTileGroup(tuples.first); std::unique_ptr<LogicalTile> logical_tile(LogicalTileFactory::GetTile()); // Add relevant columns to logical tile logical_tile->AddColumns(tile_group, full_column_ids_); logical_tile->AddPositionList(std::move(tuples.second)); if (column_ids_.size() != 0) { logical_tile->ProjectColumns(full_column_ids_, column_ids_); } result_.push_back(logical_tile.release()); } done_ = true; LOG_TRACE("Result tiles : %lu", result_.size()); return true; }
/** * @brief Adds a column to the logical tile, using the position lists. * @return true on success, false otherwise. */ bool InsertExecutor::DExecute() { if (done_) return false; assert(!done_); assert(executor_context_ != nullptr); const planner::InsertPlan &node = GetPlanNode<planner::InsertPlan>(); storage::DataTable *target_table_ = node.GetTable(); oid_t bulk_insert_count = node.GetBulkInsertCount(); assert(target_table_); auto transaction_ = executor_context_->GetTransaction(); auto executor_pool = executor_context_->GetExecutorContextPool(); // Inserting a logical tile. if (children_.size() == 1) { LOG_INFO("Insert executor :: 1 child \n"); if (!children_[0]->Execute()) { return false; } std::unique_ptr<LogicalTile> logical_tile(children_[0]->GetOutput()); assert(logical_tile.get() != nullptr); auto target_table_schema = target_table_->GetSchema(); auto column_count = target_table_schema->GetColumnCount(); std::unique_ptr<storage::Tuple> tuple( new storage::Tuple(target_table_schema, true)); // Go over the logical tile for (oid_t tuple_id : *logical_tile) { expression::ContainerTuple<LogicalTile> cur_tuple(logical_tile.get(), tuple_id); // Materialize the logical tile tuple for (oid_t column_itr = 0; column_itr < column_count; column_itr++) tuple->SetValue(column_itr, cur_tuple.GetValue(column_itr), executor_pool); peloton::ItemPointer location = target_table_->InsertTuple(transaction_, tuple.get()); if (location.block == INVALID_OID) { transaction_->SetResult(peloton::Result::RESULT_FAILURE); return false; } transaction_->RecordInsert(location); executor_context_->num_processed += 1; // insert one } return true; } // Inserting a collection of tuples from plan node else if (children_.size() == 0) { LOG_INFO("Insert executor :: 0 child \n"); // Extract expressions from plan node and construct the tuple. // For now we just handle a single tuple auto schema = target_table_->GetSchema(); std::unique_ptr<storage::Tuple> tuple(new storage::Tuple(schema, true)); auto project_info = node.GetProjectInfo(); // There should be no direct maps assert(project_info); assert(project_info->GetDirectMapList().size() == 0); for (auto target : project_info->GetTargetList()) { peloton::Value value = target.second->Evaluate(nullptr, nullptr, executor_context_); tuple->SetValue(target.first, value, executor_pool); } // Bulk Insert Mode for (oid_t insert_itr = 0; insert_itr < bulk_insert_count; insert_itr++) { // Carry out insertion ItemPointer location = target_table_->InsertTuple(transaction_, tuple.get()); LOG_INFO("Inserted into location: %lu, %lu", location.block, location.offset); if (location.block == INVALID_OID) { LOG_INFO("Failed to Insert. Set txn failure."); transaction_->SetResult(peloton::Result::RESULT_FAILURE); return false; } transaction_->RecordInsert(location); // Logging { auto &log_manager = logging::LogManager::GetInstance(); if (log_manager.IsInLoggingMode()) { auto logger = log_manager.GetBackendLogger(); auto record = logger->GetTupleRecord( LOGRECORD_TYPE_TUPLE_INSERT, transaction_->GetTransactionId(), target_table_->GetOid(), location, INVALID_ITEMPOINTER, tuple.get()); logger->Log(record); } } } executor_context_->num_processed += 1; // insert one done_ = true; return true; } return true; }
TEST(AggregateTests, PlainSumCountDistinctTest) { /* * SELECT SUM(a), COUNT(b), COUNT(DISTINCT b) from table */ const int tuple_count = TESTS_TUPLES_PER_TILEGROUP; // Create a table and wrap it in logical tiles auto &txn_manager = concurrency::TransactionManager::GetInstance(); auto txn = txn_manager.BeginTransaction(); auto txn_id = txn->GetTransactionId(); std::unique_ptr<storage::DataTable> data_table( ExecutorTestsUtil::CreateTable(tuple_count, false)); ExecutorTestsUtil::PopulateTable(txn, data_table.get(), 2 * tuple_count, false, true, true); txn_manager.CommitTransaction(); std::unique_ptr<executor::LogicalTile> source_logical_tile1( executor::LogicalTileFactory::WrapTileGroup(data_table->GetTileGroup(0), txn_id)); std::unique_ptr<executor::LogicalTile> source_logical_tile2( executor::LogicalTileFactory::WrapTileGroup(data_table->GetTileGroup(1), txn_id)); // (1-5) Setup plan node // 1) Set up group-by columns std::vector<oid_t> group_by_columns; // 2) Set up project info planner::ProjectInfo::DirectMapList direct_map_list = { {0, {1, 0}}, {1, {1, 1}}, {2, {1, 2}}}; auto proj_info = new planner::ProjectInfo(planner::ProjectInfo::TargetList(), std::move(direct_map_list)); // 3) Set up unique aggregates std::vector<planner::AggregatePlan::AggTerm> agg_terms; planner::AggregatePlan::AggTerm sumA(EXPRESSION_TYPE_AGGREGATE_SUM, expression::TupleValueFactory(0, 0), false); planner::AggregatePlan::AggTerm countB(EXPRESSION_TYPE_AGGREGATE_COUNT, expression::TupleValueFactory(0, 1), false); // Flag distinct planner::AggregatePlan::AggTerm countDistinctB( EXPRESSION_TYPE_AGGREGATE_COUNT, expression::TupleValueFactory(0, 1), true); // Flag distinct agg_terms.push_back(sumA); agg_terms.push_back(countB); agg_terms.push_back(countDistinctB); // 4) Set up predicate (empty) expression::AbstractExpression* predicate = nullptr; // 5) Create output table schema auto data_table_schema = data_table.get()->GetSchema(); std::vector<oid_t> set = {0, 1, 1}; std::vector<catalog::Column> columns; for (auto column_index : set) { columns.push_back(data_table_schema->GetColumn(column_index)); } auto output_table_schema = new catalog::Schema(columns); // OK) Create the plan node planner::AggregatePlan node(proj_info, predicate, std::move(agg_terms), std::move(group_by_columns), output_table_schema, AGGREGATE_TYPE_PLAIN); // Create and set up executor auto txn2 = txn_manager.BeginTransaction(); std::unique_ptr<executor::ExecutorContext> context( new executor::ExecutorContext(txn2)); executor::AggregateExecutor executor(&node, context.get()); MockExecutor child_executor; executor.AddChild(&child_executor); EXPECT_CALL(child_executor, DInit()).WillOnce(Return(true)); EXPECT_CALL(child_executor, DExecute()) .WillOnce(Return(true)) .WillOnce(Return(true)) .WillOnce(Return(false)); EXPECT_CALL(child_executor, GetOutput()) .WillOnce(Return(source_logical_tile1.release())) .WillOnce(Return(source_logical_tile2.release())); EXPECT_TRUE(executor.Init()); EXPECT_TRUE(executor.Execute()); txn_manager.CommitTransaction(); /* Verify result */ std::unique_ptr<executor::LogicalTile> result_tile(executor.GetOutput()); EXPECT_TRUE(result_tile.get() != nullptr); EXPECT_TRUE(result_tile->GetValue(0, 0) .OpEquals(ValueFactory::GetIntegerValue(50)) .IsTrue()); EXPECT_TRUE(result_tile->GetValue(0, 1) .OpEquals(ValueFactory::GetIntegerValue(10)) .IsTrue()); EXPECT_TRUE(result_tile->GetValue(0, 2) .OpLessThanOrEqual(ValueFactory::GetIntegerValue(3)) .IsTrue()); }
void TimestampOrderingTransactionManager::PerformUpdate( TransactionContext *const current_txn, const ItemPointer &location, const ItemPointer &new_location) { PL_ASSERT(current_txn->GetIsolationLevel() != IsolationLevelType::READ_ONLY); ItemPointer old_location = location; LOG_TRACE("Performing Update old tuple %u %u", old_location.block, old_location.offset); LOG_TRACE("Performing Update new tuple %u %u", new_location.block, new_location.offset); auto &manager = catalog::Manager::GetInstance(); auto tile_group_header = manager.GetTileGroup(old_location.block)->GetHeader(); auto new_tile_group_header = manager.GetTileGroup(new_location.block)->GetHeader(); auto transaction_id = current_txn->GetTransactionId(); // if we can perform update, then we must have already locked the older // version. PL_ASSERT(tile_group_header->GetTransactionId(old_location.offset) == transaction_id); PL_ASSERT(tile_group_header->GetPrevItemPointer(old_location.offset) .IsNull() == true); // check whether the new version is empty. PL_ASSERT(new_tile_group_header->GetTransactionId(new_location.offset) == INVALID_TXN_ID); PL_ASSERT(new_tile_group_header->GetBeginCommitId(new_location.offset) == MAX_CID); PL_ASSERT(new_tile_group_header->GetEndCommitId(new_location.offset) == MAX_CID); // if the executor doesn't call PerformUpdate after AcquireOwnership, // no one will possibly release the write lock acquired by this txn. // Set double linked list tile_group_header->SetPrevItemPointer(old_location.offset, new_location); new_tile_group_header->SetNextItemPointer(new_location.offset, old_location); new_tile_group_header->SetTransactionId(new_location.offset, transaction_id); // we should guarantee that the newer version is all set before linking the // newer version to older version. COMPILER_MEMORY_FENCE; InitTupleReserved(new_tile_group_header, new_location.offset); // we must be updating the latest version. // Set the header information for the new version ItemPointer *index_entry_ptr = tile_group_header->GetIndirection(old_location.offset); // if there's no primary index on a table, then index_entry_ptr == nullptr. if (index_entry_ptr != nullptr) { new_tile_group_header->SetIndirection(new_location.offset, index_entry_ptr); // Set the index header in an atomic way. // We do it atomically because we don't want any one to see a half-done // pointer. // In case of contention, no one can update this pointer when we are // updating it // because we are holding the write lock. This update should success in // its first trial. UNUSED_ATTRIBUTE auto res = AtomicUpdateItemPointer(index_entry_ptr, new_location); PL_ASSERT(res == true); } // Add the old tuple into the update set current_txn->RecordUpdate(old_location); // Increment table update op stats if (static_cast<StatsType>(settings::SettingsManager::GetInt(settings::SettingId::stats_mode)) != StatsType::INVALID) { stats::BackendStatsContext::GetInstance()->IncrementTableUpdates( new_location.block); } }
void TileGroupHeader::PrintVisibility(txn_id_t txn_id, cid_t at_cid) { oid_t active_tuple_slots = GetNextTupleSlot(); std::stringstream os; os << "\t-----------------------------------------------------------\n"; for (oid_t header_itr = 0; header_itr < active_tuple_slots; header_itr++) { bool own = (txn_id == GetTransactionId(header_itr)); bool activated = (at_cid >= GetBeginCommitId(header_itr)); bool invalidated = (at_cid >= GetEndCommitId(header_itr)); txn_id_t txn_id = GetTransactionId(header_itr); cid_t beg_commit_id = GetBeginCommitId(header_itr); cid_t end_commit_id = GetEndCommitId(header_itr); bool insert_commit = GetInsertCommit(header_itr); bool delete_commit = GetDeleteCommit(header_itr); int width = 10; os << "\tslot :: " << std::setw(width) << header_itr; os << " txn id : "; if (txn_id == MAX_TXN_ID) os << std::setw(width) << "MAX_TXN_ID"; else os << std::setw(width) << txn_id; os << " beg cid : "; if (beg_commit_id == MAX_CID) os << std::setw(width) << "MAX_CID"; else os << std::setw(width) << beg_commit_id; os << " end cid : "; if (end_commit_id == MAX_CID) os << std::setw(width) << "MAX_CID"; else os << std::setw(width) << end_commit_id; os << " insert commit : "; if (insert_commit == true) os << "O"; else os << "X"; os << " delete commit : "; if (delete_commit == true) os << "O"; else os << "X"; peloton::ItemPointer location = GetPrevItemPointer(header_itr); os << " prev : " << "[ " << location.block << " , " << location.offset << " ]"; //<< os << " own : " << own; os << " activated : " << activated; os << " invalidated : " << invalidated << " "; // Visible iff past Insert || Own Insert if ((!own && activated && !invalidated) || (own && !activated && !invalidated)) os << "\t\t[ true ]\n"; else os << "\t\t[ false ]\n"; } os << "\t-----------------------------------------------------------\n"; std::cout << os.str().c_str(); }
bool HybridScanExecutor::ExecPrimaryIndexLookup() { assert(!index_done_); std::vector<ItemPointer *> tuple_location_ptrs; assert(index_->GetIndexType() == INDEX_CONSTRAINT_TYPE_PRIMARY_KEY); if (0 == key_column_ids_.size()) { index_->ScanAllKeys(tuple_location_ptrs); } else { index_->Scan(values_, key_column_ids_, expr_types_, SCAN_DIRECTION_TYPE_FORWARD, tuple_location_ptrs); } LOG_INFO("Tuple_locations.size(): %lu", tuple_location_ptrs.size()); auto &transaction_manager = concurrency::TransactionManagerFactory::GetInstance(); if (tuple_location_ptrs.size() == 0) { index_done_ = true; return false; } //std::set<oid_t> oid_ts; std::map<oid_t, std::vector<oid_t>> visible_tuples; // for every tuple that is found in the index. for (auto tuple_location_ptr : tuple_location_ptrs) { ItemPointer tuple_location = *tuple_location_ptr; if (type_ == planner::HYBRID && tuple_location.block >= (block_threshold)) { item_pointers_.insert(tuple_location); // oid_ts.insert(tuple_location.block); } auto &manager = catalog::Manager::GetInstance(); auto tile_group = manager.GetTileGroup(tuple_location.block); auto tile_group_header = tile_group.get()->GetHeader(); // perform transaction read size_t chain_length = 0; while (true) { ++chain_length; if (transaction_manager.IsVisible(tile_group_header, tuple_location.offset)) { visible_tuples[tuple_location.block].push_back(tuple_location.offset); auto res = transaction_manager.PerformRead(tuple_location); if (!res) { transaction_manager.SetTransactionResult(RESULT_FAILURE); return res; } break; } else { ItemPointer old_item = tuple_location; cid_t old_end_cid = tile_group_header->GetEndCommitId(old_item.offset); tuple_location = tile_group_header->GetNextItemPointer(old_item.offset); // there must exist a visible version. assert(tuple_location.IsNull() == false); cid_t max_committed_cid = transaction_manager.GetMaxCommittedCid(); // check whether older version is garbage. if (old_end_cid < max_committed_cid) { assert(tile_group_header->GetTransactionId(old_item.offset) == INITIAL_TXN_ID || tile_group_header->GetTransactionId(old_item.offset) == INVALID_TXN_ID); if (tile_group_header->SetAtomicTransactionId(old_item.offset, INVALID_TXN_ID) == true) { // atomically swap item pointer held in the index bucket. AtomicUpdateItemPointer(tuple_location_ptr, tuple_location); // currently, let's assume only primary index exists. //gc::GCManagerFactory::GetInstance().RecycleTupleSlot( // table_->GetOid(), old_item.block, old_item.offset, // max_committed_cid); } } tile_group = manager.GetTileGroup(tuple_location.block); tile_group_header = tile_group.get()->GetHeader(); } } } // Construct a logical tile for each block for (auto tuples : visible_tuples) { auto &manager = catalog::Manager::GetInstance(); auto tile_group = manager.GetTileGroup(tuples.first); std::unique_ptr<LogicalTile> logical_tile(LogicalTileFactory::GetTile()); // Add relevant columns to logical tile logical_tile->AddColumns(tile_group, full_column_ids_); logical_tile->AddPositionList(std::move(tuples.second)); if (column_ids_.size() != 0) { logical_tile->ProjectColumns(full_column_ids_, column_ids_); } result_.push_back(logical_tile.release()); } index_done_ = true; LOG_TRACE("Result tiles : %lu", result_.size()); return true; }
bool HybridScanExecutor::ExecPrimaryIndexLookup() { PL_ASSERT(index_done_ == false); const planner::HybridScanPlan &node = GetPlanNode<planner::HybridScanPlan>(); bool acquire_owner = GetPlanNode<planner::AbstractScan>().IsForUpdate(); auto key_column_ids_ = node.GetKeyColumnIds(); auto expr_type_ = node.GetExprTypes(); std::vector<ItemPointer *> tuple_location_ptrs; PL_ASSERT(index_->GetIndexType() == INDEX_CONSTRAINT_TYPE_PRIMARY_KEY); if (0 == key_column_ids_.size()) { LOG_TRACE("Scan all keys"); index_->ScanAllKeys(tuple_location_ptrs); } else { LOG_TRACE("Scan"); index_->Scan(values_, key_column_ids_, expr_type_, SCAN_DIRECTION_TYPE_FORWARD, tuple_location_ptrs, &node.GetIndexPredicate().GetConjunctionList()[0]); } LOG_TRACE("Result tuple count: %lu", tuple_location_ptrs.size()); auto &transaction_manager = concurrency::TransactionManagerFactory::GetInstance(); auto current_txn = executor_context_->GetTransaction(); if (tuple_location_ptrs.size() == 0) { index_done_ = true; return false; } std::map<oid_t, std::vector<oid_t>> visible_tuples; // for every tuple that is found in the index. for (auto tuple_location_ptr : tuple_location_ptrs) { ItemPointer tuple_location = *tuple_location_ptr; if (type_ == HYBRID_SCAN_TYPE_HYBRID && tuple_location.block >= (block_threshold)) { item_pointers_.insert(tuple_location); } auto &manager = catalog::Manager::GetInstance(); auto tile_group = manager.GetTileGroup(tuple_location.block); auto tile_group_header = tile_group.get()->GetHeader(); // perform transaction read size_t chain_length = 0; while (true) { ++chain_length; auto visibility = transaction_manager.IsVisible(current_txn, tile_group_header, tuple_location.offset); if (visibility == VISIBILITY_OK) { visible_tuples[tuple_location.block].push_back(tuple_location.offset); auto res = transaction_manager.PerformRead(current_txn, tuple_location, acquire_owner); if (!res) { transaction_manager.SetTransactionResult(current_txn, RESULT_FAILURE); return res; } break; } else { ItemPointer old_item = tuple_location; cid_t old_end_cid = tile_group_header->GetEndCommitId(old_item.offset); tuple_location = tile_group_header->GetNextItemPointer(old_item.offset); // there must exist a visible version. assert(tuple_location.IsNull() == false); cid_t max_committed_cid = transaction_manager.GetMaxCommittedCid(); // check whether older version is garbage. if (old_end_cid < max_committed_cid) { assert(tile_group_header->GetTransactionId(old_item.offset) == INITIAL_TXN_ID || tile_group_header->GetTransactionId(old_item.offset) == INVALID_TXN_ID); if (tile_group_header->SetAtomicTransactionId( old_item.offset, INVALID_TXN_ID) == true) { // atomically swap item pointer held in the index bucket. AtomicUpdateItemPointer(tuple_location_ptr, tuple_location); } } tile_group = manager.GetTileGroup(tuple_location.block); tile_group_header = tile_group.get()->GetHeader(); } } } // Construct a logical tile for each block for (auto tuples : visible_tuples) { auto &manager = catalog::Manager::GetInstance(); auto tile_group = manager.GetTileGroup(tuples.first); std::unique_ptr<LogicalTile> logical_tile(LogicalTileFactory::GetTile()); // Add relevant columns to logical tile logical_tile->AddColumns(tile_group, full_column_ids_); logical_tile->AddPositionList(std::move(tuples.second)); if (column_ids_.size() != 0) { logical_tile->ProjectColumns(full_column_ids_, column_ids_); } result_.push_back(logical_tile.release()); } index_done_ = true; LOG_TRACE("Result tiles : %lu", result_.size()); return true; }
TEST(AggregateTests, HashDistinctTest) { /* * SELECT d, a, b, c FROM table GROUP BY a, b, c, d; */ const int tuple_count = TESTS_TUPLES_PER_TILEGROUP; // Create a table and wrap it in logical tiles auto &txn_manager = concurrency::TransactionManager::GetInstance(); auto txn = txn_manager.BeginTransaction(); auto txn_id = txn->GetTransactionId(); std::unique_ptr<storage::DataTable> data_table( ExecutorTestsUtil::CreateTable(tuple_count, false)); ExecutorTestsUtil::PopulateTable(txn, data_table.get(), 2 * tuple_count, false, true, true); // let it be random txn_manager.CommitTransaction(); std::unique_ptr<executor::LogicalTile> source_logical_tile1( executor::LogicalTileFactory::WrapTileGroup(data_table->GetTileGroup(0), txn_id)); std::unique_ptr<executor::LogicalTile> source_logical_tile2( executor::LogicalTileFactory::WrapTileGroup(data_table->GetTileGroup(1), txn_id)); // (1-5) Setup plan node // 1) Set up group-by columns std::vector<oid_t> group_by_columns = {0, 1, 2, 3}; // 2) Set up project info planner::ProjectInfo::DirectMapList direct_map_list = { {0, {0, 3}}, {1, {0, 0}}, {2, {0, 1}}, {3, {0, 2}}}; auto proj_info = new planner::ProjectInfo(planner::ProjectInfo::TargetList(), std::move(direct_map_list)); // 3) Set up unique aggregates (empty) std::vector<planner::AggregatePlan::AggTerm> agg_terms; // 4) Set up predicate (empty) expression::AbstractExpression* predicate = nullptr; // 5) Create output table schema auto data_table_schema = data_table.get()->GetSchema(); std::vector<oid_t> set = {3, 0, 1, 2}; std::vector<catalog::Column> columns; for (auto column_index : set) { columns.push_back(data_table_schema->GetColumn(column_index)); } auto output_table_schema = new catalog::Schema(columns); // OK) Create the plan node planner::AggregatePlan node(proj_info, predicate, std::move(agg_terms), std::move(group_by_columns), output_table_schema, AGGREGATE_TYPE_HASH); // Create and set up executor auto txn2 = txn_manager.BeginTransaction(); std::unique_ptr<executor::ExecutorContext> context( new executor::ExecutorContext(txn2)); executor::AggregateExecutor executor(&node, context.get()); MockExecutor child_executor; executor.AddChild(&child_executor); EXPECT_CALL(child_executor, DInit()).WillOnce(Return(true)); EXPECT_CALL(child_executor, DExecute()) .WillOnce(Return(true)) .WillOnce(Return(true)) .WillOnce(Return(false)); EXPECT_CALL(child_executor, GetOutput()) .WillOnce(Return(source_logical_tile1.release())) .WillOnce(Return(source_logical_tile2.release())); EXPECT_TRUE(executor.Init()); EXPECT_TRUE(executor.Execute()); txn_manager.CommitTransaction(); /* Verify result */ std::unique_ptr<executor::LogicalTile> result_tile(executor.GetOutput()); EXPECT_TRUE(result_tile.get() != nullptr); for (auto tuple_id : *result_tile) { int colA = ValuePeeker::PeekAsInteger(result_tile->GetValue(tuple_id, 1)); (void)colA; } }
Result PessimisticTxnManager::CommitTransaction() { LOG_TRACE("Committing peloton txn : %lu ", current_txn->GetTransactionId()); auto &manager = catalog::Manager::GetInstance(); auto &rw_set = current_txn->GetRWSet(); //***************************************************** // we can optimize read-only transaction. if (current_txn->IsReadOnly() == true) { // validate read set. for (auto &tile_group_entry : rw_set) { oid_t tile_group_id = tile_group_entry.first; auto tile_group = manager.GetTileGroup(tile_group_id); auto tile_group_header = tile_group->GetHeader(); for (auto &tuple_entry : tile_group_entry.second) { auto tuple_slot = tuple_entry.first; // if this tuple is not newly inserted. if (tuple_entry.second == RW_TYPE_READ) { // Release read locks if (pessimistic_released_rdlock.find(tile_group_id) == pessimistic_released_rdlock.end() || pessimistic_released_rdlock[tile_group_id].find(tuple_slot) == pessimistic_released_rdlock[tile_group_id].end()) { ReleaseReadLock(tile_group_header, tuple_slot); pessimistic_released_rdlock[tile_group_id].insert(tuple_slot); } } else { assert(tuple_entry.second == RW_TYPE_INS_DEL); } } } // is it always true??? Result ret = current_txn->GetResult(); EndTransaction(); return ret; } //***************************************************** // generate transaction id. cid_t end_commit_id = GetNextCommitId(); auto &log_manager = logging::LogManager::GetInstance(); log_manager.LogBeginTransaction(end_commit_id); // install everything. for (auto &tile_group_entry : rw_set) { oid_t tile_group_id = tile_group_entry.first; auto tile_group = manager.GetTileGroup(tile_group_id); auto tile_group_header = tile_group->GetHeader(); for (auto &tuple_entry : tile_group_entry.second) { auto tuple_slot = tuple_entry.first; if (tuple_entry.second == RW_TYPE_READ) { // Release read locks if (pessimistic_released_rdlock.find(tile_group_id) == pessimistic_released_rdlock.end() || pessimistic_released_rdlock[tile_group_id].find(tuple_slot) == pessimistic_released_rdlock[tile_group_id].end()) { ReleaseReadLock(tile_group_header, tuple_slot); pessimistic_released_rdlock[tile_group_id].insert(tuple_slot); } } else if (tuple_entry.second == RW_TYPE_UPDATE) { // we must guarantee that, at any time point, only one version is // visible. ItemPointer new_version = tile_group_header->GetNextItemPointer(tuple_slot); ItemPointer old_version(tile_group_id, tuple_slot); // logging. log_manager.LogUpdate(current_txn, end_commit_id, old_version, new_version); auto new_tile_group_header = manager.GetTileGroup(new_version.block)->GetHeader(); new_tile_group_header->SetEndCommitId(new_version.offset, MAX_CID); new_tile_group_header->SetBeginCommitId(new_version.offset, end_commit_id); COMPILER_MEMORY_FENCE; tile_group_header->SetEndCommitId(tuple_slot, end_commit_id); COMPILER_MEMORY_FENCE; new_tile_group_header->SetTransactionId(new_version.offset, INITIAL_TXN_ID); tile_group_header->SetTransactionId(tuple_slot, INITIAL_TXN_ID); } else if (tuple_entry.second == RW_TYPE_DELETE) { ItemPointer new_version = tile_group_header->GetNextItemPointer(tuple_slot); ItemPointer delete_location(tile_group_id, tuple_slot); // logging. log_manager.LogDelete(end_commit_id, delete_location); // we do not change begin cid for old tuple. auto new_tile_group_header = manager.GetTileGroup(new_version.block)->GetHeader(); new_tile_group_header->SetEndCommitId(new_version.offset, MAX_CID); new_tile_group_header->SetBeginCommitId(new_version.offset, end_commit_id); COMPILER_MEMORY_FENCE; tile_group_header->SetEndCommitId(tuple_slot, end_commit_id); COMPILER_MEMORY_FENCE; new_tile_group_header->SetTransactionId(new_version.offset, INVALID_TXN_ID); tile_group_header->SetTransactionId(tuple_slot, INITIAL_TXN_ID); } else if (tuple_entry.second == RW_TYPE_INSERT) { assert(tile_group_header->GetTransactionId(tuple_slot) == current_txn->GetTransactionId()); // set the begin commit id to persist insert ItemPointer insert_location(tile_group_id, tuple_slot); log_manager.LogInsert(current_txn, end_commit_id, insert_location); tile_group_header->SetEndCommitId(tuple_slot, MAX_CID); tile_group_header->SetBeginCommitId(tuple_slot, end_commit_id); COMPILER_MEMORY_FENCE; tile_group_header->SetTransactionId(tuple_slot, INITIAL_TXN_ID); } else if (tuple_entry.second == RW_TYPE_INS_DEL) { assert(tile_group_header->GetTransactionId(tuple_slot) == current_txn->GetTransactionId()); tile_group_header->SetEndCommitId(tuple_slot, MAX_CID); tile_group_header->SetBeginCommitId(tuple_slot, MAX_CID); COMPILER_MEMORY_FENCE; // set the begin commit id to persist insert tile_group_header->SetTransactionId(tuple_slot, INVALID_TXN_ID); } } } log_manager.LogCommitTransaction(end_commit_id); EndTransaction(); pessimistic_released_rdlock.clear(); return Result::RESULT_SUCCESS; }
void ExecuteJoinTest(PlanNodeType join_algorithm, PelotonJoinType join_type, oid_t join_test_type) { //===--------------------------------------------------------------------===// // Mock table scan executors //===--------------------------------------------------------------------===// MockExecutor left_table_scan_executor, right_table_scan_executor; // Create a table and wrap it in logical tile size_t tile_group_size = TESTS_TUPLES_PER_TILEGROUP; size_t left_table_tile_group_count = 3; size_t right_table_tile_group_count = 2; auto &txn_manager = concurrency::TransactionManager::GetInstance(); auto txn = txn_manager.BeginTransaction(); auto txn_id = txn->GetTransactionId(); // Left table has 3 tile groups std::unique_ptr<storage::DataTable> left_table( ExecutorTestsUtil::CreateTable(tile_group_size)); ExecutorTestsUtil::PopulateTable( txn, left_table.get(), tile_group_size * left_table_tile_group_count, false, false, false); // Right table has 2 tile groups std::unique_ptr<storage::DataTable> right_table( ExecutorTestsUtil::CreateTable(tile_group_size)); ExecutorTestsUtil::PopulateTable( txn, right_table.get(), tile_group_size * right_table_tile_group_count, false, false, false); txn_manager.CommitTransaction(); //std::cout << (*left_table); //std::cout << (*right_table); // Wrap the input tables with logical tiles std::unique_ptr<executor::LogicalTile> left_table_logical_tile1( executor::LogicalTileFactory::WrapTileGroup(left_table->GetTileGroup(0), txn_id)); std::unique_ptr<executor::LogicalTile> left_table_logical_tile2( executor::LogicalTileFactory::WrapTileGroup(left_table->GetTileGroup(1), txn_id)); std::unique_ptr<executor::LogicalTile> left_table_logical_tile3( executor::LogicalTileFactory::WrapTileGroup(left_table->GetTileGroup(2), txn_id)); std::unique_ptr<executor::LogicalTile> right_table_logical_tile1( executor::LogicalTileFactory::WrapTileGroup( right_table->GetTileGroup(0), txn_id)); std::unique_ptr<executor::LogicalTile> right_table_logical_tile2( executor::LogicalTileFactory::WrapTileGroup( right_table->GetTileGroup(1), txn_id)); // Left scan executor returns logical tiles from the left table EXPECT_CALL(left_table_scan_executor, DInit()).WillOnce(Return(true)); //===--------------------------------------------------------------------===// // Setup left table //===--------------------------------------------------------------------===// if(join_test_type == BASIC_TEST) { EXPECT_CALL(left_table_scan_executor, DExecute()) .WillOnce(Return(true)) .WillOnce(Return(true)) .WillOnce(Return(true)) .WillOnce(Return(false)); EXPECT_CALL(left_table_scan_executor, GetOutput()) .WillOnce(Return(left_table_logical_tile1.release())) .WillOnce(Return(left_table_logical_tile2.release())) .WillOnce(Return(left_table_logical_tile3.release())); } // Right scan executor returns logical tiles from the right table EXPECT_CALL(right_table_scan_executor, DInit()).WillOnce(Return(true)); //===--------------------------------------------------------------------===// // Setup right table //===--------------------------------------------------------------------===// if(join_test_type == BASIC_TEST) { EXPECT_CALL(right_table_scan_executor, DExecute()) .WillOnce(Return(true)) .WillOnce(Return(true)) .WillOnce(Return(false)); EXPECT_CALL(right_table_scan_executor, GetOutput()) .WillOnce(Return(right_table_logical_tile1.release())) .WillOnce(Return(right_table_logical_tile2.release())); } //===--------------------------------------------------------------------===// // Setup join plan nodes and executors and run them //===--------------------------------------------------------------------===// oid_t result_tuple_count = 0; oid_t tuples_with_null = 0; auto projection = JoinTestsUtil::CreateProjection(); // Construct predicate expression::AbstractExpression *predicate = JoinTestsUtil::CreateJoinPredicate(); // Differ based on join algorithm switch (join_algorithm) { case PLAN_NODE_TYPE_NESTLOOP: { // Create nested loop join plan node. planner::NestedLoopJoinPlan nested_loop_join_node(join_type, predicate, projection); // Run the nested loop join executor executor::NestedLoopJoinExecutor nested_loop_join_executor( &nested_loop_join_node, nullptr); // Construct the executor tree nested_loop_join_executor.AddChild(&left_table_scan_executor); nested_loop_join_executor.AddChild(&right_table_scan_executor); // Run the nested loop join executor EXPECT_TRUE(nested_loop_join_executor.Init()); while (nested_loop_join_executor.Execute() == true) { std::unique_ptr<executor::LogicalTile> result_logical_tile( nested_loop_join_executor.GetOutput()); if (result_logical_tile != nullptr) { result_tuple_count += result_logical_tile->GetTupleCount(); tuples_with_null += CountTuplesWithNullFields(result_logical_tile.get()); //std::cout << (*result_logical_tile); } } } break; case PLAN_NODE_TYPE_MERGEJOIN: { // Create join clauses std::vector<planner::MergeJoinPlan::JoinClause> join_clauses; join_clauses = CreateJoinClauses(); // Create merge join plan node planner::MergeJoinPlan merge_join_node(join_type, predicate, projection, join_clauses); // Construct the merge join executor executor::MergeJoinExecutor merge_join_executor(&merge_join_node, nullptr); // Construct the executor tree merge_join_executor.AddChild(&left_table_scan_executor); merge_join_executor.AddChild(&right_table_scan_executor); // Run the merge join executor EXPECT_TRUE(merge_join_executor.Init()); while (merge_join_executor.Execute() == true) { std::unique_ptr<executor::LogicalTile> result_logical_tile( merge_join_executor.GetOutput()); if (result_logical_tile != nullptr) { result_tuple_count += result_logical_tile->GetTupleCount(); tuples_with_null += CountTuplesWithNullFields(result_logical_tile.get()); //std::cout << (*result_logical_tile); } } } break; case PLAN_NODE_TYPE_HASHJOIN: { // Create hash plan node expression::AbstractExpression *right_table_attr_1 = new expression::TupleValueExpression(1, 1); std::vector<std::unique_ptr<const expression::AbstractExpression> > hash_keys; hash_keys.emplace_back(right_table_attr_1); // Create hash plan node planner::HashPlan hash_plan_node(hash_keys); // Construct the hash executor executor::HashExecutor hash_executor(&hash_plan_node, nullptr); // Create hash join plan node. planner::HashJoinPlan hash_join_plan_node(join_type, predicate, projection); // Construct the hash join executor executor::HashJoinExecutor hash_join_executor(&hash_join_plan_node, nullptr); // Construct the executor tree hash_join_executor.AddChild(&left_table_scan_executor); hash_join_executor.AddChild(&hash_executor); hash_executor.AddChild(&right_table_scan_executor); // Run the hash_join_executor EXPECT_TRUE(hash_join_executor.Init()); while (hash_join_executor.Execute() == true) { std::unique_ptr<executor::LogicalTile> result_logical_tile( hash_join_executor.GetOutput()); if (result_logical_tile != nullptr) { result_tuple_count += result_logical_tile->GetTupleCount(); tuples_with_null += CountTuplesWithNullFields(result_logical_tile.get()); // std::cout << (*result_logical_tile); } } } break; default: throw Exception("Unsupported join algorithm : " + std::to_string(join_algorithm)); break; } //===--------------------------------------------------------------------===// // Execute test //===--------------------------------------------------------------------===// if(join_test_type == BASIC_TEST) { // Check output switch (join_type) { case JOIN_TYPE_INNER: EXPECT_EQ(result_tuple_count, 10); EXPECT_EQ(tuples_with_null, 0); break; case JOIN_TYPE_LEFT: EXPECT_EQ(result_tuple_count, 15); EXPECT_EQ(tuples_with_null, 5); break; case JOIN_TYPE_RIGHT: EXPECT_EQ(result_tuple_count, 10); EXPECT_EQ(tuples_with_null, 0); break; case JOIN_TYPE_OUTER: EXPECT_EQ(result_tuple_count, 15); EXPECT_EQ(tuples_with_null, 5); break; default: throw Exception("Unsupported join type : " + std::to_string(join_type)); break; } } }
/** * @brief Build a executor tree and execute it. * Use std::vector<Value> as params to make it more elegant for networking * Before ExecutePlan, a node first receives value list, so we should pass * value list directly rather than passing Postgres's ParamListInfo * @return status of execution. */ peloton_status PlanExecutor::ExecutePlan(const planner::AbstractPlan *plan, const std::vector<Value> ¶ms, TupleDesc tuple_desc) { peloton_status p_status; if (plan == nullptr) return p_status; LOG_TRACE("PlanExecutor Start "); bool status; bool init_failure = false; bool single_statement_txn = false; List *slots = NULL; auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = peloton::concurrency::current_txn; // This happens for single statement queries in PG if (txn == nullptr) { single_statement_txn = true; txn = txn_manager.BeginTransaction(); } PL_ASSERT(txn); LOG_TRACE("Txn ID = %lu ", txn->GetTransactionId()); LOG_TRACE("Building the executor tree"); // Use const std::vector<Value> ¶ms to make it more elegant for network std::unique_ptr<executor::ExecutorContext> executor_context( BuildExecutorContext(params, txn)); //auto executor_context = BuildExecutorContext(param_list, txn); // Build the executor tree std::unique_ptr<executor::AbstractExecutor> executor_tree( BuildExecutorTree(nullptr, plan, executor_context.get())); LOG_TRACE("Initializing the executor tree"); // Initialize the executor tree status = executor_tree->Init(); // Abort and cleanup if (status == false) { init_failure = true; txn->SetResult(Result::RESULT_FAILURE); goto cleanup; } LOG_TRACE("Running the executor tree"); // Execute the tree until we get result tiles from root node for (;;) { status = executor_tree->Execute(); // Stop if (status == false) { break; } std::unique_ptr<executor::LogicalTile> logical_tile( executor_tree->GetOutput()); // Some executors don't return logical tiles (e.g., Update). if (logical_tile.get() == nullptr) { continue; } // Go over the logical tile for (oid_t tuple_id : *logical_tile) { expression::ContainerTuple<executor::LogicalTile> cur_tuple( logical_tile.get(), tuple_id); auto slot = TupleTransformer::GetPostgresTuple(&cur_tuple, tuple_desc); if (slot != nullptr) { slots = lappend(slots, slot); } } } // Set the result p_status.m_processed = executor_context->num_processed; p_status.m_result_slots = slots; // final cleanup cleanup: LOG_TRACE("About to commit: single stmt: %d, init_failure: %d, status: %d", single_statement_txn, init_failure, txn->GetResult()); // should we commit or abort ? if (single_statement_txn == true || init_failure == true) { auto status = txn->GetResult(); switch (status) { case Result::RESULT_SUCCESS: // Commit p_status.m_result = txn_manager.CommitTransaction(); break; case Result::RESULT_FAILURE: default: // Abort p_status.m_result = txn_manager.AbortTransaction(); } } // clean up executor tree CleanExecutorTree(executor_tree.get()); return p_status; }
/** * @brief Build a executor tree and execute it. * Use std::vector<Value> as params to make it more elegant for networking * Before ExecutePlan, a node first receives value list, so we should pass * value list directly rather than passing Postgres's ParamListInfo * @return number of executed tuples and logical_tile_list */ int PlanExecutor::ExecutePlan(const planner::AbstractPlan *plan, const std::vector<Value> ¶ms, std::vector<std::unique_ptr<executor::LogicalTile>>& logical_tile_list) { if (plan == nullptr) return -1; LOG_TRACE("PlanExecutor Start "); bool status; bool init_failure = false; bool single_statement_txn = false; auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = peloton::concurrency::current_txn; // This happens for single statement queries in PG if (txn == nullptr) { single_statement_txn = true; txn = txn_manager.BeginTransaction(); } PL_ASSERT(txn); LOG_TRACE("Txn ID = %lu ", txn->GetTransactionId()); LOG_TRACE("Building the executor tree"); // Use const std::vector<Value> ¶ms to make it more elegant for network std::unique_ptr<executor::ExecutorContext> executor_context( BuildExecutorContext(params, txn)); // Build the executor tree std::unique_ptr<executor::AbstractExecutor> executor_tree( BuildExecutorTree(nullptr, plan, executor_context.get())); LOG_TRACE("Initializing the executor tree"); // Initialize the executor tree status = executor_tree->Init(); // Abort and cleanup if (status == false) { init_failure = true; txn->SetResult(Result::RESULT_FAILURE); goto cleanup; } LOG_TRACE("Running the executor tree"); // Execute the tree until we get result tiles from root node for (;;) { status = executor_tree->Execute(); // Stop if (status == false) { break; } std::unique_ptr<executor::LogicalTile> logical_tile( executor_tree->GetOutput()); // Some executors don't return logical tiles (e.g., Update). if (logical_tile.get() == nullptr) { continue; } logical_tile_list.push_back(std::move(logical_tile)); } // final cleanup cleanup: LOG_TRACE("About to commit: single stmt: %d, init_failure: %d, status: %d", single_statement_txn, init_failure, txn->GetResult()); // clean up executor tree CleanExecutorTree(executor_tree.get()); // should we commit or abort ? if (single_statement_txn == true || init_failure == true) { auto status = txn->GetResult(); switch (status) { case Result::RESULT_SUCCESS: // Commit return executor_context->num_processed; break; case Result::RESULT_FAILURE: default: // Abort return -1; } } return executor_context->num_processed; }
/** * @brief Delete the table tuples using the position list in the logical tile. * * If truncate is on, then it will truncate the table itself. * @return true on success, false otherwise. */ bool DeleteExecutor::DExecute() { assert(target_table_); // Retrieve next tile. const bool success = children_[0]->Execute(); if (!success) { return false; } std::unique_ptr<LogicalTile> source_tile(children_[0]->GetOutput()); storage::Tile *tile = source_tile->GetBaseTile(0); storage::TileGroup *tile_group = tile->GetTileGroup(); auto &pos_lists = source_tile.get()->GetPositionLists(); auto tile_group_id = tile_group->GetTileGroupId(); auto transaction_ = executor_context_->GetTransaction(); LOG_INFO("Source tile : %p Tuples : %lu \n", source_tile.get(), source_tile->GetTupleCount()); LOG_INFO("Transaction ID: %lu\n", transaction_->GetTransactionId()); // Delete each tuple for (oid_t visible_tuple_id : *source_tile) { oid_t physical_tuple_id = pos_lists[0][visible_tuple_id]; LOG_INFO("Visible Tuple id : %lu, Physical Tuple id : %lu \n", visible_tuple_id, physical_tuple_id); peloton::ItemPointer delete_location(tile_group_id, physical_tuple_id); // Logging { auto &log_manager = logging::LogManager::GetInstance(); if (log_manager.IsInLoggingMode()) { auto logger = log_manager.GetBackendLogger(); auto record = logger->GetTupleRecord( LOGRECORD_TYPE_TUPLE_DELETE, transaction_->GetTransactionId(), target_table_->GetOid(), INVALID_ITEMPOINTER, delete_location); logger->Log(record); } } // try to delete the tuple // this might fail due to a concurrent operation that has latched the tuple bool status = target_table_->DeleteTuple(transaction_, delete_location); if (status == false) { LOG_INFO("Fail to delete. Set txn failure"); transaction_->SetResult(peloton::Result::RESULT_FAILURE); return false; } executor_context_->num_processed += 1; // deleted one transaction_->RecordDelete(delete_location); } return true; }
/** * @brief Build a executor tree and execute it. * Use std::vector<common::Value> as params to make it more elegant for * networking * Before ExecutePlan, a node first receives value list, so we should pass * value list directly rather than passing Postgres's ParamListInfo * @return status of execution. */ peloton_status PlanExecutor::ExecutePlan( const planner::AbstractPlan *plan, const std::vector<common::Value> ¶ms, std::vector<ResultType> &result, const std::vector<int> &result_format) { peloton_status p_status; if (plan == nullptr) return p_status; LOG_TRACE("PlanExecutor Start "); bool status; bool init_failure = false; bool single_statement_txn = false; auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); // auto txn = peloton::concurrency::current_txn; // This happens for single statement queries in PG // if (txn == nullptr) { single_statement_txn = true; auto txn = txn_manager.BeginTransaction(); // } PL_ASSERT(txn); LOG_TRACE("Txn ID = %lu ", txn->GetTransactionId()); LOG_TRACE("Building the executor tree"); // Use const std::vector<common::Value> ¶ms to make it more elegant for // network std::unique_ptr<executor::ExecutorContext> executor_context( BuildExecutorContext(params, txn)); // Build the executor tree std::unique_ptr<executor::AbstractExecutor> executor_tree( BuildExecutorTree(nullptr, plan, executor_context.get())); LOG_TRACE("Initializing the executor tree"); // Initialize the executor tree status = executor_tree->Init(); // Abort and cleanup if (status == false) { init_failure = true; txn->SetResult(Result::RESULT_FAILURE); goto cleanup; } LOG_TRACE("Running the executor tree"); result.clear(); // Execute the tree until we get result tiles from root node while (status == true) { status = executor_tree->Execute(); std::unique_ptr<executor::LogicalTile> logical_tile( executor_tree->GetOutput()); // Some executors don't return logical tiles (e.g., Update). if (logical_tile.get() != nullptr) { LOG_TRACE("Final Answer: %s", logical_tile->GetInfo().c_str()); // Printing the answers std::unique_ptr<catalog::Schema> output_schema( logical_tile->GetPhysicalSchema()); // Physical schema of the tile std::vector<std::vector<std::string>> answer_tuples; answer_tuples = std::move(logical_tile->GetAllValuesAsStrings(result_format)); // Construct the returned results for (auto &tuple : answer_tuples) { unsigned int col_index = 0; auto &schema_columns = output_schema->GetColumns(); for (auto &column : schema_columns) { auto column_name = column.GetName(); auto res = ResultType(); PlanExecutor::copyFromTo(column_name, res.first); LOG_TRACE("column name: %s", column_name.c_str()); PlanExecutor::copyFromTo(tuple[col_index++], res.second); if (tuple[col_index - 1].c_str() != nullptr) { LOG_TRACE("column content: %s", tuple[col_index - 1].c_str()); } result.push_back(res); } } } } // Set the result p_status.m_processed = executor_context->num_processed; p_status.m_result_slots = nullptr; // final cleanup cleanup: LOG_TRACE("About to commit: single stmt: %d, init_failure: %d, status: %d", single_statement_txn, init_failure, txn->GetResult()); // should we commit or abort ? if (single_statement_txn == true || init_failure == true) { auto status = txn->GetResult(); switch (status) { case Result::RESULT_SUCCESS: // Commit LOG_TRACE("Commit Transaction"); p_status.m_result = txn_manager.CommitTransaction(txn); break; case Result::RESULT_FAILURE: default: // Abort LOG_TRACE("Abort Transaction"); p_status.m_result = txn_manager.AbortTransaction(txn); } } // clean up executor tree CleanExecutorTree(executor_tree.get()); return p_status; }
// Validate that MVCC storage is correct, it assumes an old-to-new chain // Invariants // 1. Transaction id should either be INVALID_TXNID or INITIAL_TXNID // 2. Begin commit id should <= end commit id // 3. Timestamp consistence // 4. Version doubly linked list consistency static void ValidateMVCC_OldToNew(storage::DataTable *table) { auto &catalog_manager = catalog::Manager::GetInstance(); LOG_INFO("Validating MVCC storage"); int tile_group_count = table->GetTileGroupCount(); LOG_INFO("The table has %d tile groups in the table", tile_group_count); for (int tile_group_offset = 0; tile_group_offset < tile_group_count; tile_group_offset++) { LOG_INFO("Validate tile group #%d", tile_group_offset); auto tile_group = table->GetTileGroup(tile_group_offset); auto tile_group_header = tile_group->GetHeader(); size_t tuple_count = tile_group->GetAllocatedTupleCount(); LOG_INFO("Tile group #%d has allocated %lu tuples", tile_group_offset, tuple_count); // 1. Transaction id should either be INVALID_TXNID or INITIAL_TXNID for (oid_t tuple_slot = 0; tuple_slot < tuple_count; tuple_slot++) { txn_id_t txn_id = tile_group_header->GetTransactionId(tuple_slot); EXPECT_TRUE(txn_id == INVALID_TXN_ID || txn_id == INITIAL_TXN_ID) << "Transaction id is not INVALID_TXNID or INITIAL_TXNID"; } LOG_INFO("[OK] All tuples have valid txn id"); // double avg_version_chain_length = 0.0; for (oid_t tuple_slot = 0; tuple_slot < tuple_count; tuple_slot++) { txn_id_t txn_id = tile_group_header->GetTransactionId(tuple_slot); cid_t begin_cid = tile_group_header->GetBeginCommitId(tuple_slot); cid_t end_cid = tile_group_header->GetEndCommitId(tuple_slot); ItemPointer next_location = tile_group_header->GetNextItemPointer(tuple_slot); ItemPointer prev_location = tile_group_header->GetPrevItemPointer(tuple_slot); // 2. Begin commit id should <= end commit id EXPECT_TRUE(begin_cid <= end_cid) << "Tuple begin commit id is less than or equal to end commit id"; // This test assumes a oldest-to-newest version chain if (txn_id != INVALID_TXN_ID) { EXPECT_TRUE(begin_cid != MAX_CID) << "Non invalid txn shouldn't have a MAX_CID begin commit id"; // The version is an oldest version if (prev_location.IsNull()) { if (next_location.IsNull()) { EXPECT_EQ(end_cid, MAX_CID) << "Single version has a non MAX_CID end commit time"; } else { cid_t prev_end_cid = end_cid; ItemPointer prev_location(tile_group->GetTileGroupId(), tuple_slot); while (!next_location.IsNull()) { auto next_tile_group = catalog_manager.GetTileGroup(next_location.block); auto next_tile_group_header = next_tile_group->GetHeader(); txn_id_t next_txn_id = next_tile_group_header->GetTransactionId( next_location.offset); if (next_txn_id == INVALID_TXN_ID) { // If a version in the version chain has a INVALID_TXN_ID, it // must be at the tail // of the chain. It is either because we have deleted a tuple // (so append a invalid tuple), // or because this new version is aborted. EXPECT_TRUE( next_tile_group_header->GetNextItemPointer( next_location.offset).IsNull()) << "Invalid version in a version chain and is not delete"; } cid_t next_begin_cid = next_tile_group_header->GetBeginCommitId( next_location.offset); cid_t next_end_cid = next_tile_group_header->GetEndCommitId(next_location.offset); // 3. Timestamp consistence if (next_begin_cid == MAX_CID) { // It must be an aborted version, it should be at the end of the // chain EXPECT_TRUE( next_tile_group_header->GetNextItemPointer( next_location.offset).IsNull()) << "Version with MAX_CID begin cid is not version tail"; } else { EXPECT_EQ(prev_end_cid, next_begin_cid) << "Prev end commit id should equal net begin commit id"; ItemPointer next_prev_location = next_tile_group_header->GetPrevItemPointer( next_location.offset); // 4. Version doubly linked list consistency EXPECT_TRUE(next_prev_location.offset == prev_location.offset && next_prev_location.block == prev_location.block) << "Next version's prev version does not match"; } prev_location = next_location; prev_end_cid = next_end_cid; next_location = next_tile_group_header->GetNextItemPointer( next_location.offset); } // Now prev_location is at the tail of the version chain ItemPointer last_location = prev_location; auto last_tile_group = catalog_manager.GetTileGroup(last_location.block); auto last_tile_group_header = last_tile_group->GetHeader(); // txn_id_t last_txn_id = // last_tile_group_header->GetTransactionId(last_location.offset); cid_t last_end_cid = last_tile_group_header->GetEndCommitId(last_location.offset); EXPECT_TRUE( last_tile_group_header->GetNextItemPointer(last_location.offset) .IsNull()) << "Last version has a next pointer"; EXPECT_EQ(last_end_cid, MAX_CID) << "Last version doesn't end with MAX_CID"; } } } else { EXPECT_TRUE(tile_group_header->GetNextItemPointer(tuple_slot).IsNull()) << "Invalid tuple must not have next item pointer"; } } LOG_INFO("[OK] oldest-to-newest version chain validated"); } }
/** * @brief Creates logical tile from tile group and applies scan predicate. * @return true on success, false otherwise. */ bool SeqScanExecutor::DExecute() { // Scanning over a logical tile. if (children_.size() == 1) { // FIXME Check all requirements for children_.size() == 0 case. LOG_TRACE("Seq Scan executor :: 1 child \n"); assert(target_table_ == nullptr); assert(column_ids_.size() == 0); while (children_[0]->Execute()) { std::unique_ptr<LogicalTile> tile(children_[0]->GetOutput()); if (predicate_ != nullptr) { // Invalidate tuples that don't satisfy the predicate. for (oid_t tuple_id : *tile) { expression::ContainerTuple<LogicalTile> tuple(tile.get(), tuple_id); if (predicate_->Evaluate(&tuple, nullptr, executor_context_) .IsFalse()) { tile->RemoveVisibility(tuple_id); } } } if (0 == tile->GetTupleCount()) { // Avoid returning empty tiles continue; } /* Hopefully we needn't do projections here */ SetOutput(tile.release()); return true; } return false; } // Scanning a table else if (children_.size() == 0) { LOG_TRACE("Seq Scan executor :: 0 child \n"); assert(target_table_ != nullptr); assert(column_ids_.size() > 0); // Retrieve next tile group. while (current_tile_group_offset_ < table_tile_group_count_) { auto tile_group = target_table_->GetTileGroup(current_tile_group_offset_++); storage::TileGroupHeader *tile_group_header = tile_group->GetHeader(); auto transaction_ = executor_context_->GetTransaction(); txn_id_t txn_id = transaction_->GetTransactionId(); cid_t commit_id = transaction_->GetLastCommitId(); oid_t active_tuple_count = tile_group->GetNextTupleSlot(); // Print tile group visibility // tile_group_header->PrintVisibility(txn_id, commit_id); // Construct logical tile. std::unique_ptr<LogicalTile> logical_tile(LogicalTileFactory::GetTile()); logical_tile->AddColumns(tile_group, column_ids_); // Construct position list by looping through tile group // and applying the predicate. std::vector<oid_t> position_list; for (oid_t tuple_id = 0; tuple_id < active_tuple_count; tuple_id++) { if (tile_group_header->IsVisible(tuple_id, txn_id, commit_id) == false) { continue; } expression::ContainerTuple<storage::TileGroup> tuple(tile_group.get(), tuple_id); if (predicate_ == nullptr) { position_list.push_back(tuple_id); } else { auto eval = predicate_->Evaluate(&tuple, nullptr, executor_context_).IsTrue(); if (eval == true) position_list.push_back(tuple_id); } } logical_tile->AddPositionList(std::move(position_list)); // Don't return empty tiles if (0 == logical_tile->GetTupleCount()) { continue; } SetOutput(logical_tile.release()); return true; } } return false; }