/*@brief return all the columns this index indexed * @param index_oid * @param txn Transaction * @return a vector of column oid(logical position) */ std::vector<oid_t> IndexCatalog::GetIndexedAttributes( oid_t index_oid, concurrency::Transaction *txn) { std::vector<oid_t> column_ids({6}); // Indexed attributes oid_t index_offset = 0; // Index of index_oid std::vector<type::Value> values; values.push_back(type::ValueFactory::GetIntegerValue(index_oid).Copy()); std::vector<oid_t> key_attrs; std::string temp; auto result_tiles = GetResultWithIndexScan(column_ids, index_offset, values, txn); PL_ASSERT(result_tiles->size() <= 1); // index_oid is unique if (result_tiles->size() != 0) { PL_ASSERT((*result_tiles)[0]->GetTupleCount() <= 1); if ((*result_tiles)[0]->GetTupleCount() != 0) { temp = (*result_tiles)[0]->GetValue(0, 0).ToString(); } } LOG_TRACE("the string value for index keys is %s", temp.c_str()); // using " " as delimiter to split up string and turn into vector of oid_t std::stringstream os(temp.c_str()); // Turn the string into a stream. std::string tok; while (std::getline(os, tok, ' ')) { key_attrs.push_back(std::stoi(tok)); } LOG_TRACE("the size for indexed key is %lu", key_attrs.size()); return key_attrs; }
/* @brief First try get cached object from transaction cache. If cache miss, * construct database object from pg_database, and insert into the * cache. */ std::shared_ptr<DatabaseCatalogObject> DatabaseCatalog::GetDatabaseObject( const std::string &database_name, concurrency::TransactionContext *txn) { if (txn == nullptr) { throw CatalogException("Transaction is invalid!"); } // try get from cache auto database_object = txn->catalog_cache.GetDatabaseObject(database_name); if (database_object) return database_object; // cache miss, get from pg_database std::vector<oid_t> column_ids(all_column_ids); oid_t index_offset = IndexId::SKEY_DATABASE_NAME; // Index of database_name std::vector<type::Value> values; values.push_back( type::ValueFactory::GetVarcharValue(database_name, nullptr).Copy()); auto result_tiles = GetResultWithIndexScan(column_ids, index_offset, values, txn); if (result_tiles->size() == 1 && (*result_tiles)[0]->GetTupleCount() == 1) { auto database_object = std::make_shared<DatabaseCatalogObject>((*result_tiles)[0].get(), txn); if (database_object) { // insert into cache bool success = txn->catalog_cache.InsertDatabaseObject(database_object); PELOTON_ASSERT(success == true); (void)success; } return database_object; } // return empty object if not found return nullptr; }
int64_t QueryMetricsCatalog::GetNumParams(concurrency::TransactionContext *txn, const std::string &name) { std::vector<oid_t> column_ids({ColumnId::NUM_PARAMS}); // num_params oid_t index_offset = IndexId::PRIMARY_KEY; // Primary key index std::vector<type::Value> values; values.push_back(type::ValueFactory::GetVarcharValue(name, nullptr).Copy()); auto result_tiles = GetResultWithIndexScan(txn, column_ids, index_offset, values); int64_t num_params = 0; PELOTON_ASSERT(result_tiles->size() <= 1); // unique if (result_tiles->size() != 0) { PELOTON_ASSERT((*result_tiles)[0]->GetTupleCount() <= 1); if ((*result_tiles)[0]->GetTupleCount() != 0) { num_params = (*result_tiles)[0] ->GetValue(0, 0) .GetAs<int>(); // After projection left 1 column } } return num_params; }
stats::QueryMetric::QueryParamBuf QueryMetricsCatalog::GetParamTypes(concurrency::TransactionContext *txn, const std::string &name) { std::vector<oid_t> column_ids({ColumnId::PARAM_TYPES}); // param_types oid_t index_offset = IndexId::PRIMARY_KEY; // Primary key index std::vector<type::Value> values; values.push_back(type::ValueFactory::GetVarcharValue(name, nullptr).Copy()); values.push_back(type::ValueFactory::GetIntegerValue(database_oid_).Copy()); auto result_tiles = GetResultWithIndexScan(txn, column_ids, index_offset, values); stats::QueryMetric::QueryParamBuf param_types; PELOTON_ASSERT(result_tiles->size() <= 1); // unique if (result_tiles->size() != 0) { PELOTON_ASSERT((*result_tiles)[0]->GetTupleCount() <= 1); if ((*result_tiles)[0]->GetTupleCount() != 0) { auto param_types_value = (*result_tiles)[0]->GetValue(0, 0); param_types.buf = const_cast<uchar *>( reinterpret_cast<const uchar *>(param_types_value.GetData())); param_types.len = param_types_value.GetLength(); } } return param_types; }
/** * GetColumnSamples - Query column samples by db_id, table_id and column_id. */ void TupleSamplesStorage::GetColumnSamples( oid_t database_id, oid_t table_id, oid_t column_id, std::vector<type::Value> &column_samples) { auto catalog = catalog::Catalog::GetInstance(); std::string samples_table_name = GenerateSamplesTableName(database_id, table_id); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); auto data_table = catalog->GetTableWithName(std::string(SAMPLES_DB_NAME), std::string(DEFAULT_SCHEMA_NAME), samples_table_name, txn); std::vector<oid_t> column_ids({column_id}); auto result_tiles = GetTuplesWithSeqScan(data_table, column_ids, txn); txn_manager.CommitTransaction(txn); LOG_DEBUG("Result tiles count: %lu", result_tiles->size()); if (result_tiles->size() != 0) { auto tile = (*result_tiles)[0].get(); LOG_DEBUG("Tuple count: %lu", tile->GetTupleCount()); for (size_t tuple_id = 0; tuple_id < tile->GetTupleCount(); ++tuple_id) { column_samples.push_back(tile->GetValue(tuple_id, 0)); } } }
std::shared_ptr<DatabaseCatalogObject> DatabaseCatalog::GetDatabaseObject( oid_t database_oid, concurrency::TransactionContext *txn) { if (txn == nullptr) { throw CatalogException("Transaction is invalid!"); } // try get from cache auto database_object = txn->catalog_cache.GetDatabaseObject(database_oid); if (database_object) return database_object; // cache miss, get from pg_database std::vector<oid_t> column_ids(all_column_ids); oid_t index_offset = IndexId::PRIMARY_KEY; // Index of database_oid std::vector<type::Value> values; values.push_back(type::ValueFactory::GetIntegerValue(database_oid).Copy()); auto result_tiles = GetResultWithIndexScan(column_ids, index_offset, values, txn); if (result_tiles->size() == 1 && (*result_tiles)[0]->GetTupleCount() == 1) { auto database_object = std::make_shared<DatabaseCatalogObject>((*result_tiles)[0].get(), txn); // insert into cache bool success = txn->catalog_cache.InsertDatabaseObject(database_object); PELOTON_ASSERT(success == true); (void)success; return database_object; } else { LOG_DEBUG("Found %lu database tiles with oid %u", result_tiles->size(), database_oid); } // return empty object if not found return nullptr; }
std::shared_ptr<IndexCatalogEntry> IndexCatalog::GetIndexCatalogEntry( concurrency::TransactionContext *txn, const std::string &database_name, const std::string &schema_name, const std::string &index_name) { if (txn == nullptr) { throw CatalogException("Transaction is invalid!"); } // try get from cache auto index_object = txn->catalog_cache.GetCachedIndexObject(database_name, schema_name, index_name); if (index_object) { return index_object; } // cache miss, get from pg_index std::vector<oid_t> column_ids(all_column_ids); oid_t index_offset = IndexId::SKEY_INDEX_NAME; // Index of index_name & schema_name std::vector<type::Value> values; values.push_back( type::ValueFactory::GetVarcharValue(index_name, nullptr).Copy()); values.push_back( type::ValueFactory::GetVarcharValue(schema_name, nullptr).Copy()); auto result_tiles = GetResultWithIndexScan(txn, column_ids, index_offset, values); if (result_tiles->size() == 1 && (*result_tiles)[0]->GetTupleCount() == 1) { auto index_object = std::make_shared<IndexCatalogEntry>((*result_tiles)[0].get()); // fetch all indexes into table object (cannot use the above index object) auto pg_table = Catalog::GetInstance() ->GetSystemCatalogs(database_oid_) ->GetTableCatalog(); auto table_object = pg_table->GetTableCatalogEntry(txn, index_object->GetTableOid()); PELOTON_ASSERT(table_object && table_object->GetTableOid() == index_object->GetTableOid()); return table_object->GetIndexCatalogEntry(index_name); } else { LOG_DEBUG("Found %lu index with name %s", result_tiles->size(), index_name.c_str()); } // return empty object if not found return nullptr; }
std::unique_ptr<trigger::TriggerList> TriggerCatalog::GetTriggers( oid_t table_oid, concurrency::TransactionContext *txn) { // LOG_DEBUG("Get triggers for table %d", table_oid); // select trigger_name, fire condition, function_name, function_args std::vector<oid_t> column_ids( {ColumnId::TRIGGER_NAME, ColumnId::TRIGGER_TYPE, ColumnId::FIRE_CONDITION, ColumnId::FUNCTION_OID, ColumnId::FUNCTION_ARGS}); oid_t index_offset = IndexId::TABLE_KEY_1; std::vector<type::Value> values; // where database_oid = args.database_oid and table_oid = args.table_oid and // trigger_type = args.trigger_type values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); // the result is a vector of executor::LogicalTile auto result_tiles = GetResultWithIndexScan(column_ids, index_offset, values, txn); // carefull! the result tile could be null! // if (result_tiles == nullptr) { // LOG_INFO("no trigger on table %d", table_oid); // } else { // LOG_INFO("size of the result tiles = %lu", result_tiles->size()); // } // create the trigger list std::unique_ptr<trigger::TriggerList> new_trigger_list{ new trigger::TriggerList()}; if (result_tiles != nullptr) { for (unsigned int i = 0; i < result_tiles->size(); i++) { size_t tuple_count = (*result_tiles)[i]->GetTupleCount(); for (size_t j = 0; j < tuple_count; j++) { // create a new trigger instance trigger::Trigger new_trigger( (*result_tiles)[i]->GetValue(j, 0).ToString(), (*result_tiles)[i]->GetValue(j, 1).GetAs<int16_t>(), (*result_tiles)[i]->GetValue(j, 3).ToString(), (*result_tiles)[i]->GetValue(j, 4).ToString(), (*result_tiles)[i]->GetValue(j, 2).GetData()); new_trigger_list->AddTrigger(new_trigger); } } } return new_trigger_list; }
/*@brief get all index records from the same table * this function may be useful when calling DropTable * @param table_oid * @param txn Transaction * @return a vector of index oid */ std::vector<oid_t> IndexCatalog::GetIndexOids(oid_t table_oid, concurrency::Transaction *txn) { std::vector<oid_t> column_ids({0}); // index_oid oid_t index_offset = 2; // Index of table_oid std::vector<type::Value> values; values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); auto result_tiles = GetResultWithIndexScan(column_ids, index_offset, values, txn); std::vector<oid_t> index_oids; for (auto &tile : (*result_tiles)) { for (auto tuple_id : *tile) { index_oids.emplace_back( tile->GetValue(tuple_id, 0) .GetAs<oid_t>()); // After projection left 1 column } } return index_oids; }
/*@brief get all index records from the same table * this function may be useful when calling DropTable * @param table_oid * @param txn TransactionContext * @return a vector of index catalog objects */ const std::unordered_map<oid_t, std::shared_ptr<IndexCatalogEntry>> IndexCatalog::GetIndexCatalogEntries( concurrency::TransactionContext *txn, oid_t table_oid) { if (txn == nullptr) { throw CatalogException("Transaction is invalid!"); } // try get from cache auto pg_table = Catalog::GetInstance() ->GetSystemCatalogs(database_oid_) ->GetTableCatalog(); auto table_object = pg_table->GetTableCatalogEntry(txn, table_oid); PELOTON_ASSERT(table_object && table_object->GetTableOid() == table_oid); auto index_objects = table_object->GetIndexCatalogEntries(true); if (index_objects.empty() == false) return index_objects; // cache miss, get from pg_index std::vector<oid_t> column_ids(all_column_ids); oid_t index_offset = IndexId::SKEY_TABLE_OID; // Index of table_oid std::vector<type::Value> values; values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); auto result_tiles = GetResultWithIndexScan(txn, column_ids, index_offset, values); for (auto &tile : (*result_tiles)) { for (auto tuple_id : *tile) { auto index_object = std::make_shared<IndexCatalogEntry>(tile.get(), tuple_id); table_object->InsertIndexCatalogEntry(index_object); } } return table_object->GetIndexCatalogEntries(); }
bool IndexCatalog::IsUniqueKeys(oid_t index_oid, concurrency::Transaction *txn) { std::vector<oid_t> column_ids({5}); // unique_keys oid_t index_offset = 0; // Index of index_oid std::vector<type::Value> values; values.push_back(type::ValueFactory::GetIntegerValue(index_oid).Copy()); auto result_tiles = GetResultWithIndexScan(column_ids, index_offset, values, txn); bool unique_keys = false; PL_ASSERT(result_tiles->size() <= 1); // index_oid is unique if (result_tiles->size() != 0) { PL_ASSERT((*result_tiles)[0]->GetTupleCount() <= 1); if ((*result_tiles)[0]->GetTupleCount() != 0) { unique_keys = (*result_tiles)[0] ->GetValue(0, 0) .GetAs<bool>(); // After projection left 1 column } } return unique_keys; }
oid_t IndexCatalog::GetTableOid(oid_t index_oid, concurrency::Transaction *txn) { std::vector<oid_t> column_ids({2}); // table_oid oid_t index_offset = 0; // Index of index_oid std::vector<type::Value> values; values.push_back(type::ValueFactory::GetIntegerValue(index_oid).Copy()); auto result_tiles = GetResultWithIndexScan(column_ids, index_offset, values, txn); oid_t table_oid = INVALID_OID; PL_ASSERT(result_tiles->size() <= 1); // table_oid is unique if (result_tiles->size() != 0) { PL_ASSERT((*result_tiles)[0]->GetTupleCount() <= 1); if ((*result_tiles)[0]->GetTupleCount() != 0) { table_oid = (*result_tiles)[0] ->GetValue(0, 0) .GetAs<oid_t>(); // After projection left 1 column } } return table_oid; }
oid_t TriggerCatalog::GetTriggerOid(std::string trigger_name, oid_t table_oid, concurrency::TransactionContext *txn) { std::vector<oid_t> column_ids({ColumnId::TRIGGER_OID}); oid_t index_offset = IndexId::NAME_TABLE_KEY_2; std::vector<type::Value> values; values.push_back(type::ValueFactory::GetVarcharValue(trigger_name).Copy()); values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); auto result_tiles = GetResultWithIndexScan(column_ids, index_offset, values, txn); oid_t trigger_oid = INVALID_OID; if (result_tiles->size() == 0) { // LOG_INFO("trigger %s doesn't exist", trigger_name.c_str()); } else { // LOG_INFO("size of the result tiles = %lu", result_tiles->size()); PELOTON_ASSERT((*result_tiles)[0]->GetTupleCount() <= 1); if ((*result_tiles)[0]->GetTupleCount() != 0) { trigger_oid = (*result_tiles)[0]->GetValue(0, 0).GetAs<oid_t>(); } } return trigger_oid; }
std::string IndexCatalog::GetIndexName(oid_t index_oid, concurrency::Transaction *txn) { std::vector<oid_t> column_ids({1}); // index_name oid_t index_offset = 0; // Index of index_oid std::vector<type::Value> values; values.push_back(type::ValueFactory::GetIntegerValue(index_oid).Copy()); auto result_tiles = GetResultWithIndexScan(column_ids, index_offset, values, txn); std::string index_name; PL_ASSERT(result_tiles->size() <= 1); // index_oid is unique if (result_tiles->size() != 0) { PL_ASSERT((*result_tiles)[0]->GetTupleCount() <= 1); if ((*result_tiles)[0]->GetTupleCount() != 0) { index_name = (*result_tiles)[0] ->GetValue(0, 0) .ToString(); // After projection left 1 column } } return index_name; }
IndexConstraintType IndexCatalog::GetIndexConstraint( oid_t index_oid, concurrency::Transaction *txn) { std::vector<oid_t> column_ids({4}); // index_constraint oid_t index_offset = 0; // Index of index_oid std::vector<type::Value> values; values.push_back(type::ValueFactory::GetIntegerValue(index_oid).Copy()); auto result_tiles = GetResultWithIndexScan(column_ids, index_offset, values, txn); IndexConstraintType index_constraint = IndexConstraintType::INVALID; PL_ASSERT(result_tiles->size() <= 1); // index_oid is unique if (result_tiles->size() != 0) { PL_ASSERT((*result_tiles)[0]->GetTupleCount() <= 1); if ((*result_tiles)[0]->GetTupleCount() != 0) { index_constraint = static_cast<IndexConstraintType>( (*result_tiles)[0] ->GetValue(0, 0) .GetAs<int>()); // After projection left 1 column } } return index_constraint; }
void ExecuteTileGroupTest() { const int tuples_per_tilegroup_count = 10; const int tile_group_count = 5; const int tuple_count = tuples_per_tilegroup_count * tile_group_count; const oid_t col_count = 250; const bool is_inlined = true; const bool indexes = false; std::vector<catalog::Column> columns; for (oid_t col_itr = 0; col_itr <= col_count; col_itr++) { auto column = catalog::Column(VALUE_TYPE_INTEGER, GetTypeSize(VALUE_TYPE_INTEGER), "FIELD" + std::to_string(col_itr), is_inlined); columns.push_back(column); } catalog::Schema *table_schema = new catalog::Schema(columns); std::string table_name("TEST_TABLE"); ///////////////////////////////////////////////////////// // Create table. ///////////////////////////////////////////////////////// bool own_schema = true; bool adapt_table = true; std::unique_ptr<storage::DataTable> table(storage::TableFactory::GetDataTable( INVALID_OID, INVALID_OID, table_schema, table_name, tuples_per_tilegroup_count, own_schema, adapt_table)); // PRIMARY INDEX if (indexes == true) { std::vector<oid_t> key_attrs; auto tuple_schema = table->GetSchema(); catalog::Schema *key_schema; index::IndexMetadata *index_metadata; bool unique; key_attrs = {0}; key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs); key_schema->SetIndexedColumns(key_attrs); unique = true; index_metadata = new index::IndexMetadata( "primary_index", 123, INDEX_TYPE_BTREE, INDEX_CONSTRAINT_TYPE_PRIMARY_KEY, tuple_schema, key_schema, unique); index::Index *pkey_index = index::IndexFactory::GetInstance(index_metadata); table->AddIndex(pkey_index); } ///////////////////////////////////////////////////////// // Load in the data ///////////////////////////////////////////////////////// // Insert tuples into tile_group. auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); const bool allocate = true; auto txn = txn_manager.BeginTransaction(); auto testing_pool = TestingHarness::GetInstance().GetTestingPool(); for (int rowid = 0; rowid < tuple_count; rowid++) { int populate_value = rowid; storage::Tuple tuple(table_schema, allocate); for (oid_t col_itr = 0; col_itr <= col_count; col_itr++) { auto value = ValueFactory::GetIntegerValue(populate_value + col_itr); tuple.SetValue(col_itr, value, testing_pool); } ItemPointer tuple_slot_id = table->InsertTuple(&tuple); EXPECT_TRUE(tuple_slot_id.block != INVALID_OID); EXPECT_TRUE(tuple_slot_id.offset != INVALID_OID); txn_manager.PerformInsert(tuple_slot_id.block, tuple_slot_id.offset); } txn_manager.CommitTransaction(); ///////////////////////////////////////////////////////// // Do a seq scan with predicate on top of the table ///////////////////////////////////////////////////////// txn = txn_manager.BeginTransaction(); std::unique_ptr<executor::ExecutorContext> context( new executor::ExecutorContext(txn)); // Column ids to be added to logical tile after scan. // std::vector<oid_t> column_ids; // for(oid_t col_itr = 0 ; col_itr <= 200; col_itr++) { // column_ids.push_back(col_itr); //} std::vector<oid_t> column_ids({198, 206}); // Create and set up seq scan executor planner::SeqScanPlan seq_scan_node(table.get(), nullptr, column_ids); int expected_num_tiles = tile_group_count; executor::SeqScanExecutor seq_scan_executor(&seq_scan_node, context.get()); // Create and set up materialization executor std::vector<catalog::Column> output_columns; std::unordered_map<oid_t, oid_t> old_to_new_cols; oid_t col_itr = 0; for (auto column_id : column_ids) { auto column = catalog::Column(VALUE_TYPE_INTEGER, GetTypeSize(VALUE_TYPE_INTEGER), "FIELD" + std::to_string(column_id), is_inlined); output_columns.push_back(column); old_to_new_cols[col_itr] = col_itr; col_itr++; } std::unique_ptr<catalog::Schema> output_schema( new catalog::Schema(output_columns)); bool physify_flag = true; // is going to create a physical tile planner::MaterializationPlan mat_node(old_to_new_cols, output_schema.release(), physify_flag); executor::MaterializationExecutor mat_executor(&mat_node, nullptr); mat_executor.AddChild(&seq_scan_executor); EXPECT_TRUE(mat_executor.Init()); std::vector<std::unique_ptr<executor::LogicalTile>> result_tiles; for (int i = 0; i < expected_num_tiles; i++) { EXPECT_TRUE(mat_executor.Execute()); std::unique_ptr<executor::LogicalTile> result_tile( mat_executor.GetOutput()); EXPECT_THAT(result_tile, NotNull()); result_tiles.emplace_back(result_tile.release()); } EXPECT_FALSE(mat_executor.Execute()); txn_manager.CommitTransaction(); }
// Index scan of table with index predicate. TEST_F(IndexScanTests, IndexPredicateTest) { // First, generate the table with index std::unique_ptr<storage::DataTable> data_table( ExecutorTestsUtil::CreateAndPopulateTable()); // Column ids to be added to logical tile after scan. std::vector<oid_t> column_ids({0, 1, 3}); //===--------------------------------------------------------------------===// // ATTR 0 <= 110 //===--------------------------------------------------------------------===// auto index = data_table->GetIndex(0); std::vector<oid_t> key_column_ids; std::vector<ExpressionType> expr_types; std::vector<Value> values; std::vector<expression::AbstractExpression *> runtime_keys; key_column_ids.push_back(0); expr_types.push_back( ExpressionType::EXPRESSION_TYPE_COMPARE_LESSTHANOREQUALTO); values.push_back(ValueFactory::GetIntegerValue(110)); // Create index scan desc planner::IndexScanPlan::IndexScanDesc index_scan_desc( index, key_column_ids, expr_types, values, runtime_keys); expression::AbstractExpression *predicate = nullptr; // Create plan node. planner::IndexScanPlan node(data_table.get(), predicate, column_ids, index_scan_desc); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); std::unique_ptr<executor::ExecutorContext> context( new executor::ExecutorContext(txn)); // Run the executor executor::IndexScanExecutor executor(&node, context.get()); int expected_num_tiles = 3; EXPECT_TRUE(executor.Init()); std::vector<std::unique_ptr<executor::LogicalTile>> result_tiles; for (int i = 0; i < expected_num_tiles; i++) { EXPECT_TRUE(executor.Execute()); std::unique_ptr<executor::LogicalTile> result_tile(executor.GetOutput()); EXPECT_THAT(result_tile, NotNull()); result_tiles.emplace_back(result_tile.release()); } EXPECT_FALSE(executor.Execute()); EXPECT_EQ(result_tiles.size(), expected_num_tiles); EXPECT_EQ(result_tiles[0].get()->GetTupleCount(), 5); EXPECT_EQ(result_tiles[1].get()->GetTupleCount(), 5); EXPECT_EQ(result_tiles[2].get()->GetTupleCount(), 2); txn_manager.CommitTransaction(); }
column_id_set class_mapper_base::imports() const { return column_ids(); }