bool ConfigTable::RetrieveAllEntries(std::deque<ConfigEntry>& out) {
bool ret = true;
std::string query;
query += "SELECT * FROM ";
query += table_name();
query += ";";
std::string error;
SelectResult res;
if(Select(query, res, error)) {
int step = 0;
for(int i=0; i<res.row()+1; i++) {
step = i*res.col();
if(step > 0) {
ConfigEntry entry;
ExtractEntryResults(res, step, entry);
out.push_back(entry);
}
}
ret = true;
}
else {
log::LogString("ct_122851", error);
}
return ret;
}
bool ConfigTable::IsValueInManifest(const std::string& value) {
bool ret = false;
std::string query;
query += "SELECT EXISTS(SELECT * FROM ";
query += table_name();
query += " WHERE value=\"";
query += value;
query += "\");";
std::string error;
SelectResult res;
if(Select(query, res, error)) {
int step = 0;
for(int i=0; i<res.row()+1; i++) {
step = i*res.col();
if(step > 0) {
std::string r = res.results()[0+step];
if(r == "1") {
ret = true;
break;
}
}
}
}
else {
log::LogString("ct_nnasd92s", error);
}
return ret;
}
void CreateTable() {
const oid_t col_count = state.column_count + 1;
const bool is_inlined = true;
// Create schema first
std::vector<catalog::Column> columns;
for (oid_t col_itr = 0; col_itr < col_count; col_itr++) {
auto column =
catalog::Column(VALUE_TYPE_INTEGER, GetTypeSize(VALUE_TYPE_INTEGER),
"" + std::to_string(col_itr), is_inlined);
columns.push_back(column);
}
catalog::Schema *table_schema = new catalog::Schema(columns);
std::string table_name("SDBENCHTABLE");
/////////////////////////////////////////////////////////
// Create table.
/////////////////////////////////////////////////////////
bool own_schema = true;
bool adapt_table = true;
sdbench_table.reset(storage::TableFactory::GetDataTable(
INVALID_OID, INVALID_OID, table_schema, table_name,
state.tuples_per_tilegroup, own_schema, adapt_table));
}
bool ConfigTable::RetrieveConfigValueByState(const std::string& state, std::string& out) {
bool ret = false;
std::string query;
query += "SELECT * FROM ";
query += table_name();
query += " WHERE state=\"";
query += state;
query += "\";";
std::string error;
SelectResult res;
if(Select(query, res, error)) {
int step = 0;
for(int i=0; i<res.row()+1; i++) {
step = i*res.col();
if(step > 0) {
out = res.results()[2+step];
ret = true;
break;
}
}
}
else {
log::LogString("ct_9d11f", error);
}
return ret;
}
bool ConfigTable::RetrieveConfigEntry(const std::string& key, ConfigEntry& out) {
bool ret = false;
std::string query;
query += "SELECT * FROM ";
query += table_name();
query += " WHERE config_key=\"";
query += key;
query += "\";";
std::string error;
SelectResult res;
if(Select(query, res, error)) {
int step = 0;
for(int i=0; i<res.row()+1; i++) {
step = i*res.col();
if(step > 0) {
ExtractEntryResults(res, step, out);
ret = true;
break;
}
}
}
else {
log::LogString("ct_1045jhkdaf", error);
}
return ret;
}
void CreateTable() {
const oid_t col_count = state.column_count + 1;
const bool is_inlined = true;
// Create schema first
std::vector<catalog::Column> columns;
for (oid_t col_itr = 0; col_itr < col_count; col_itr++) {
auto column =
catalog::Column(VALUE_TYPE_INTEGER, GetTypeSize(VALUE_TYPE_INTEGER),
"" + std::to_string(col_itr), is_inlined);
columns.push_back(column);
}
catalog::Schema *table_schema = new catalog::Schema(columns);
std::string table_name("HYADAPTTABLE");
/////////////////////////////////////////////////////////
// Create table.
/////////////////////////////////////////////////////////
bool own_schema = true;
bool adapt_table = true;
sdbench_table.reset(storage::TableFactory::GetDataTable(
INVALID_OID, INVALID_OID, table_schema, table_name,
state.tuples_per_tilegroup, own_schema, adapt_table));
// PRIMARY INDEXES
for(int index_itr = 0; index_itr < state.index_count; index_itr++) {
std::vector<oid_t> key_attrs;
auto tuple_schema = sdbench_table->GetSchema();
catalog::Schema *key_schema;
index::IndexMetadata *index_metadata;
bool unique;
key_attrs = {0};
key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs);
key_schema->SetIndexedColumns(key_attrs);
unique = true;
index_metadata = new index::IndexMetadata(
"primary_index",
index_itr,
INDEX_TYPE_SKIPLIST,
INDEX_CONSTRAINT_TYPE_PRIMARY_KEY,
tuple_schema,
key_schema,
unique);
index::Index *pkey_index = index::IndexFactory::GetInstance(index_metadata);
sdbench_table->AddIndex(pkey_index);
}
}
const char *table_path( )
{
static const std::string path =
std::string( names::client_table )
+ '.'
+ table_name( );
return path.c_str( );
}
TEST_FIXTURE(table_service_test_base, table_name_unicode)
{
utility::string_t table_name(_XPLATSTR("表格1"));
azure::storage::cloud_table_client client = get_table_client();
azure::storage::cloud_table table = client.get_table_reference(table_name);
CHECK(false == table.exists());
CHECK_STORAGE_EXCEPTION(table.create(), error_bad_request);
}
TEST_F(LoggingTests, BasicLogManagerTest) {
peloton_logging_mode = LOGGING_TYPE_INVALID;
auto &log_manager = logging::LogManager::GetInstance();
log_manager.DropFrontendLoggers();
log_manager.SetLoggingStatus(LOGGING_STATUS_TYPE_INVALID);
// just start, write a few records and exit
catalog::Schema *table_schema = new catalog::Schema(
{ExecutorTestsUtil::GetColumnInfo(0), ExecutorTestsUtil::GetColumnInfo(1),
ExecutorTestsUtil::GetColumnInfo(2),
ExecutorTestsUtil::GetColumnInfo(3)});
std::string table_name("TEST_TABLE");
// Create table.
bool own_schema = true;
bool adapt_table = false;
storage::DataTable *table = storage::TableFactory::GetDataTable(
12345, 123456, table_schema, table_name, 1, own_schema, adapt_table);
storage::Database test_db(12345);
test_db.AddTable(table);
catalog::Manager::GetInstance().AddDatabase(&test_db);
concurrency::TransactionManager &txn_manager =
concurrency::TransactionManagerFactory::GetInstance();
txn_manager.BeginTransaction();
ExecutorTestsUtil::PopulateTable(table, 5, true, false, false);
txn_manager.CommitTransaction();
peloton_logging_mode = LOGGING_TYPE_NVM_WAL;
log_manager.SetSyncCommit(true);
EXPECT_FALSE(log_manager.ContainsFrontendLogger());
log_manager.StartStandbyMode();
log_manager.GetFrontendLogger(0)->SetTestMode(true);
log_manager.StartRecoveryMode();
log_manager.WaitForModeTransition(LOGGING_STATUS_TYPE_LOGGING, true);
EXPECT_TRUE(log_manager.ContainsFrontendLogger());
log_manager.SetGlobalMaxFlushedCommitId(4);
concurrency::Transaction test_txn;
cid_t commit_id = 5;
log_manager.PrepareLogging();
log_manager.LogBeginTransaction(commit_id);
ItemPointer insert_loc(table->GetTileGroup(1)->GetTileGroupId(), 0);
ItemPointer delete_loc(table->GetTileGroup(2)->GetTileGroupId(), 0);
ItemPointer update_old(table->GetTileGroup(3)->GetTileGroupId(), 0);
ItemPointer update_new(table->GetTileGroup(4)->GetTileGroupId(), 0);
log_manager.LogInsert(commit_id, insert_loc);
log_manager.LogUpdate(commit_id, update_old, update_new);
log_manager.LogInsert(commit_id, delete_loc);
log_manager.LogCommitTransaction(commit_id);
// TODO: Check the flushed commit id
// since we are doing sync commit we should have reached 5 already
//EXPECT_EQ(commit_id, log_manager.GetPersistentFlushedCommitId());
log_manager.EndLogging();
}
/* Config Notes*
* Root directory cong
* (config_key == alias,
* value == post_id,
* state == relative_path (set by local))
*/
bool ConfigTable::CreateTable() {
std::string exc;
exc += "CREATE TABLE IF NOT EXISTS ";
exc += table_name();
exc += " (type TEXT, config_key TEXT, value TEXT, state TEXT,";
exc += " PRIMARY KEY(type ASC, config_key ASC, value ASC, state ASC));";
std::string error;
bool ret = Exec(exc, error);
if(!ret)
log::LogString("ct_102482", error);
return ret;
}
void CreateTable(std::unique_ptr<storage::DataTable> &hyadapt_table,
bool build_indexes) {
const bool is_inlined = true;
// Create schema first
std::vector<catalog::Column> columns;
for (oid_t col_itr = 0; col_itr < column_count; col_itr++) {
auto column = catalog::Column(type::TypeId::INTEGER,
type::Type::GetTypeSize(type::TypeId::INTEGER),
std::to_string(col_itr), is_inlined);
columns.push_back(column);
}
catalog::Schema *table_schema = new catalog::Schema(columns);
std::string table_name("HYADAPT_TABLE");
/////////////////////////////////////////////////////////
// Create table.
/////////////////////////////////////////////////////////
bool own_schema = true;
bool adapt_table = true;
hyadapt_table.reset(storage::TableFactory::GetDataTable(
INVALID_OID, INVALID_OID, table_schema, table_name, tuples_per_tile_group,
own_schema, adapt_table));
// PRIMARY INDEX
if (build_indexes == true) {
std::vector<oid_t> key_attrs;
auto tuple_schema = hyadapt_table->GetSchema();
catalog::Schema *key_schema;
index::IndexMetadata *index_metadata;
bool unique;
key_attrs = {0};
key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs);
key_schema->SetIndexedColumns(key_attrs);
unique = true;
index_metadata = new index::IndexMetadata(
"primary_index", 123, INVALID_OID, INVALID_OID, IndexType::BWTREE,
IndexConstraintType::PRIMARY_KEY, tuple_schema, key_schema, key_attrs,
unique);
std::shared_ptr<index::Index> pkey_index(
index::IndexFactory::GetIndex(index_metadata));
hyadapt_table->AddIndex(pkey_index);
}
}
storage::DataTable *ExecutorTestsUtil::CreateTable(
int tuples_per_tilegroup_count, bool indexes) {
catalog::Schema *table_schema = new catalog::Schema(
{GetColumnInfo(0), GetColumnInfo(1), GetColumnInfo(2), GetColumnInfo(3)});
std::string table_name("TEST_TABLE");
// Create table.
bool own_schema = true;
bool adapt_table = false;
storage::DataTable *table = storage::TableFactory::GetDataTable(
INVALID_OID, INVALID_OID, table_schema, table_name,
tuples_per_tilegroup_count, own_schema, adapt_table);
if (indexes == true) {
// PRIMARY INDEX
std::vector<oid_t> key_attrs;
auto tuple_schema = table->GetSchema();
catalog::Schema *key_schema;
index::IndexMetadata *index_metadata;
bool unique;
key_attrs = {0};
key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs);
key_schema->SetIndexedColumns(key_attrs);
unique = true;
index_metadata = new index::IndexMetadata(
"primary_btree_index", 123, INDEX_TYPE_BTREE,
INDEX_CONSTRAINT_TYPE_PRIMARY_KEY, tuple_schema, key_schema, unique);
index::Index *pkey_index = index::IndexFactory::GetInstance(index_metadata);
table->AddIndex(pkey_index);
// SECONDARY INDEX
key_attrs = {0, 1};
key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs);
key_schema->SetIndexedColumns(key_attrs);
unique = false;
index_metadata = new index::IndexMetadata(
"secondary_btree_index", 124, INDEX_TYPE_BTREE,
INDEX_CONSTRAINT_TYPE_DEFAULT, tuple_schema, key_schema, unique);
index::Index *sec_index = index::IndexFactory::GetInstance(index_metadata);
table->AddIndex(sec_index);
}
return table;
}
bool ConfigTable::set_state_for_key(const std::string& key, const std::string& state) {
std::string exc;
exc += "UPDATE ";
exc += table_name();
exc += " SET state=\"";
exc += state;
exc += "\" WHERE config_key=\"";
exc += key;
exc += "\";";
std::string error;
bool ret = Exec(exc, error);
if(!ret)
log::LogString("ct_18915", error);
return ret;
}
bool ConfigTable::set_state_for_value(const std::string& value, const std::string& state) {
std::string exc;
exc += "UPDATE ";
exc += table_name();
exc += " SET state=\"";
exc += state;
exc += "\" WHERE value=\"";
exc += value;
exc += "\";";
std::string error;
bool ret = Exec(exc, error);
if(!ret)
log::LogString("ct_11110", error);
return ret;
}
main(int argc, char *argv[]){
/** Check the number of parameters */
if (argc < 6) {
/** Tell the user how to run the program */
cerr << "Usage: " << argv[0] << " key_file rows encrows noise num_threads" << endl;
return 1;
}
/** Set the random seed for noise parameter generation */
srand(time(NULL));
mr_init_threading();
PFC pfc(AES_SECURITY);
SecureSelect *db=NULL;
int m=0;
string key_file(argv[1]);
string table_name(argv[2]);
string enctable_name(argv[3]);
int rand_lim = atoi(argv[4]);
int num_threads = atoi(argv[5]);
db = new SecureSelect(&pfc,pfc.order());
if(!db->LoadKey(key_file))
return 0;
if(rand_lim<1){
cout << "Random paramter < 1, it has to be >= 1" << endl;
return 0;
}
#ifdef VERBOSE
int start = getMilliCount();
#endif
db->EncryptRowsMT(table_name,enctable_name,rand_lim, num_threads);
// db->EncryptRows(table_name,enctable_name,rand_lim);
#ifdef VERBOSE
int milliSecondsElapsed = getMilliSpan(start);
cout << "\texec time " << milliSecondsElapsed << endl;
#endif
}
Status Base::destroy() {
if (_connection == nullptr) {
return Status::status_ailment();
}
std::string id;
try {
id = _fields.at("id");
} catch (std::out_of_range &e) {
return Status::status_ailment();
}
fmt::MemoryWriter buf;
buf << "DELETE FROM " << table_name() << " WHERE id = " << id;
_connection->execute_sql(buf.str());
return Status::ok();
}
// Write the compact table
void CompactHashtableWriter::dump(char** top, char* end) {
NumberSeq summary;
char* old_top = *top;
juint* p = (juint*)(*top);
uintx base_address = uintx(MetaspaceShared::shared_rs()->base());
// Now write the following at the beginning of the table:
// base_address (uintx)
// num_entries (juint)
// num_buckets (juint)
*p++ = high(base_address);
*p++ = low (base_address); // base address
*p++ = _num_entries; // number of entries in the table
*p++ = _num_buckets; // number of buckets in the table
juint* first_bucket = NULL;
juint* compact_table = dump_table(p, &first_bucket, &summary);
juint* bucket_end = dump_buckets(compact_table, first_bucket, &summary);
assert(bucket_end <= (juint*)end, "cannot write past end");
*top = (char*)bucket_end;
if (PrintSharedSpaces) {
double avg_cost = 0.0;
if (_num_entries > 0) {
avg_cost = double(_required_bytes)/double(_num_entries);
}
tty->print_cr("Shared %s table stats -------- base: " PTR_FORMAT,
table_name(), (intptr_t)base_address);
tty->print_cr("Number of entries : %9d", _num_entries);
tty->print_cr("Total bytes used : %9d", (int)((*top) - old_top));
tty->print_cr("Average bytes per entry : %9.3f", avg_cost);
tty->print_cr("Average bucket size : %9.3f", summary.avg());
tty->print_cr("Variance of bucket size : %9.3f", summary.variance());
tty->print_cr("Std. dev. of bucket size: %9.3f", summary.sd());
tty->print_cr("Maximum bucket size : %9d", (int)summary.maximum());
}
}
bool ConfigTable::InsertConfigValue(const std::string& type,
const std::string& key,
const std::string& value,
const std::string& state) {
bool ret = false;
std::string exc;
if(!IsKeyInManifest(key))
exc += "INSERT OR REPLACE INTO ";
else
exc += "REPLACE INTO ";
exc += table_name();
exc += " (type, config_key, value, state) VALUES (?,?,?,?);";
std::string error;
ret = PrepareStatement(exc, error); if(!ret) {log::ls("m_340s",error);return ret;}
ret = BindText(1, type, error); if(!ret) {log::ls("m_341s",error);return ret;}
ret = BindText(2, key, error); if(!ret) {log::ls("m_342s",error);return ret;}
ret = BindText(3, value, error); if(!ret) {log::ls("m_343s",error);return ret;}
ret = BindText(4, state, error); if(!ret) {log::ls("m_344s",error);return ret;}
ret = StepStatement(error); if(!ret) {log::ls("m_353s",error);return ret;}
ret = FinalizeStatement(error); if(!ret) {log::ls("m_354s",error);return ret;}
return ret;
}
void ExecuteTileGroupTest() {
const int tuples_per_tilegroup_count = 10;
const int tile_group_count = 5;
const int tuple_count = tuples_per_tilegroup_count * tile_group_count;
const oid_t col_count = 250;
const bool is_inlined = true;
const bool indexes = false;
std::vector<catalog::Column> columns;
for (oid_t col_itr = 0; col_itr <= col_count; col_itr++) {
auto column =
catalog::Column(VALUE_TYPE_INTEGER, GetTypeSize(VALUE_TYPE_INTEGER),
"FIELD" + std::to_string(col_itr), is_inlined);
columns.push_back(column);
}
catalog::Schema *table_schema = new catalog::Schema(columns);
std::string table_name("TEST_TABLE");
/////////////////////////////////////////////////////////
// Create table.
/////////////////////////////////////////////////////////
bool own_schema = true;
bool adapt_table = true;
std::unique_ptr<storage::DataTable> table(storage::TableFactory::GetDataTable(
INVALID_OID, INVALID_OID, table_schema, table_name,
tuples_per_tilegroup_count, own_schema, adapt_table));
// PRIMARY INDEX
if (indexes == true) {
std::vector<oid_t> key_attrs;
auto tuple_schema = table->GetSchema();
catalog::Schema *key_schema;
index::IndexMetadata *index_metadata;
bool unique;
key_attrs = {0};
key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs);
key_schema->SetIndexedColumns(key_attrs);
unique = true;
index_metadata = new index::IndexMetadata(
"primary_index", 123, INDEX_TYPE_BTREE,
INDEX_CONSTRAINT_TYPE_PRIMARY_KEY, tuple_schema, key_schema, unique);
index::Index *pkey_index = index::IndexFactory::GetInstance(index_metadata);
table->AddIndex(pkey_index);
}
/////////////////////////////////////////////////////////
// Load in the data
/////////////////////////////////////////////////////////
// Insert tuples into tile_group.
auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance();
const bool allocate = true;
auto txn = txn_manager.BeginTransaction();
auto testing_pool = TestingHarness::GetInstance().GetTestingPool();
for (int rowid = 0; rowid < tuple_count; rowid++) {
int populate_value = rowid;
storage::Tuple tuple(table_schema, allocate);
for (oid_t col_itr = 0; col_itr <= col_count; col_itr++) {
auto value = ValueFactory::GetIntegerValue(populate_value + col_itr);
tuple.SetValue(col_itr, value, testing_pool);
}
ItemPointer tuple_slot_id = table->InsertTuple(&tuple);
EXPECT_TRUE(tuple_slot_id.block != INVALID_OID);
EXPECT_TRUE(tuple_slot_id.offset != INVALID_OID);
txn_manager.PerformInsert(tuple_slot_id.block, tuple_slot_id.offset);
}
txn_manager.CommitTransaction();
/////////////////////////////////////////////////////////
// Do a seq scan with predicate on top of the table
/////////////////////////////////////////////////////////
txn = txn_manager.BeginTransaction();
std::unique_ptr<executor::ExecutorContext> context(
new executor::ExecutorContext(txn));
// Column ids to be added to logical tile after scan.
// std::vector<oid_t> column_ids;
// for(oid_t col_itr = 0 ; col_itr <= 200; col_itr++) {
// column_ids.push_back(col_itr);
//}
std::vector<oid_t> column_ids({198, 206});
// Create and set up seq scan executor
planner::SeqScanPlan seq_scan_node(table.get(), nullptr, column_ids);
int expected_num_tiles = tile_group_count;
executor::SeqScanExecutor seq_scan_executor(&seq_scan_node, context.get());
// Create and set up materialization executor
std::vector<catalog::Column> output_columns;
std::unordered_map<oid_t, oid_t> old_to_new_cols;
oid_t col_itr = 0;
for (auto column_id : column_ids) {
auto column =
catalog::Column(VALUE_TYPE_INTEGER, GetTypeSize(VALUE_TYPE_INTEGER),
"FIELD" + std::to_string(column_id), is_inlined);
output_columns.push_back(column);
old_to_new_cols[col_itr] = col_itr;
col_itr++;
}
std::unique_ptr<catalog::Schema> output_schema(
new catalog::Schema(output_columns));
bool physify_flag = true; // is going to create a physical tile
planner::MaterializationPlan mat_node(old_to_new_cols,
output_schema.release(), physify_flag);
executor::MaterializationExecutor mat_executor(&mat_node, nullptr);
mat_executor.AddChild(&seq_scan_executor);
EXPECT_TRUE(mat_executor.Init());
std::vector<std::unique_ptr<executor::LogicalTile>> result_tiles;
for (int i = 0; i < expected_num_tiles; i++) {
EXPECT_TRUE(mat_executor.Execute());
std::unique_ptr<executor::LogicalTile> result_tile(
mat_executor.GetOutput());
EXPECT_THAT(result_tile, NotNull());
result_tiles.emplace_back(result_tile.release());
}
EXPECT_FALSE(mat_executor.Execute());
txn_manager.CommitTransaction();
}
storage::DataTable *ConstraintsTestsUtil::CreateTable(
int tuples_per_tilegroup_count, bool indexes) {
catalog::Schema *table_schema = new catalog::Schema(
{GetColumnInfo(0), GetColumnInfo(1), GetColumnInfo(2), GetColumnInfo(3)});
std::string table_name("TEST_TABLE");
// Create table.
bool own_schema = true;
bool adapt_table = false;
storage::DataTable *table = storage::TableFactory::GetDataTable(
INVALID_OID, INVALID_OID, table_schema, table_name,
tuples_per_tilegroup_count, own_schema, adapt_table);
if (indexes == true) {
// PRIMARY INDEX
std::vector<oid_t> key_attrs;
auto tuple_schema = table->GetSchema();
catalog::Schema *key_schema;
index::IndexMetadata *index_metadata;
bool unique;
key_attrs = {0};
key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs);
key_schema->SetIndexedColumns(key_attrs);
unique = true;
index_metadata = new index::IndexMetadata(
"primary_btree_index", 123, table->GetOid(), table->GetDatabaseOid(),
IndexType::BWTREE, IndexConstraintType::PRIMARY_KEY, tuple_schema,
key_schema, key_attrs, unique);
std::shared_ptr<index::Index> pkey_index(
index::IndexFactory::GetIndex(index_metadata));
table->AddIndex(pkey_index);
// SECONDARY INDEX
key_attrs = {0, 1};
key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs);
key_schema->SetIndexedColumns(key_attrs);
unique = false;
index_metadata = new index::IndexMetadata(
"secondary_btree_index", 124, table->GetOid(), table->GetDatabaseOid(),
IndexType::BWTREE, IndexConstraintType::DEFAULT, tuple_schema,
key_schema, key_attrs, unique);
std::shared_ptr<index::Index> sec_index(
index::IndexFactory::GetIndex(index_metadata));
table->AddIndex(sec_index);
// SECONDARY INDEX - UNIQUE INDEX
key_attrs = {3};
key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs);
key_schema->SetIndexedColumns(key_attrs);
unique = false;
index_metadata = new index::IndexMetadata(
"unique_btree_index", 125, table->GetOid(), table->GetDatabaseOid(),
IndexType::BWTREE, IndexConstraintType::UNIQUE, tuple_schema,
key_schema, key_attrs, unique);
std::shared_ptr<index::Index> unique_index(
index::IndexFactory::GetIndex(index_metadata));
table->AddIndex(unique_index);
}
return table;
}
void CreateYCSBDatabase() {
const oid_t col_count = state.column_count + 1;
const bool is_inlined = true;
/////////////////////////////////////////////////////////
// Create tables
/////////////////////////////////////////////////////////
// Clean up
delete ycsb_database;
ycsb_database = nullptr;
user_table = nullptr;
auto &manager = catalog::Manager::GetInstance();
ycsb_database = new storage::Database(ycsb_database_oid);
manager.AddDatabase(ycsb_database);
bool own_schema = true;
bool adapt_table = false;
// Create schema first
std::vector<catalog::Column> columns;
auto column =
catalog::Column(VALUE_TYPE_INTEGER, GetTypeSize(VALUE_TYPE_INTEGER),
"YCSB_KEY", is_inlined);
columns.push_back(column);
for (oid_t col_itr = 1; col_itr < col_count; col_itr++) {
auto column =
catalog::Column(VALUE_TYPE_INTEGER, ycsb_field_length,
"FIELD" + std::to_string(col_itr), is_inlined);
columns.push_back(column);
}
catalog::Schema *table_schema = new catalog::Schema(columns);
std::string table_name("USERTABLE");
user_table = storage::TableFactory::GetDataTable(
ycsb_database_oid, user_table_oid, table_schema, table_name, 1000,
own_schema, adapt_table);
ycsb_database->AddTable(user_table);
// Primary index on user key
std::vector<oid_t> key_attrs;
auto tuple_schema = user_table->GetSchema();
catalog::Schema *key_schema;
index::IndexMetadata *index_metadata;
bool unique;
key_attrs = {0};
key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs);
key_schema->SetIndexedColumns(key_attrs);
unique = true;
index_metadata = new index::IndexMetadata(
"primary_index", user_table_pkey_index_oid, INDEX_TYPE_BTREE,
INDEX_CONSTRAINT_TYPE_INVALID, tuple_schema, key_schema, unique);
index::Index *pkey_index = index::IndexFactory::GetInstance(index_metadata);
user_table->AddIndex(pkey_index);
}
void CreateYCSBDatabase() {
const oid_t col_count = state.column_count + 1;
const bool is_inlined = false;
/////////////////////////////////////////////////////////
// Create tables
/////////////////////////////////////////////////////////
// Clean up
delete ycsb_database;
ycsb_database = nullptr;
user_table = nullptr;
auto catalog = catalog::Catalog::GetInstance();
ycsb_database = new storage::Database(ycsb_database_oid);
catalog->AddDatabase(ycsb_database);
bool own_schema = true;
bool adapt_table = false;
// Create schema first
std::vector<catalog::Column> columns;
auto column =
catalog::Column(type::Type::INTEGER, type::Type::GetTypeSize(type::Type::INTEGER),
"YCSB_KEY", is_inlined);
columns.push_back(column);
if (state.string_mode == true) {
for (oid_t col_itr = 1; col_itr < col_count; col_itr++) {
auto column =
catalog::Column(type::Type::VARCHAR, 100,
"FIELD" + std::to_string(col_itr), is_inlined);
columns.push_back(column);
}
} else {
for (oid_t col_itr = 1; col_itr < col_count; col_itr++) {
auto column =
catalog::Column(type::Type::INTEGER, type::Type::GetTypeSize(type::Type::INTEGER),
"FIELD" + std::to_string(col_itr), is_inlined);
columns.push_back(column);
}
}
catalog::Schema *table_schema = new catalog::Schema(columns);
std::string table_name("USERTABLE");
user_table = storage::TableFactory::GetDataTable(
ycsb_database_oid, user_table_oid, table_schema, table_name,
DEFAULT_TUPLES_PER_TILEGROUP, own_schema, adapt_table);
ycsb_database->AddTable(user_table);
// Primary index on user key
std::vector<oid_t> key_attrs;
auto tuple_schema = user_table->GetSchema();
catalog::Schema *key_schema;
index::IndexMetadata *index_metadata;
bool unique;
key_attrs = {0};
key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs);
key_schema->SetIndexedColumns(key_attrs);
unique = true;
index_metadata = new index::IndexMetadata(
"primary_index", user_table_pkey_index_oid, user_table_oid,
ycsb_database_oid, state.index, INDEX_CONSTRAINT_TYPE_PRIMARY_KEY,
tuple_schema, key_schema, key_attrs, unique);
std::shared_ptr<index::Index> pkey_index(
index::IndexFactory::GetIndex(index_metadata));
user_table->AddIndex(pkey_index);
}
//! ofObject = true => returns list of objects this object depends on
//! ofObject = false => returns list of objects that depend on this object
void MetadataItem::getDependencies(std::vector<Dependency>& list,
bool ofObject)
{
DatabasePtr d = getDatabase();
int mytype = -1; // map DBH type to RDB$DEPENDENT TYPE
NodeType dep_types[] = { ntTable, ntView, ntTrigger, ntUnknown, ntUnknown,
ntProcedure,ntUnknown, ntException,ntUnknown, ntUnknown,
ntUnknown, ntUnknown, ntUnknown, ntUnknown, ntGenerator,
ntFunction
};
int type_count = sizeof(dep_types)/sizeof(NodeType);
for (int i = 0; i < type_count; i++)
if (typeM == dep_types[i])
mytype = i;
// system tables should be treated as tables
if (typeM == ntSysTable)
mytype = 0;
int mytype2 = mytype;
// views count as relations(tables) when other object refer to them
if (mytype == 1 && !ofObject)
mytype2 = 0;
if (typeM == ntUnknown || mytype == -1)
throw FRError(_("Unsupported type"));
IBPP::Database& db = d->getIBPPDatabase();
IBPP::Transaction tr1 = IBPP::TransactionFactory(db, IBPP::amRead);
tr1->Start();
IBPP::Statement st1 = IBPP::StatementFactory(db, tr1);
wxString o1 = (ofObject ? "DEPENDENT" : "DEPENDED_ON");
wxString o2 = (ofObject ? "DEPENDED_ON" : "DEPENDENT");
wxString sql =
"select RDB$" + o2 + "_TYPE, RDB$" + o2 + "_NAME, RDB$FIELD_NAME \n "
" from RDB$DEPENDENCIES \n "
" where RDB$" + o1 + "_TYPE in (?,?) and RDB$" + o1 + "_NAME = ? \n ";
int params = 1;
if ((typeM == ntTable || typeM == ntSysTable || typeM == ntView) && ofObject) // get deps for computed columns
{ // view needed to bind with generators
sql += " union \n"
" SELECT DISTINCT d.rdb$depended_on_type, d.rdb$depended_on_name, d.rdb$field_name \n"
" FROM rdb$relation_fields f \n"
" LEFT JOIN rdb$dependencies d ON d.rdb$dependent_name = f.rdb$field_source \n"
" WHERE d.rdb$dependent_type = 3 AND f.rdb$relation_name = ? \n";
params++;
}
if (!ofObject) // find tables that have calculated columns based on "this" object
{
sql += "union \n"
" SELECT distinct cast(0 as smallint), f.rdb$relation_name, f.rdb$field_name \n"
" from rdb$relation_fields f \n"
" left join rdb$dependencies d on d.rdb$dependent_name = f.rdb$field_source \n"
" where d.rdb$dependent_type = 3 and d.rdb$depended_on_name = ? ";
params++;
}
// get the exact table and fields for views
// rdb$dependencies covers deps. for WHERE clauses in SELECTs in VIEW body
// but we also need mapping for column list in SELECT. These 2 queries cover it:
if (ofObject && typeM == ntView)
{
sql += " union \n"
" select distinct cast(0 as smallint), vr.RDB$RELATION_NAME, f.RDB$BASE_FIELD \n"
" from RDB$RELATION_FIELDS f \n"
" join RDB$VIEW_RELATIONS vr on f.RDB$VIEW_CONTEXT = vr.RDB$VIEW_CONTEXT \n"
" and f.RDB$RELATION_NAME = vr.RDB$VIEW_NAME \n"
" where f.rdb$relation_name = ? \n";
params++;
}
// views can depend on other views as well
// we might need to add procedures here one day when Firebird gains support for it
if (!ofObject && (typeM == ntView || typeM == ntTable || typeM == ntSysTable))
{
sql += " union \n"
" select distinct cast(0 as smallint), f.RDB$RELATION_NAME, f.RDB$BASE_FIELD \n"
" from RDB$RELATION_FIELDS f \n"
" join RDB$VIEW_RELATIONS vr on f.RDB$VIEW_CONTEXT = vr.RDB$VIEW_CONTEXT \n"
" and f.RDB$RELATION_NAME = vr.RDB$VIEW_NAME \n"
" where vr.rdb$relation_name = ? \n";
params++;
}
sql += " order by 1, 2, 3";
st1->Prepare(wx2std(sql, d->getCharsetConverter()));
st1->Set(1, mytype);
st1->Set(2, mytype2);
for (int i = 0; i < params; i++)
st1->Set(3 + i, wx2std(getName_(), d->getCharsetConverter()));
st1->Execute();
MetadataItem* last = 0;
Dependency* dep = 0;
while (st1->Fetch())
{
int object_type;
st1->Get(1, &object_type);
if (object_type > type_count) // some system object, not interesting for us
continue;
NodeType t = dep_types[object_type];
if (t == ntUnknown) // ditto
continue;
std::string objname_std;
st1->Get(2, objname_std);
wxString objname(std2wxIdentifier(objname_std,
d->getCharsetConverter()));
MetadataItem* current = d->findByNameAndType(t, objname);
if (!current)
{
if (t == ntTable) {
// maybe it's a view masked as table
current = d->findByNameAndType(ntView, objname);
// or possibly a system table
if (!current)
current = d->findByNameAndType(ntSysTable, objname);
}
if (!ofObject && t == ntTrigger)
{
// system trigger dependent of this object indicates possible check constraint on a table
// that references this object. So, let's check if this trigger is used for check constraint
// and get that table's name
IBPP::Statement st2 = IBPP::StatementFactory(db, tr1);
st2->Prepare(
"select r.rdb$relation_name from rdb$relation_constraints r "
" join rdb$check_constraints c on r.rdb$constraint_name=c.rdb$constraint_name "
" and r.rdb$constraint_type = 'CHECK' where c.rdb$trigger_name = ? "
);
st2->Set(1, objname_std);
st2->Execute();
if (st2->Fetch()) // table using that trigger found
{
std::string s;
st2->Get(1, s);
wxString tablecheck(std2wxIdentifier(s, d->getCharsetConverter()));
if (getName_() != tablecheck) // avoid self-reference
current = d->findByNameAndType(ntTable, tablecheck);
}
}
if (!current)
continue;
}
if (current != last) // new object
{
Dependency de(current);
list.push_back(de);
dep = &list.back();
last = current;
}
if (!st1->IsNull(3))
{
std::string s;
st1->Get(3, s);
dep->addField(std2wxIdentifier(s, d->getCharsetConverter()));
}
}
// TODO: perhaps this could be moved to Table?
// call MetadataItem::getDependencies() and then add this
if ((typeM == ntTable || typeM == ntSysTable) && ofObject) // foreign keys of this table + computed columns
{
Table *t = dynamic_cast<Table *>(this);
std::vector<ForeignKey> *f = t->getForeignKeys();
for (std::vector<ForeignKey>::const_iterator it = f->begin();
it != f->end(); ++it)
{
MetadataItem *table = d->findByNameAndType(ntTable,
(*it).getReferencedTable());
if (!table)
{
throw FRError(wxString::Format(_("Table %s not found."),
(*it).getReferencedTable().c_str()));
}
Dependency de(table);
de.setFields((*it).getReferencedColumns());
list.push_back(de);
}
// Add check constraints here (CHECKS are checked via system triggers), example:
// table1::check( table1.field1 > select max(field2) from table2 )
// So, table vs any object from this ^^^ select
// Algorithm: 1.find all system triggers bound to that CHECK constraint
// 2.find dependencies for those system triggers
// 3.display those dependencies as deps. of this table
st1->Prepare("select distinct c.rdb$trigger_name from rdb$relation_constraints r "
" join rdb$check_constraints c on r.rdb$constraint_name=c.rdb$constraint_name "
" and r.rdb$constraint_type = 'CHECK' where r.rdb$relation_name= ? "
);
st1->Set(1, wx2std(getName_(), d->getCharsetConverter()));
st1->Execute();
std::vector<Dependency> tempdep;
while (st1->Fetch())
{
std::string s;
st1->Get(1, s);
Trigger t(d->shared_from_this(),
std2wxIdentifier(s, d->getCharsetConverter()));
t.getDependencies(tempdep, true);
}
// remove duplicates, and self-references from "tempdep"
while (true)
{
std::vector<Dependency>::iterator to_remove = tempdep.end();
for (std::vector<Dependency>::iterator it = tempdep.begin();
it != tempdep.end(); ++it)
{
if ((*it).getDependentObject() == this)
{
to_remove = it;
break;
}
to_remove = std::find(it + 1, tempdep.end(), (*it));
if (to_remove != tempdep.end())
break;
}
if (to_remove == tempdep.end())
break;
else
tempdep.erase(to_remove);
}
list.insert(list.end(), tempdep.begin(), tempdep.end());
}
// TODO: perhaps this could be moved to Table?
if ((typeM == ntTable || typeM == ntSysTable) && !ofObject) // foreign keys of other tables
{
st1->Prepare(
"select r1.rdb$relation_name, i.rdb$field_name "
" from rdb$relation_constraints r1 "
" join rdb$ref_constraints c on r1.rdb$constraint_name = c.rdb$constraint_name "
" join rdb$relation_constraints r2 on c.RDB$CONST_NAME_UQ = r2.rdb$constraint_name "
" join rdb$index_segments i on r1.rdb$index_name=i.rdb$index_name "
" where r2.rdb$relation_name=? "
" and r1.rdb$constraint_type='FOREIGN KEY' "
);
st1->Set(1, wx2std(getName_(), d->getCharsetConverter()));
st1->Execute();
wxString lasttable;
Dependency* dep = 0;
while (st1->Fetch())
{
std::string s;
st1->Get(1, s);
wxString table_name(std2wxIdentifier(s, d->getCharsetConverter()));
st1->Get(2, s);
wxString field_name(std2wxIdentifier(s, d->getCharsetConverter()));
if (table_name != lasttable) // new
{
MetadataItem* table = d->findByNameAndType(ntTable, table_name);
if (!table)
continue; // dummy check
Dependency de(table);
list.push_back(de);
dep = &list.back();
lasttable = table_name;
}
dep->addField(field_name);
}
}
tr1->Commit();
}
/**
* @brief insert data
* @detail check whether the table we have created or not. forbid to insert data
* into an nonexistent table.
* There are two different cases when we insert data:
* 1. insert all columns
* 2. insert part of all columns
* To case 1, we just insert all rows values and make sure the value count and
* type match each column.
* To case 2, make sure each value correspond to its column and its type.
* If no exceptions and errors, data will be stored into tables and hdfs proper
* time.
* @return a result code cooperate with the client.
*/
RetCode InsertExec::Execute(ExecutedResult *exec_result) {
int ret = rSuccess;
string table_name(insert_ast_->table_name_);
TableDescriptor *table =
Environment::getInstance()->getCatalog()->getTable(table_name);
if (table == NULL) {
ret = rTableNotExisted;
exec_result->SetError("The table " + table_name + " does not exist!");
ELOG(ret, "table name: " << table_name);
return ret;
}
unsigned col_count = 0;
AstColumn *col = dynamic_cast<AstColumn *>(insert_ast_->col_list_);
if (NULL == col) {
is_all_col_ = true; // insert all columns
} else { // get insert column count
++col_count;
while ((col = dynamic_cast<AstColumn *>(col->next_))) ++col_count;
}
AstInsertValList *insert_value_list =
dynamic_cast<AstInsertValList *>(insert_ast_->insert_val_list_);
if (NULL == insert_value_list) {
LOG(ERROR) << "No value!" << endl;
exec_result->SetError("No value!");
ret = common::rStmtHandlerInsertNoValue;
} else {
std::ostringstream ostr;
int changed_row_num = 0;
// get data in one row like (...), (...), (...) by while loop.
while (insert_value_list) {
// make sure: the insert column count = insert value count = used column
// count = used value count
AstInsertVals *insert_value =
dynamic_cast<AstInsertVals *>(insert_value_list->insert_vals_);
col = dynamic_cast<AstColumn *>(insert_ast_->col_list_);
if (is_all_col_) {
// by scdong: Claims adds a default row_id attribute for all tables
// which is attribute(0),
// when inserting tuples we should begin to construct the string_tuple
// from the second attribute.
for (unsigned int position = 1;
position < table_desc_->getNumberOfAttribute(); position++) {
// check value count
if (insert_value == NULL) {
LOG(ERROR) << "Value count is too few. Expected value count is "
<< table_desc_->getNumberOfAttribute() << endl;
exec_result->SetError("Value count is too few");
return claims::common::rFailure;
}
// insert value to ostringstream and if has warning return 1; look
// out the order!
ret = InsertValueToStream(insert_value, table_desc_, position, ostr);
// move to next
insert_value = dynamic_cast<AstInsertVals *>(insert_value->next_);
ostr << "|";
}
if (rSuccess != ret) break;
// check insert value count
if (NULL != insert_value) {
LOG(ERROR) << "Value count is too many";
exec_result->SetError("Value count is too many");
return claims::common::rFailure;
}
} else { // insert part of columns
// get insert value count and check whether it match column count
unsigned insert_value_count = 0;
while (insert_value) {
++insert_value_count;
insert_value = dynamic_cast<AstInsertVals *>(insert_value->next_);
}
if (insert_value_count != col_count) {
LOG(ERROR) << "Column count doesn't match value count. "
"insert_value_count is " << insert_value_count
<< ", col_count is: " << col_count << endl;
exec_result->SetError("Column count doesn't match value count");
return claims::common::rFailure;
}
unsigned int used_col_count = 0;
// by scdong: Claims adds a default row_id attribute for all tables
// which is attribute(0), when inserting tuples we should begin to
// construct the string_tuple from the second attribute.
for (unsigned int position = 1;
position < table_desc_->getNumberOfAttribute(); position++) {
// find the matched column and value by name
col = dynamic_cast<AstColumn *>(insert_ast_->col_list_);
AstInsertVals *insert_value =
dynamic_cast<AstInsertVals *>(insert_value_list->insert_vals_);
// take attention that attrName is tablename.colname
// here is temporary code and need to change later
while (col &&
(table_desc_->getAttribute(position).attrName).compare(
table_desc_->getTableName() + "." + col->column_name_)) {
col = dynamic_cast<AstColumn *>(col->next_);
insert_value = dynamic_cast<AstInsertVals *>(insert_value->next_);
}
// if find the column count is proved to match the insert value
// count, so column exist, then insert_value exist
if (col && insert_value) {
++used_col_count;
// insert value to ostringstream and if has warning return 1; look
// out the order!
if (rSuccess != (ret = InsertValueToStream(insert_value, table,
position, ostr))) {
ELOG(ret, "failed to insert value to stream");
exec_result->SetError("Not supported type to insert");
return claims::common::rFailure;
}
}
ostr << "|";
}
// check if every insert column is existed
if (used_col_count != col_count) {
exec_result->SetError("Some columns don't exist");
LOG(ERROR) << "Some columns don't exist. "
"used_col_count is " << used_col_count
<< ", col_count is: " << col_count << endl;
return claims::common::rFailure;
}
}
if (!is_correct_) break;
insert_value_list =
dynamic_cast<AstInsertValList *>(insert_value_list->next_);
if (insert_value_list != NULL) ostr << "\n";
++changed_row_num;
} // while
if (!is_correct_) return claims::common::rFailure;
DataInjector *injector = new DataInjector(table);
// str() will copy string buffer without the last '\n'
ret = injector->InsertFromString(ostr.str() + "\n", exec_result);
if (rSuccess == ret) {
ostr.clear();
ostr.str("");
ostr << "insert data successfully. " << changed_row_num
<< " rows changed.";
exec_result->SetResult(ostr.str(), NULL);
} else {
LOG(ERROR) << "failed to insert tuples into table:"
<< table->getTableName() << endl;
exec_result->SetError("failed to insert tuples into table ");
}
DELETE_PTR(injector);
Environment::getInstance()->getCatalog()->saveCatalog();
}
return ret;
}
Status Base::save() {
if (_connection == nullptr) {
return Status::status_ailment();
}
before_save();
set_updated_at();
fmt::MemoryWriter buf;
if (_new_record) {
before_create();
set_created_at();
buf << "INSERT INTO " << table_name() << " (";
std::vector<std::tuple<std::string, std::string> > values;
for (auto &one : _fields) {
if (one.first == "id") continue;
values.push_back(std::make_tuple(one.first, one.second));
}
auto size = values.size();
for (auto &one : values) {
size -= 1;
buf << std::get<0>(one);
if (0 < size) {
buf << ", ";
}
}
buf << ") VALUES (";
size = values.size();
for (auto &one : values) {
size -= 1;
buf << "'" << std::get<1>(one) << "'";
if (0 < size) {
buf << ", ";
}
}
buf << ");";
} else {
before_update();
buf << "UPDATE " << table_name() << " SET ";
auto size = _dirty_keys.size();
for (auto &key : _dirty_keys) {
size -= 1;
buf << key << " = '" << _fields[key] << "'";
if (0 < size) {
buf << ", ";
}
}
buf << " WHERE id = " << _fields["id"];
}
_connection->execute_sql(buf.str());
if (_new_record) {
auto id = _connection->last_row_id();
_set_field("id", fmt::format("{0}", id));
after_create();
} else {
after_update();
}
after_save();
return Status::ok();
}
