/**
* @brief create and eturn the backend logger based on logging type
* @param logging type can be stdout(debug), aries, peloton
*/
BackendLogger *BackendLogger::GetBackendLogger(LoggingType logging_type) {
BackendLogger *backend_logger = nullptr;
if (IsBasedOnWriteAheadLogging(logging_type) == true) {
backend_logger = new WriteAheadBackendLogger();
} else if (IsBasedOnWriteBehindLogging(logging_type) == true) {
backend_logger = new WriteBehindBackendLogger();
} else {
LOG_ERROR("Unsupported logging type");
}
return backend_logger;
}
/** * @brief Return the frontend logger based on logging type
* @param logging type can be write ahead logging or write behind logging
*/
FrontendLogger *FrontendLogger::GetFrontendLogger(LoggingType logging_type,
bool test_mode) {
FrontendLogger *frontend_logger = nullptr;
LOG_INFO("Logging_type is %d", (int)logging_type);
if (IsBasedOnWriteAheadLogging(logging_type) == true) {
frontend_logger = new WriteAheadFrontendLogger(test_mode);
} else if (IsBasedOnWriteBehindLogging(logging_type) == true) {
frontend_logger = new WriteBehindFrontendLogger();
} else {
LOG_ERROR("Unsupported logging type");
}
return frontend_logger;
}
void LogManager::LogInsert(cid_t commit_id, const ItemPointer &new_location) {
if (this->IsInLoggingMode()) {
auto logger = this->GetBackendLogger();
auto &manager = catalog::Manager::GetInstance();
auto new_tuple_tile_group = manager.GetTileGroup(new_location.block);
auto tile_group = manager.GetTileGroup(new_location.block);
std::unique_ptr<LogRecord> record;
std::unique_ptr<storage::Tuple> tuple;
if (IsBasedOnWriteAheadLogging(logging_type_)) {
auto schema =
manager.GetTableWithOid(tile_group->GetDatabaseId(),
tile_group->GetTableId())->GetSchema();
tuple.reset(new storage::Tuple(schema, true));
for (oid_t col = 0; col < schema->GetColumnCount(); col++) {
tuple->SetValue(
col, new_tuple_tile_group->GetValue(new_location.offset, col),
logger->GetVarlenPool());
}
record.reset(logger->GetTupleRecord(
LOGRECORD_TYPE_TUPLE_INSERT, commit_id, tile_group->GetTableId(),
new_tuple_tile_group->GetDatabaseId(), new_location,
INVALID_ITEMPOINTER, tuple.get()));
} else {
// do not construct the tuple for the wbl case
record.reset(logger->GetTupleRecord(LOGRECORD_TYPE_TUPLE_INSERT,
commit_id, tile_group->GetTableId(),
new_tuple_tile_group->GetDatabaseId(),
new_location, INVALID_ITEMPOINTER));
}
logger->Log(record.get());
}
}
void BuildLog(oid_t db_oid, oid_t table_oid) {
std::chrono::time_point<std::chrono::system_clock> start, end;
std::chrono::duration<double, std::milli> elapsed_milliseconds;
// Build a pool
auto logging_pool = new VarlenPool(BACKEND_TYPE_MM);
// Create db
CreateDatabase(db_oid);
auto& manager = catalog::Manager::GetInstance();
storage::Database* db = manager.GetDatabaseWithOid(db_oid);
// Create table, drop it and create again
// so that table can have a newly added tile group and
// not just the default tile group
storage::DataTable* table = CreateUserTable(db_oid, table_oid);
db->AddTable(table);
// Tuple count
oid_t per_backend_tuple_count = state.tuple_count / state.backend_count;
// Create Tuples
auto tuples =
CreateTuples(table->GetSchema(), per_backend_tuple_count, logging_pool);
//===--------------------------------------------------------------------===//
// ACTIVE PROCESSING
//===--------------------------------------------------------------------===//
start = std::chrono::system_clock::now();
// Execute the workload to build the log
std::vector<std::thread> thread_group;
oid_t num_threads = state.backend_count;
// Launch a group of threads
for (uint64_t thread_itr = 0; thread_itr < num_threads; ++thread_itr) {
thread_group.push_back(std::thread(RunBackends, table, tuples));
}
// Join the threads with the main thread
for (uint64_t thread_itr = 0; thread_itr < num_threads; ++thread_itr) {
thread_group[thread_itr].join();
}
end = std::chrono::system_clock::now();
elapsed_milliseconds = end - start;
// Build log time
if (state.experiment_type == EXPERIMENT_TYPE_ACTIVE ||
state.experiment_type == EXPERIMENT_TYPE_WAIT) {
WriteOutput(elapsed_milliseconds.count());
} else if (state.experiment_type == EXPERIMENT_TYPE_STORAGE) {
auto log_file_size = GetLogFileSize();
LOG_INFO("Log file size :: %lu", log_file_size);
WriteOutput(log_file_size);
}
// Clean up data
for (auto tuple : tuples) {
delete tuple;
}
// Check the tuple count if needed
if (state.check_tuple_count) {
oid_t total_expected = 0;
CheckTupleCount(db_oid, table_oid, total_expected);
}
// We can only drop the table in case of WAL
if (IsBasedOnWriteAheadLogging(peloton_logging_mode) == true) {
db->DropTableWithOid(table_oid);
DropDatabase(db_oid);
}
}
StorageManager::StorageManager()
: data_file_address(nullptr),
data_file_len(0),
data_file_offset(0) {
// Check if we need a data pool
if (IsBasedOnWriteAheadLogging(peloton_logging_mode) == true ||
peloton_logging_mode == LOGGING_TYPE_INVALID) {
return;
}
int data_fd;
std::string data_file_name;
struct stat data_stat;
// Initialize file size
if (peloton_data_file_size != 0)
data_file_len = peloton_data_file_size * 1024 * 1024; // MB
else
data_file_len = DATA_FILE_LEN;
// Check for relevant file system
bool found_file_system = false;
switch (peloton_logging_mode) {
// Check for NVM FS for data
case LOGGING_TYPE_NVM_NVM:
case LOGGING_TYPE_NVM_HDD: {
int status = stat(NVM_DIR, &data_stat);
if (status == 0 && S_ISDIR(data_stat.st_mode)) {
data_file_name = std::string(NVM_DIR) + std::string(DATA_FILE_NAME);
found_file_system = true;
}
} break;
// Check for HDD FS
case LOGGING_TYPE_HDD_NVM:
case LOGGING_TYPE_HDD_HDD: {
int status = stat(HDD_DIR, &data_stat);
if (status == 0 && S_ISDIR(data_stat.st_mode)) {
data_file_name = std::string(HDD_DIR) + std::string(DATA_FILE_NAME);
found_file_system = true;
}
} break;
default:
break;
}
// Fallback to tmp directory if needed
if (found_file_system == false) {
int status = stat(TMP_DIR, &data_stat);
if (status == 0 && S_ISDIR(data_stat.st_mode)) {
data_file_name = std::string(TMP_DIR) + std::string(DATA_FILE_NAME);
} else {
throw Exception("Could not find temp directory : " +
std::string(TMP_DIR));
}
}
// TODO:
LOG_TRACE("DATA DIR :: %s ", data_file_name.c_str());
// Create a data file
if ((data_fd = open(data_file_name.c_str(), O_CREAT | O_RDWR, 0666)) < 0) {
perror(data_file_name.c_str());
exit(EXIT_FAILURE);
}
// Allocate the data file
if ((errno = posix_fallocate(data_fd, 0, data_file_len)) != 0) {
perror("posix_fallocate");
exit(EXIT_FAILURE);
}
// map the data file in memory
if ((data_file_address = mmap(NULL,
data_file_len,
PROT_READ | PROT_WRITE,
MAP_SHARED,
data_fd, 0)) == MAP_FAILED) {
perror("mmap");
exit(EXIT_FAILURE);
}
// close the pmem file -- it will remain mapped
close(data_fd);
}
StorageManager::StorageManager()
: data_file_address(nullptr),
is_pmem(false),
data_file_len(0),
data_file_offset(0) {
// Check if we need a data pool
if (IsBasedOnWriteAheadLogging(peloton_logging_mode) == true ||
peloton_logging_mode == LOGGING_TYPE_INVALID) {
return;
}
int data_fd;
std::string data_file_name;
struct stat data_stat;
// Initialize file size
if (peloton_data_file_size != 0)
data_file_len = peloton_data_file_size * 1024 * 1024; // MB
else
data_file_len = DATA_FILE_LEN;
// Check for relevant file system
bool found_file_system = false;
switch (peloton_logging_mode) {
// Check for NVM FS for data
case LOGGING_TYPE_NVM_NVM:
case LOGGING_TYPE_NVM_HDD:
case LOGGING_TYPE_NVM_SSD: {
int status = stat(NVM_DIR, &data_stat);
if (status == 0 && S_ISDIR(data_stat.st_mode)) {
data_file_name = std::string(NVM_DIR) + std::string(DATA_FILE_NAME);
found_file_system = true;
}
} break;
// Check for SSD FS
case LOGGING_TYPE_SSD_NVM:
case LOGGING_TYPE_SSD_SSD:
case LOGGING_TYPE_SSD_HDD: {
int status = stat(SSD_DIR, &data_stat);
if (status == 0 && S_ISDIR(data_stat.st_mode)) {
data_file_name = std::string(SSD_DIR) + std::string(DATA_FILE_NAME);
found_file_system = true;
}
} break;
// Check for HDD FS
case LOGGING_TYPE_HDD_NVM:
case LOGGING_TYPE_HDD_SSD:
case LOGGING_TYPE_HDD_HDD: {
int status = stat(HDD_DIR, &data_stat);
if (status == 0 && S_ISDIR(data_stat.st_mode)) {
data_file_name = std::string(HDD_DIR) + std::string(DATA_FILE_NAME);
found_file_system = true;
}
} break;
default:
break;
}
// Fallback to tmp directory if needed
if (found_file_system == false) {
int status = stat(TMP_DIR, &data_stat);
if (status == 0 && S_ISDIR(data_stat.st_mode)) {
data_file_name = std::string(TMP_DIR) + std::string(DATA_FILE_NAME);
}
else {
throw Exception("Could not find temp directory : " + std::string(TMP_DIR));
}
}
LOG_INFO("DATA DIR :: %s ", data_file_name.c_str());
// TODO:
std::cout << "Data path :: " << data_file_name << "\n";
// Create a data file
if ((data_fd = open(data_file_name.c_str(), O_CREAT | O_RDWR, 0666)) < 0) {
perror(data_file_name.c_str());
exit(EXIT_FAILURE);
}
// Allocate the data file
if ((errno = posix_fallocate(data_fd, 0, data_file_len)) != 0) {
perror("posix_fallocate");
exit(EXIT_FAILURE);
}
// memory map the data file
if ((data_file_address = reinterpret_cast<char *>(pmem_map(data_fd))) ==
NULL) {
perror("pmem_map");
exit(EXIT_FAILURE);
}
// true only if the entire range [addr, addr+len) consists of persistent
// memory
is_pmem = pmem_is_pmem(data_file_address, data_file_len);
// close the pmem file -- it will remain mapped
close(data_fd);
}