/** * @brief create and eturn the backend logger based on logging type * @param logging type can be stdout(debug), aries, peloton */ BackendLogger *BackendLogger::GetBackendLogger(LoggingType logging_type) { BackendLogger *backend_logger = nullptr; if (IsBasedOnWriteAheadLogging(logging_type) == true) { backend_logger = new WriteAheadBackendLogger(); } else if (IsBasedOnWriteBehindLogging(logging_type) == true) { backend_logger = new WriteBehindBackendLogger(); } else { LOG_ERROR("Unsupported logging type"); } return backend_logger; }
/** * @brief Return the frontend logger based on logging type * @param logging type can be write ahead logging or write behind logging */ FrontendLogger *FrontendLogger::GetFrontendLogger(LoggingType logging_type, bool test_mode) { FrontendLogger *frontend_logger = nullptr; LOG_INFO("Logging_type is %d", (int)logging_type); if (IsBasedOnWriteAheadLogging(logging_type) == true) { frontend_logger = new WriteAheadFrontendLogger(test_mode); } else if (IsBasedOnWriteBehindLogging(logging_type) == true) { frontend_logger = new WriteBehindFrontendLogger(); } else { LOG_ERROR("Unsupported logging type"); } return frontend_logger; }
void LogManager::LogInsert(cid_t commit_id, const ItemPointer &new_location) { if (this->IsInLoggingMode()) { auto logger = this->GetBackendLogger(); auto &manager = catalog::Manager::GetInstance(); auto new_tuple_tile_group = manager.GetTileGroup(new_location.block); auto tile_group = manager.GetTileGroup(new_location.block); std::unique_ptr<LogRecord> record; std::unique_ptr<storage::Tuple> tuple; if (IsBasedOnWriteAheadLogging(logging_type_)) { auto schema = manager.GetTableWithOid(tile_group->GetDatabaseId(), tile_group->GetTableId())->GetSchema(); tuple.reset(new storage::Tuple(schema, true)); for (oid_t col = 0; col < schema->GetColumnCount(); col++) { tuple->SetValue( col, new_tuple_tile_group->GetValue(new_location.offset, col), logger->GetVarlenPool()); } record.reset(logger->GetTupleRecord( LOGRECORD_TYPE_TUPLE_INSERT, commit_id, tile_group->GetTableId(), new_tuple_tile_group->GetDatabaseId(), new_location, INVALID_ITEMPOINTER, tuple.get())); } else { // do not construct the tuple for the wbl case record.reset(logger->GetTupleRecord(LOGRECORD_TYPE_TUPLE_INSERT, commit_id, tile_group->GetTableId(), new_tuple_tile_group->GetDatabaseId(), new_location, INVALID_ITEMPOINTER)); } logger->Log(record.get()); } }
void BuildLog(oid_t db_oid, oid_t table_oid) { std::chrono::time_point<std::chrono::system_clock> start, end; std::chrono::duration<double, std::milli> elapsed_milliseconds; // Build a pool auto logging_pool = new VarlenPool(BACKEND_TYPE_MM); // Create db CreateDatabase(db_oid); auto& manager = catalog::Manager::GetInstance(); storage::Database* db = manager.GetDatabaseWithOid(db_oid); // Create table, drop it and create again // so that table can have a newly added tile group and // not just the default tile group storage::DataTable* table = CreateUserTable(db_oid, table_oid); db->AddTable(table); // Tuple count oid_t per_backend_tuple_count = state.tuple_count / state.backend_count; // Create Tuples auto tuples = CreateTuples(table->GetSchema(), per_backend_tuple_count, logging_pool); //===--------------------------------------------------------------------===// // ACTIVE PROCESSING //===--------------------------------------------------------------------===// start = std::chrono::system_clock::now(); // Execute the workload to build the log std::vector<std::thread> thread_group; oid_t num_threads = state.backend_count; // Launch a group of threads for (uint64_t thread_itr = 0; thread_itr < num_threads; ++thread_itr) { thread_group.push_back(std::thread(RunBackends, table, tuples)); } // Join the threads with the main thread for (uint64_t thread_itr = 0; thread_itr < num_threads; ++thread_itr) { thread_group[thread_itr].join(); } end = std::chrono::system_clock::now(); elapsed_milliseconds = end - start; // Build log time if (state.experiment_type == EXPERIMENT_TYPE_ACTIVE || state.experiment_type == EXPERIMENT_TYPE_WAIT) { WriteOutput(elapsed_milliseconds.count()); } else if (state.experiment_type == EXPERIMENT_TYPE_STORAGE) { auto log_file_size = GetLogFileSize(); LOG_INFO("Log file size :: %lu", log_file_size); WriteOutput(log_file_size); } // Clean up data for (auto tuple : tuples) { delete tuple; } // Check the tuple count if needed if (state.check_tuple_count) { oid_t total_expected = 0; CheckTupleCount(db_oid, table_oid, total_expected); } // We can only drop the table in case of WAL if (IsBasedOnWriteAheadLogging(peloton_logging_mode) == true) { db->DropTableWithOid(table_oid); DropDatabase(db_oid); } }
StorageManager::StorageManager() : data_file_address(nullptr), data_file_len(0), data_file_offset(0) { // Check if we need a data pool if (IsBasedOnWriteAheadLogging(peloton_logging_mode) == true || peloton_logging_mode == LOGGING_TYPE_INVALID) { return; } int data_fd; std::string data_file_name; struct stat data_stat; // Initialize file size if (peloton_data_file_size != 0) data_file_len = peloton_data_file_size * 1024 * 1024; // MB else data_file_len = DATA_FILE_LEN; // Check for relevant file system bool found_file_system = false; switch (peloton_logging_mode) { // Check for NVM FS for data case LOGGING_TYPE_NVM_NVM: case LOGGING_TYPE_NVM_HDD: { int status = stat(NVM_DIR, &data_stat); if (status == 0 && S_ISDIR(data_stat.st_mode)) { data_file_name = std::string(NVM_DIR) + std::string(DATA_FILE_NAME); found_file_system = true; } } break; // Check for HDD FS case LOGGING_TYPE_HDD_NVM: case LOGGING_TYPE_HDD_HDD: { int status = stat(HDD_DIR, &data_stat); if (status == 0 && S_ISDIR(data_stat.st_mode)) { data_file_name = std::string(HDD_DIR) + std::string(DATA_FILE_NAME); found_file_system = true; } } break; default: break; } // Fallback to tmp directory if needed if (found_file_system == false) { int status = stat(TMP_DIR, &data_stat); if (status == 0 && S_ISDIR(data_stat.st_mode)) { data_file_name = std::string(TMP_DIR) + std::string(DATA_FILE_NAME); } else { throw Exception("Could not find temp directory : " + std::string(TMP_DIR)); } } // TODO: LOG_TRACE("DATA DIR :: %s ", data_file_name.c_str()); // Create a data file if ((data_fd = open(data_file_name.c_str(), O_CREAT | O_RDWR, 0666)) < 0) { perror(data_file_name.c_str()); exit(EXIT_FAILURE); } // Allocate the data file if ((errno = posix_fallocate(data_fd, 0, data_file_len)) != 0) { perror("posix_fallocate"); exit(EXIT_FAILURE); } // map the data file in memory if ((data_file_address = mmap(NULL, data_file_len, PROT_READ | PROT_WRITE, MAP_SHARED, data_fd, 0)) == MAP_FAILED) { perror("mmap"); exit(EXIT_FAILURE); } // close the pmem file -- it will remain mapped close(data_fd); }
StorageManager::StorageManager() : data_file_address(nullptr), is_pmem(false), data_file_len(0), data_file_offset(0) { // Check if we need a data pool if (IsBasedOnWriteAheadLogging(peloton_logging_mode) == true || peloton_logging_mode == LOGGING_TYPE_INVALID) { return; } int data_fd; std::string data_file_name; struct stat data_stat; // Initialize file size if (peloton_data_file_size != 0) data_file_len = peloton_data_file_size * 1024 * 1024; // MB else data_file_len = DATA_FILE_LEN; // Check for relevant file system bool found_file_system = false; switch (peloton_logging_mode) { // Check for NVM FS for data case LOGGING_TYPE_NVM_NVM: case LOGGING_TYPE_NVM_HDD: case LOGGING_TYPE_NVM_SSD: { int status = stat(NVM_DIR, &data_stat); if (status == 0 && S_ISDIR(data_stat.st_mode)) { data_file_name = std::string(NVM_DIR) + std::string(DATA_FILE_NAME); found_file_system = true; } } break; // Check for SSD FS case LOGGING_TYPE_SSD_NVM: case LOGGING_TYPE_SSD_SSD: case LOGGING_TYPE_SSD_HDD: { int status = stat(SSD_DIR, &data_stat); if (status == 0 && S_ISDIR(data_stat.st_mode)) { data_file_name = std::string(SSD_DIR) + std::string(DATA_FILE_NAME); found_file_system = true; } } break; // Check for HDD FS case LOGGING_TYPE_HDD_NVM: case LOGGING_TYPE_HDD_SSD: case LOGGING_TYPE_HDD_HDD: { int status = stat(HDD_DIR, &data_stat); if (status == 0 && S_ISDIR(data_stat.st_mode)) { data_file_name = std::string(HDD_DIR) + std::string(DATA_FILE_NAME); found_file_system = true; } } break; default: break; } // Fallback to tmp directory if needed if (found_file_system == false) { int status = stat(TMP_DIR, &data_stat); if (status == 0 && S_ISDIR(data_stat.st_mode)) { data_file_name = std::string(TMP_DIR) + std::string(DATA_FILE_NAME); } else { throw Exception("Could not find temp directory : " + std::string(TMP_DIR)); } } LOG_INFO("DATA DIR :: %s ", data_file_name.c_str()); // TODO: std::cout << "Data path :: " << data_file_name << "\n"; // Create a data file if ((data_fd = open(data_file_name.c_str(), O_CREAT | O_RDWR, 0666)) < 0) { perror(data_file_name.c_str()); exit(EXIT_FAILURE); } // Allocate the data file if ((errno = posix_fallocate(data_fd, 0, data_file_len)) != 0) { perror("posix_fallocate"); exit(EXIT_FAILURE); } // memory map the data file if ((data_file_address = reinterpret_cast<char *>(pmem_map(data_fd))) == NULL) { perror("pmem_map"); exit(EXIT_FAILURE); } // true only if the entire range [addr, addr+len) consists of persistent // memory is_pmem = pmem_is_pmem(data_file_address, data_file_len); // close the pmem file -- it will remain mapped close(data_fd); }