/**
* @brief Recovery system based on log file
*/
void WriteBehindFrontendLogger::DoRecovery() {
// Set log file size
log_file_size = GetLogFileSize(log_file_fd);
// Go over the log size if needed
if (log_file_size > 0) {
bool reached_end_of_file = false;
// check whether first item is LOGRECORD_TYPE_TRANSACTION_COMMIT
// if not, no need to do recovery.
// if yes, need to replay all log records before we hit
// LOGRECORD_TYPE_TRANSACTION_DONE
bool need_recovery = NeedRecovery();
if (need_recovery == true) {
TransactionRecord dummy_transaction_record(LOGRECORD_TYPE_INVALID);
cid_t current_commit_id = INVALID_CID;
// Go over each log record in the log file
while (reached_end_of_file == false) {
// Read the first byte to identify log record type
// If that is not possible, then wrap up recovery
LogRecordType log_type = GetNextLogRecordType(log_file, log_file_size);
switch (log_type) {
case LOGRECORD_TYPE_TRANSACTION_DONE:
case LOGRECORD_TYPE_TRANSACTION_COMMIT: {
// read but do nothing
ReadTransactionRecordHeader(dummy_transaction_record, log_file,
log_file_size);
} break;
case LOGRECORD_TYPE_WBL_TUPLE_INSERT: {
TupleRecord insert_record(LOGRECORD_TYPE_WBL_TUPLE_INSERT);
ReadTupleRecordHeader(insert_record, log_file, log_file_size);
auto insert_location = insert_record.GetInsertLocation();
auto info = SetInsertCommitMark(insert_location);
current_commit_id = info.first;
} break;
case LOGRECORD_TYPE_WBL_TUPLE_DELETE: {
TupleRecord delete_record(LOGRECORD_TYPE_WBL_TUPLE_DELETE);
ReadTupleRecordHeader(delete_record, log_file, log_file_size);
auto delete_location = delete_record.GetDeleteLocation();
auto info = SetDeleteCommitMark(delete_location);
current_commit_id = info.first;
} break;
case LOGRECORD_TYPE_WBL_TUPLE_UPDATE: {
TupleRecord update_record(LOGRECORD_TYPE_WBL_TUPLE_UPDATE);
ReadTupleRecordHeader(update_record, log_file, log_file_size);
auto delete_location = update_record.GetDeleteLocation();
SetDeleteCommitMark(delete_location);
auto insert_location = update_record.GetInsertLocation();
auto info = SetInsertCommitMark(insert_location);
current_commit_id = info.first;
} break;
default:
reached_end_of_file = true;
break;
}
}
// Update latest commit id
if (latest_commit_id < current_commit_id) {
latest_commit_id = current_commit_id;
}
// write out a trasaction done log record to file
// to avoid redo next time during recovery
WriteTransactionLogRecord(
TransactionRecord(LOGRECORD_TYPE_TRANSACTION_DONE));
}
// After finishing recovery, set the next oid with maximum oid
// observed during the recovery
auto &manager = catalog::Manager::GetInstance();
manager.SetNextOid(max_oid);
}
}
/**
* @brief flush all log records to the file
*/
void WriteBehindFrontendLogger::FlushLogRecords(void) {
std::vector<txn_id_t> committed_txn_list;
std::vector<txn_id_t> not_committed_txn_list;
std::set<oid_t> modified_tile_group_set;
//===--------------------------------------------------------------------===//
// Collect the log records
//===--------------------------------------------------------------------===//
for (auto record : global_queue) {
switch (record->GetType()) {
case LOGRECORD_TYPE_TRANSACTION_BEGIN:
global_peloton_log_record_pool.CreateTransactionLogList(
record->GetTransactionId());
break;
case LOGRECORD_TYPE_TRANSACTION_COMMIT:
committed_txn_list.push_back(record->GetTransactionId());
break;
case LOGRECORD_TYPE_TRANSACTION_ABORT:
// Nothing to be done for abort
break;
case LOGRECORD_TYPE_TRANSACTION_END:
case LOGRECORD_TYPE_TRANSACTION_DONE:
// if a txn is not committed (aborted or active), log records will be
// removed here
// Note that list is not be removed immediately, it is removed only
// after flush and commit.
not_committed_txn_list.push_back(record->GetTransactionId());
break;
case LOGRECORD_TYPE_WBL_TUPLE_INSERT:
case LOGRECORD_TYPE_WBL_TUPLE_DELETE:
case LOGRECORD_TYPE_WBL_TUPLE_UPDATE: {
// Check the commit information,
auto status =
CollectTupleRecord(reinterpret_cast<TupleRecord *>(record));
// Delete record if we did not collect it
if (status.first == false) {
delete record;
}
// Else, add it to the set of modified tile groups
else {
auto location = status.second.block;
if (location != INVALID_OID) modified_tile_group_set.insert(location);
}
} break;
case LOGRECORD_TYPE_INVALID:
default:
throw Exception("Invalid or unrecogized log record found");
break;
}
}
// Clear the global queue
global_queue.clear();
//===--------------------------------------------------------------------===//
// Write out the log records
//===--------------------------------------------------------------------===//
// If committed txn list is not empty
if (committed_txn_list.empty() == false) {
//===--------------------------------------------------------------------===//
// SYNC 1: Sync the TGs
//===--------------------------------------------------------------------===//
SyncTileGroups(modified_tile_group_set);
//===--------------------------------------------------------------------===//
// SYNC 2: Sync the log for TXN COMMIT record
//===--------------------------------------------------------------------===//
// Write out all the committed log records
size_t written_log_record_count = WriteLogRecords(committed_txn_list);
// Now, write a committing log entry to file
// Piggyback the number of written log records as a "txn_id" in this record
WriteTransactionLogRecord(TransactionRecord(
LOGRECORD_TYPE_TRANSACTION_COMMIT, written_log_record_count));
//===--------------------------------------------------------------------===//
// SYNC 3: Sync the changes to TG headers
//===--------------------------------------------------------------------===//
// Toggle the commit marks
auto tile_group_header_set = ToggleCommitMarks(committed_txn_list);
// Sync the TG headers
SyncTileGroupHeaders(tile_group_header_set);
//===--------------------------------------------------------------------===//
// SYNC 4 : Sync the log for TXN DONE record
//===--------------------------------------------------------------------===//
// Write out a transaction done log record to file
WriteTransactionLogRecord(
TransactionRecord(LOGRECORD_TYPE_TRANSACTION_DONE));
}
//===--------------------------------------------------------------------===//
// Clean up finished transaction log lists
//===--------------------------------------------------------------------===//
// remove any finished txn logs
for (txn_id_t txn_id : not_committed_txn_list) {
global_peloton_log_record_pool.RemoveTransactionLogList(txn_id);
}
// Notify the backend loggers
{
std::lock_guard<std::mutex> lock(backend_logger_mutex);
for (auto backend_logger : backend_loggers) {
backend_logger->Commit();
}
}
}
/**
* @brief flush all log records to the file
*/
void WriteBehindFrontendLogger::FlushLogRecords(void) {
std::vector<txn_id_t> committed_txn_list;
std::vector<txn_id_t> not_committed_txn_list;
std::set<oid_t> modified_tile_group_set;
//===--------------------------------------------------------------------===//
// Collect the log records
//===--------------------------------------------------------------------===//
size_t global_queue_size = global_queue.size();
for (oid_t global_queue_itr = 0; global_queue_itr < global_queue_size;
global_queue_itr++) {
if (global_queue[global_queue_itr] == nullptr) {
continue;
}
// FIXME change the interface of write behind logging
// switch (global_queue[global_queue_itr]->GetType()) {
// case LOGRECORD_TYPE_TRANSACTION_BEGIN:
// global_peloton_log_record_pool.CreateTxnLogList(
// global_queue[global_queue_itr]->GetTransactionId());
// break;
//
// case LOGRECORD_TYPE_TRANSACTION_COMMIT:
// committed_txn_list.push_back(
// global_queue[global_queue_itr]->GetTransactionId());
// break;
//
// case LOGRECORD_TYPE_TRANSACTION_ABORT:
// // Nothing to be done for abort
// break;
//
// case LOGRECORD_TYPE_TRANSACTION_END:
// case LOGRECORD_TYPE_TRANSACTION_DONE:
// // if a txn is not committed (aborted or active), log records will
// be
// // removed here
// // Note that list is not be removed immediately, it is removed
// only
// // after flush and commit.
// not_committed_txn_list.push_back(
// global_queue[global_queue_itr]->GetTransactionId());
// break;
//
// case LOGRECORD_TYPE_WBL_TUPLE_INSERT:
// case LOGRECORD_TYPE_WBL_TUPLE_DELETE:
// case LOGRECORD_TYPE_WBL_TUPLE_UPDATE: {
//
// LogRecord* log_record = global_queue[global_queue_itr].release();
// TupleRecord* tuple_record =
// reinterpret_cast<TupleRecord*>(log_record);
//
// // Check the commit information
// auto status =
// CollectTupleRecord(std::unique_ptr<TupleRecord>(tuple_record));
//
// // Add it to the set of modified tile groups
// if (status.first == true) {
// auto location = status.second.block;
// if (location != INVALID_OID) {
// modified_tile_group_set.insert(location);
// }
// }
//
// } break;
//
// case LOGRECORD_TYPE_INVALID:
// default:
// throw Exception("Invalid or unrecogized log record found");
// break;
// }
}
// Clean up the frontend logger's queue
global_queue.clear();
//===--------------------------------------------------------------------===//
// Write out the log records
//===--------------------------------------------------------------------===//
// If committed txn list is not empty
if (committed_txn_list.empty() == false) {
//===--------------------------------------------------------------------===//
// SYNC 1: Sync the TGs
//===--------------------------------------------------------------------===//
SyncTileGroups(modified_tile_group_set);
//===--------------------------------------------------------------------===//
// SYNC 2: Sync the log for TXN COMMIT record
//===--------------------------------------------------------------------===//
// Write out all the committed log records
size_t written_log_record_count = WriteLogRecords(committed_txn_list);
// Now, write a committing log entry to file
// Piggyback the number of written log records as a "txn_id" in this record
WriteTransactionLogRecord(TransactionRecord(
LOGRECORD_TYPE_TRANSACTION_COMMIT, written_log_record_count));
//===--------------------------------------------------------------------===//
// SYNC 3: Sync the changes to TG headers
//===--------------------------------------------------------------------===//
// Toggle the commit marks
auto tile_group_header_set = ToggleCommitMarks(committed_txn_list);
// Sync the TG headers
SyncTileGroupHeaders(tile_group_header_set);
//===--------------------------------------------------------------------===//
// SYNC 4 : Sync the log for TXN DONE record
//===--------------------------------------------------------------------===//
// Write out a transaction done log record to file
WriteTransactionLogRecord(
TransactionRecord(LOGRECORD_TYPE_TRANSACTION_DONE));
}
//===--------------------------------------------------------------------===//
// Clean up finished transaction log lists
//===--------------------------------------------------------------------===//
// remove any finished txn logs
for (txn_id_t txn_id : not_committed_txn_list) {
global_peloton_log_record_pool.RemoveTxnLogRecordList(txn_id);
}
// Notify the backend loggers
// {
// for (auto backend_logger : backend_loggers) {
// // FIXME
// assert(backend_logger);
// // backend_logger->FinishedFlushing();
// }
// }
}
