/** * @brief Recovery system based on log file */ void WriteBehindFrontendLogger::DoRecovery() { // Set log file size log_file_size = GetLogFileSize(log_file_fd); // Go over the log size if needed if (log_file_size > 0) { bool reached_end_of_file = false; // check whether first item is LOGRECORD_TYPE_TRANSACTION_COMMIT // if not, no need to do recovery. // if yes, need to replay all log records before we hit // LOGRECORD_TYPE_TRANSACTION_DONE bool need_recovery = NeedRecovery(); if (need_recovery == true) { TransactionRecord dummy_transaction_record(LOGRECORD_TYPE_INVALID); cid_t current_commit_id = INVALID_CID; // Go over each log record in the log file while (reached_end_of_file == false) { // Read the first byte to identify log record type // If that is not possible, then wrap up recovery LogRecordType log_type = GetNextLogRecordType(log_file, log_file_size); switch (log_type) { case LOGRECORD_TYPE_TRANSACTION_DONE: case LOGRECORD_TYPE_TRANSACTION_COMMIT: { // read but do nothing ReadTransactionRecordHeader(dummy_transaction_record, log_file, log_file_size); } break; case LOGRECORD_TYPE_WBL_TUPLE_INSERT: { TupleRecord insert_record(LOGRECORD_TYPE_WBL_TUPLE_INSERT); ReadTupleRecordHeader(insert_record, log_file, log_file_size); auto insert_location = insert_record.GetInsertLocation(); auto info = SetInsertCommitMark(insert_location); current_commit_id = info.first; } break; case LOGRECORD_TYPE_WBL_TUPLE_DELETE: { TupleRecord delete_record(LOGRECORD_TYPE_WBL_TUPLE_DELETE); ReadTupleRecordHeader(delete_record, log_file, log_file_size); auto delete_location = delete_record.GetDeleteLocation(); auto info = SetDeleteCommitMark(delete_location); current_commit_id = info.first; } break; case LOGRECORD_TYPE_WBL_TUPLE_UPDATE: { TupleRecord update_record(LOGRECORD_TYPE_WBL_TUPLE_UPDATE); ReadTupleRecordHeader(update_record, log_file, log_file_size); auto delete_location = update_record.GetDeleteLocation(); SetDeleteCommitMark(delete_location); auto insert_location = update_record.GetInsertLocation(); auto info = SetInsertCommitMark(insert_location); current_commit_id = info.first; } break; default: reached_end_of_file = true; break; } } // Update latest commit id if (latest_commit_id < current_commit_id) { latest_commit_id = current_commit_id; } // write out a trasaction done log record to file // to avoid redo next time during recovery WriteTransactionLogRecord( TransactionRecord(LOGRECORD_TYPE_TRANSACTION_DONE)); } // After finishing recovery, set the next oid with maximum oid // observed during the recovery auto &manager = catalog::Manager::GetInstance(); manager.SetNextOid(max_oid); } }
/** * @brief flush all log records to the file */ void WriteBehindFrontendLogger::FlushLogRecords(void) { std::vector<txn_id_t> committed_txn_list; std::vector<txn_id_t> not_committed_txn_list; std::set<oid_t> modified_tile_group_set; //===--------------------------------------------------------------------===// // Collect the log records //===--------------------------------------------------------------------===// for (auto record : global_queue) { switch (record->GetType()) { case LOGRECORD_TYPE_TRANSACTION_BEGIN: global_peloton_log_record_pool.CreateTransactionLogList( record->GetTransactionId()); break; case LOGRECORD_TYPE_TRANSACTION_COMMIT: committed_txn_list.push_back(record->GetTransactionId()); break; case LOGRECORD_TYPE_TRANSACTION_ABORT: // Nothing to be done for abort break; case LOGRECORD_TYPE_TRANSACTION_END: case LOGRECORD_TYPE_TRANSACTION_DONE: // if a txn is not committed (aborted or active), log records will be // removed here // Note that list is not be removed immediately, it is removed only // after flush and commit. not_committed_txn_list.push_back(record->GetTransactionId()); break; case LOGRECORD_TYPE_WBL_TUPLE_INSERT: case LOGRECORD_TYPE_WBL_TUPLE_DELETE: case LOGRECORD_TYPE_WBL_TUPLE_UPDATE: { // Check the commit information, auto status = CollectTupleRecord(reinterpret_cast<TupleRecord *>(record)); // Delete record if we did not collect it if (status.first == false) { delete record; } // Else, add it to the set of modified tile groups else { auto location = status.second.block; if (location != INVALID_OID) modified_tile_group_set.insert(location); } } break; case LOGRECORD_TYPE_INVALID: default: throw Exception("Invalid or unrecogized log record found"); break; } } // Clear the global queue global_queue.clear(); //===--------------------------------------------------------------------===// // Write out the log records //===--------------------------------------------------------------------===// // If committed txn list is not empty if (committed_txn_list.empty() == false) { //===--------------------------------------------------------------------===// // SYNC 1: Sync the TGs //===--------------------------------------------------------------------===// SyncTileGroups(modified_tile_group_set); //===--------------------------------------------------------------------===// // SYNC 2: Sync the log for TXN COMMIT record //===--------------------------------------------------------------------===// // Write out all the committed log records size_t written_log_record_count = WriteLogRecords(committed_txn_list); // Now, write a committing log entry to file // Piggyback the number of written log records as a "txn_id" in this record WriteTransactionLogRecord(TransactionRecord( LOGRECORD_TYPE_TRANSACTION_COMMIT, written_log_record_count)); //===--------------------------------------------------------------------===// // SYNC 3: Sync the changes to TG headers //===--------------------------------------------------------------------===// // Toggle the commit marks auto tile_group_header_set = ToggleCommitMarks(committed_txn_list); // Sync the TG headers SyncTileGroupHeaders(tile_group_header_set); //===--------------------------------------------------------------------===// // SYNC 4 : Sync the log for TXN DONE record //===--------------------------------------------------------------------===// // Write out a transaction done log record to file WriteTransactionLogRecord( TransactionRecord(LOGRECORD_TYPE_TRANSACTION_DONE)); } //===--------------------------------------------------------------------===// // Clean up finished transaction log lists //===--------------------------------------------------------------------===// // remove any finished txn logs for (txn_id_t txn_id : not_committed_txn_list) { global_peloton_log_record_pool.RemoveTransactionLogList(txn_id); } // Notify the backend loggers { std::lock_guard<std::mutex> lock(backend_logger_mutex); for (auto backend_logger : backend_loggers) { backend_logger->Commit(); } } }
/** * @brief flush all log records to the file */ void WriteBehindFrontendLogger::FlushLogRecords(void) { std::vector<txn_id_t> committed_txn_list; std::vector<txn_id_t> not_committed_txn_list; std::set<oid_t> modified_tile_group_set; //===--------------------------------------------------------------------===// // Collect the log records //===--------------------------------------------------------------------===// size_t global_queue_size = global_queue.size(); for (oid_t global_queue_itr = 0; global_queue_itr < global_queue_size; global_queue_itr++) { if (global_queue[global_queue_itr] == nullptr) { continue; } // FIXME change the interface of write behind logging // switch (global_queue[global_queue_itr]->GetType()) { // case LOGRECORD_TYPE_TRANSACTION_BEGIN: // global_peloton_log_record_pool.CreateTxnLogList( // global_queue[global_queue_itr]->GetTransactionId()); // break; // // case LOGRECORD_TYPE_TRANSACTION_COMMIT: // committed_txn_list.push_back( // global_queue[global_queue_itr]->GetTransactionId()); // break; // // case LOGRECORD_TYPE_TRANSACTION_ABORT: // // Nothing to be done for abort // break; // // case LOGRECORD_TYPE_TRANSACTION_END: // case LOGRECORD_TYPE_TRANSACTION_DONE: // // if a txn is not committed (aborted or active), log records will // be // // removed here // // Note that list is not be removed immediately, it is removed // only // // after flush and commit. // not_committed_txn_list.push_back( // global_queue[global_queue_itr]->GetTransactionId()); // break; // // case LOGRECORD_TYPE_WBL_TUPLE_INSERT: // case LOGRECORD_TYPE_WBL_TUPLE_DELETE: // case LOGRECORD_TYPE_WBL_TUPLE_UPDATE: { // // LogRecord* log_record = global_queue[global_queue_itr].release(); // TupleRecord* tuple_record = // reinterpret_cast<TupleRecord*>(log_record); // // // Check the commit information // auto status = // CollectTupleRecord(std::unique_ptr<TupleRecord>(tuple_record)); // // // Add it to the set of modified tile groups // if (status.first == true) { // auto location = status.second.block; // if (location != INVALID_OID) { // modified_tile_group_set.insert(location); // } // } // // } break; // // case LOGRECORD_TYPE_INVALID: // default: // throw Exception("Invalid or unrecogized log record found"); // break; // } } // Clean up the frontend logger's queue global_queue.clear(); //===--------------------------------------------------------------------===// // Write out the log records //===--------------------------------------------------------------------===// // If committed txn list is not empty if (committed_txn_list.empty() == false) { //===--------------------------------------------------------------------===// // SYNC 1: Sync the TGs //===--------------------------------------------------------------------===// SyncTileGroups(modified_tile_group_set); //===--------------------------------------------------------------------===// // SYNC 2: Sync the log for TXN COMMIT record //===--------------------------------------------------------------------===// // Write out all the committed log records size_t written_log_record_count = WriteLogRecords(committed_txn_list); // Now, write a committing log entry to file // Piggyback the number of written log records as a "txn_id" in this record WriteTransactionLogRecord(TransactionRecord( LOGRECORD_TYPE_TRANSACTION_COMMIT, written_log_record_count)); //===--------------------------------------------------------------------===// // SYNC 3: Sync the changes to TG headers //===--------------------------------------------------------------------===// // Toggle the commit marks auto tile_group_header_set = ToggleCommitMarks(committed_txn_list); // Sync the TG headers SyncTileGroupHeaders(tile_group_header_set); //===--------------------------------------------------------------------===// // SYNC 4 : Sync the log for TXN DONE record //===--------------------------------------------------------------------===// // Write out a transaction done log record to file WriteTransactionLogRecord( TransactionRecord(LOGRECORD_TYPE_TRANSACTION_DONE)); } //===--------------------------------------------------------------------===// // Clean up finished transaction log lists //===--------------------------------------------------------------------===// // remove any finished txn logs for (txn_id_t txn_id : not_committed_txn_list) { global_peloton_log_record_pool.RemoveTxnLogRecordList(txn_id); } // Notify the backend loggers // { // for (auto backend_logger : backend_loggers) { // // FIXME // assert(backend_logger); // // backend_logger->FinishedFlushing(); // } // } }