/*
* __curlog_reset --
* WT_CURSOR.reset method for the log cursor type.
*/
static int
__curlog_reset(WT_CURSOR *cursor)
{
WT_CURSOR_LOG *cl;
cl = (WT_CURSOR_LOG *)cursor;
cl->stepp = cl->stepp_end = NULL;
cl->step_count = 0;
WT_INIT_LSN(cl->cur_lsn);
WT_INIT_LSN(cl->next_lsn);
return (0);
}
/*
* __curlog_reset --
* WT_CURSOR.reset method for the log cursor type.
*/
static int
__curlog_reset(WT_CURSOR *cursor)
{
WT_CURSOR_LOG *cl;
WT_DECL_RET;
WT_SESSION_IMPL *session;
CURSOR_API_CALL_PREPARE_ALLOWED(cursor, session, reset, NULL);
cl = (WT_CURSOR_LOG *)cursor;
cl->stepp = cl->stepp_end = NULL;
cl->step_count = 0;
WT_INIT_LSN(cl->cur_lsn);
WT_INIT_LSN(cl->next_lsn);
err: API_END_RET(session, ret);
}
/*
* __log_wrlsn_server --
* The log wrlsn server thread.
*/
static WT_THREAD_RET
__log_wrlsn_server(void *arg)
{
WT_CONNECTION_IMPL *conn;
WT_DECL_RET;
WT_LOG *log;
WT_LSN prev;
WT_SESSION_IMPL *session;
int yield;
bool did_work;
session = arg;
conn = S2C(session);
log = conn->log;
yield = 0;
WT_INIT_LSN(&prev);
did_work = false;
while (F_ISSET(conn, WT_CONN_LOG_SERVER_RUN)) {
/*
* Write out any log record buffers if anything was done
* since last time. Only call the function to walk the
* slots if the system is not idle. On an idle system
* the alloc_lsn will not advance and the written lsn will
* match the alloc_lsn.
*/
if (__wt_log_cmp(&prev, &log->alloc_lsn) != 0 ||
__wt_log_cmp(&log->write_lsn, &log->alloc_lsn) != 0)
WT_ERR(__wt_log_wrlsn(session, &yield));
else
WT_STAT_FAST_CONN_INCR(session, log_write_lsn_skip);
prev = log->alloc_lsn;
if (yield == 0)
did_work = true;
else
did_work = false;
/*
* If __wt_log_wrlsn did work we want to yield instead of sleep.
*/
if (yield++ < WT_THOUSAND)
__wt_yield();
else
/*
* Send in false because if we did any work we would
* not be on this path.
*/
WT_ERR(__wt_cond_auto_wait(
session, conn->log_wrlsn_cond, did_work));
}
/*
* On close we need to do this one more time because there could
* be straggling log writes that need to be written.
*/
WT_ERR(__wt_log_force_write(session, 1, NULL));
WT_ERR(__wt_log_wrlsn(session, NULL));
if (0) {
err: __wt_err(session, ret, "log wrlsn server error");
}
return (WT_THREAD_RET_VALUE);
}
/*
* __wt_logmgr_create --
* Initialize the log subsystem (before running recovery).
*/
int
__wt_logmgr_create(WT_SESSION_IMPL *session, const char *cfg[])
{
WT_CONNECTION_IMPL *conn;
WT_LOG *log;
bool run;
conn = S2C(session);
/* Handle configuration. */
WT_RET(__logmgr_config(session, cfg, &run, false));
/* If logging is not configured, we're done. */
if (!run)
return (0);
FLD_SET(conn->log_flags, WT_CONN_LOG_ENABLED);
/*
* Logging is on, allocate the WT_LOG structure and open the log file.
*/
WT_RET(__wt_calloc_one(session, &conn->log));
log = conn->log;
WT_RET(__wt_spin_init(session, &log->log_lock, "log"));
WT_RET(__wt_spin_init(session, &log->log_slot_lock, "log slot"));
WT_RET(__wt_spin_init(session, &log->log_sync_lock, "log sync"));
WT_RET(__wt_spin_init(session, &log->log_writelsn_lock,
"log write LSN"));
WT_RET(__wt_rwlock_alloc(session,
&log->log_archive_lock, "log archive lock"));
if (FLD_ISSET(conn->direct_io, WT_FILE_TYPE_LOG))
log->allocsize =
WT_MAX((uint32_t)conn->buffer_alignment, WT_LOG_ALIGN);
else
log->allocsize = WT_LOG_ALIGN;
WT_INIT_LSN(&log->alloc_lsn);
WT_INIT_LSN(&log->ckpt_lsn);
WT_INIT_LSN(&log->first_lsn);
WT_INIT_LSN(&log->sync_lsn);
/*
* We only use file numbers for directory sync, so this needs to
* initialized to zero.
*/
WT_ZERO_LSN(&log->sync_dir_lsn);
WT_INIT_LSN(&log->trunc_lsn);
WT_INIT_LSN(&log->write_lsn);
WT_INIT_LSN(&log->write_start_lsn);
log->fileid = 0;
WT_RET(__wt_cond_alloc(
session, "log sync", false, &log->log_sync_cond));
WT_RET(__wt_cond_alloc(
session, "log write", false, &log->log_write_cond));
WT_RET(__wt_log_open(session));
WT_RET(__wt_log_slot_init(session));
return (0);
}
/*
* __recovery_setup_file --
* Set up the recovery slot for a file.
*/
static int
__recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config)
{
WT_CONFIG_ITEM cval;
WT_LSN lsn;
uint32_t fileid, lsnfile, lsnoffset;
WT_RET(__wt_config_getones(r->session, config, "id", &cval));
fileid = (uint32_t)cval.val;
/* Track the largest file ID we have seen. */
if (fileid > r->max_fileid)
r->max_fileid = fileid;
if (r->nfiles <= fileid) {
WT_RET(__wt_realloc_def(
r->session, &r->file_alloc, fileid + 1, &r->files));
r->nfiles = fileid + 1;
}
WT_RET(__wt_strdup(r->session, uri, &r->files[fileid].uri));
WT_RET(
__wt_config_getones(r->session, config, "checkpoint_lsn", &cval));
/* If there is checkpoint logged for the file, apply everything. */
if (cval.type != WT_CONFIG_ITEM_STRUCT)
WT_INIT_LSN(&lsn);
/* NOLINTNEXTLINE(cert-err34-c) */
else if (sscanf(cval.str,
"(%" SCNu32 ",%" SCNu32 ")", &lsnfile, &lsnoffset) == 2)
WT_SET_LSN(&lsn, lsnfile, lsnoffset);
else
WT_RET_MSG(r->session, EINVAL,
"Failed to parse checkpoint LSN '%.*s'",
(int)cval.len, cval.str);
r->files[fileid].ckpt_lsn = lsn;
__wt_verbose(r->session, WT_VERB_RECOVERY,
"Recovering %s with id %" PRIu32 " @ (%" PRIu32 ", %" PRIu32 ")",
uri, fileid, lsn.l.file, lsn.l.offset);
if ((!WT_IS_MAX_LSN(&lsn) && !WT_IS_INIT_LSN(&lsn)) &&
(WT_IS_MAX_LSN(&r->max_ckpt_lsn) ||
__wt_log_cmp(&lsn, &r->max_ckpt_lsn) > 0))
r->max_ckpt_lsn = lsn;
return (0);
}
/*
* __recovery_setup_file --
* Set up the recovery slot for a file.
*/
static int
__recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config)
{
WT_CONFIG_ITEM cval;
WT_LSN lsn;
uint32_t fileid;
WT_RET(__wt_config_getones(r->session, config, "id", &cval));
fileid = (uint32_t)cval.val;
/* Track the largest file ID we have seen. */
if (fileid > r->max_fileid)
r->max_fileid = fileid;
if (r->nfiles <= fileid) {
WT_RET(__wt_realloc_def(
r->session, &r->file_alloc, fileid + 1, &r->files));
r->nfiles = fileid + 1;
}
WT_RET(__wt_strdup(r->session, uri, &r->files[fileid].uri));
WT_RET(
__wt_config_getones(r->session, config, "checkpoint_lsn", &cval));
/* If there is checkpoint logged for the file, apply everything. */
if (cval.type != WT_CONFIG_ITEM_STRUCT)
WT_INIT_LSN(&lsn);
else if (sscanf(cval.str, "(%" PRIu32 ",%" PRIdMAX ")",
&lsn.file, (intmax_t*)&lsn.offset) != 2)
WT_RET_MSG(r->session, EINVAL,
"Failed to parse checkpoint LSN '%.*s'",
(int)cval.len, cval.str);
r->files[fileid].ckpt_lsn = lsn;
WT_RET(__wt_verbose(r->session, WT_VERB_RECOVERY,
"Recovering %s with id %u @ (%" PRIu32 ", %" PRIu64 ")",
uri, fileid, lsn.file, lsn.offset));
return (0);
}
/*
* __wt_log_wrlsn --
* Process written log slots and attempt to coalesce them if the LSNs
* are contiguous. The purpose of this function is to advance the
* write_lsn in LSN order after the buffer is written to the log file.
*/
int
__wt_log_wrlsn(WT_SESSION_IMPL *session)
{
WT_CONNECTION_IMPL *conn;
WT_DECL_RET;
WT_LOG *log;
WT_LOG_WRLSN_ENTRY written[WT_SLOT_POOL];
WT_LOGSLOT *coalescing, *slot;
WT_LSN save_lsn;
size_t written_i;
uint32_t i, save_i;
conn = S2C(session);
log = conn->log;
__wt_spin_lock(session, &log->log_writelsn_lock);
restart:
coalescing = NULL;
WT_INIT_LSN(&save_lsn);
written_i = 0;
i = 0;
/*
* Walk the array once saving any slots that are in the
* WT_LOG_SLOT_WRITTEN state.
*/
while (i < WT_SLOT_POOL) {
save_i = i;
slot = &log->slot_pool[i++];
/*
* XXX - During debugging I saw slot 0 become orphaned.
* I believe it is fixed, but check for now.
* This assertion should catch that.
*/
if (slot->slot_state == 0)
WT_ASSERT(session,
slot->slot_release_lsn.file >= log->write_lsn.file);
if (slot->slot_state != WT_LOG_SLOT_WRITTEN)
continue;
written[written_i].slot_index = save_i;
written[written_i++].lsn = slot->slot_release_lsn;
}
/*
* If we found any written slots process them. We sort them
* based on the release LSN, and then look for them in order.
*/
if (written_i > 0) {
WT_INSERTION_SORT(written, written_i,
WT_LOG_WRLSN_ENTRY, WT_WRLSN_ENTRY_CMP_LT);
/*
* We know the written array is sorted by LSN. Go
* through them either advancing write_lsn or coalesce
* contiguous ranges of written slots.
*/
for (i = 0; i < written_i; i++) {
slot = &log->slot_pool[written[i].slot_index];
/*
* The log server thread pushes out slots periodically.
* Sometimes they are empty slots. If we find an
* empty slot, where empty means the start and end LSN
* are the same, free it and continue.
*/
if (__wt_log_cmp(&slot->slot_start_lsn,
&slot->slot_release_lsn) == 0 &&
__wt_log_cmp(&slot->slot_start_lsn,
&slot->slot_end_lsn) == 0) {
__wt_log_slot_free(session, slot);
continue;
}
if (coalescing != NULL) {
/*
* If the write_lsn changed, we may be able to
* process slots. Try again.
*/
if (__wt_log_cmp(
&log->write_lsn, &save_lsn) != 0)
goto restart;
if (__wt_log_cmp(&coalescing->slot_end_lsn,
&written[i].lsn) != 0) {
coalescing = slot;
continue;
}
/*
* If we get here we have a slot to coalesce
* and free.
*/
coalescing->slot_last_offset =
slot->slot_last_offset;
coalescing->slot_end_lsn = slot->slot_end_lsn;
WT_STAT_FAST_CONN_INCR(
session, log_slot_coalesced);
/*
* Copy the flag for later closing.
*/
if (F_ISSET(slot, WT_SLOT_CLOSEFH))
F_SET(coalescing, WT_SLOT_CLOSEFH);
} else {
/*
* If this written slot is not the next LSN,
* try to start coalescing with later slots.
* A synchronous write may update write_lsn
* so save the last one we saw to check when
* coalescing slots.
*/
save_lsn = log->write_lsn;
if (__wt_log_cmp(
&log->write_lsn, &written[i].lsn) != 0) {
coalescing = slot;
continue;
}
/*
* If we get here we have a slot to process.
* Advance the LSN and process the slot.
*/
WT_ASSERT(session, __wt_log_cmp(&written[i].lsn,
&slot->slot_release_lsn) == 0);
if (slot->slot_start_lsn.offset !=
slot->slot_last_offset)
slot->slot_start_lsn.offset =
slot->slot_last_offset;
log->write_start_lsn = slot->slot_start_lsn;
log->write_lsn = slot->slot_end_lsn;
WT_ERR(__wt_cond_signal(
session, log->log_write_cond));
WT_STAT_FAST_CONN_INCR(session, log_write_lsn);
/*
* Signal the close thread if needed.
*/
if (F_ISSET(slot, WT_SLOT_CLOSEFH))
WT_ERR(__wt_cond_signal(
session, conn->log_file_cond));
}
__wt_log_slot_free(session, slot);
}
}
err: __wt_spin_unlock(session, &log->log_writelsn_lock);
return (ret);
}
/*
* __wt_txn_recover --
* Run recovery.
*/
int
__wt_txn_recover(WT_SESSION_IMPL *session)
{
WT_CONNECTION_IMPL *conn;
WT_CURSOR *metac;
WT_DECL_RET;
WT_RECOVERY r;
struct WT_RECOVERY_FILE *metafile;
char *config;
bool eviction_started, needs_rec, was_backup;
conn = S2C(session);
WT_CLEAR(r);
WT_INIT_LSN(&r.ckpt_lsn);
eviction_started = false;
was_backup = F_ISSET(conn, WT_CONN_WAS_BACKUP);
/* We need a real session for recovery. */
WT_RET(__wt_open_internal_session(conn, "txn-recover",
false, WT_SESSION_NO_LOGGING, &session));
r.session = session;
F_SET(conn, WT_CONN_RECOVERING);
WT_ERR(__wt_metadata_search(session, WT_METAFILE_URI, &config));
WT_ERR(__recovery_setup_file(&r, WT_METAFILE_URI, config));
WT_ERR(__wt_metadata_cursor_open(session, NULL, &metac));
metafile = &r.files[WT_METAFILE_ID];
metafile->c = metac;
/*
* If no log was found (including if logging is disabled), or if the
* last checkpoint was done with logging disabled, recovery should not
* run. Scan the metadata to figure out the largest file ID.
*/
if (!FLD_ISSET(S2C(session)->log_flags, WT_CONN_LOG_EXISTED) ||
WT_IS_MAX_LSN(&metafile->ckpt_lsn)) {
WT_ERR(__recovery_file_scan(&r));
conn->next_file_id = r.max_fileid;
goto done;
}
/*
* First, do a pass through the log to recover the metadata, and
* establish the last checkpoint LSN. Skip this when opening a hot
* backup: we already have the correct metadata in that case.
*/
if (!was_backup) {
r.metadata_only = true;
/*
* If this is a read-only connection, check if the checkpoint
* LSN in the metadata file is up to date, indicating a clean
* shutdown.
*/
if (F_ISSET(conn, WT_CONN_READONLY)) {
WT_ERR(__wt_log_needs_recovery(
session, &metafile->ckpt_lsn, &needs_rec));
if (needs_rec)
WT_ERR_MSG(session, WT_RUN_RECOVERY,
"Read-only database needs recovery");
}
if (WT_IS_INIT_LSN(&metafile->ckpt_lsn))
WT_ERR(__wt_log_scan(session,
NULL, WT_LOGSCAN_FIRST, __txn_log_recover, &r));
else {
/*
* Start at the last checkpoint LSN referenced in the
* metadata. If we see the end of a checkpoint while
* scanning, we will change the full scan to start from
* there.
*/
r.ckpt_lsn = metafile->ckpt_lsn;
ret = __wt_log_scan(session,
&metafile->ckpt_lsn, 0, __txn_log_recover, &r);
if (ret == ENOENT)
ret = 0;
WT_ERR(ret);
}
}
/* Scan the metadata to find the live files and their IDs. */
WT_ERR(__recovery_file_scan(&r));
/*
* We no longer need the metadata cursor: close it to avoid pinning any
* resources that could block eviction during recovery.
*/
r.files[0].c = NULL;
WT_ERR(metac->close(metac));
/*
* Now, recover all the files apart from the metadata.
* Pass WT_LOGSCAN_RECOVER so that old logs get truncated.
*/
r.metadata_only = false;
WT_ERR(__wt_verbose(session, WT_VERB_RECOVERY,
"Main recovery loop: starting at %" PRIu32 "/%" PRIu32,
r.ckpt_lsn.l.file, r.ckpt_lsn.l.offset));
WT_ERR(__wt_log_needs_recovery(session, &r.ckpt_lsn, &needs_rec));
/*
* Check if the database was shut down cleanly. If not
* return an error if the user does not want automatic
* recovery.
*/
if (needs_rec &&
(FLD_ISSET(conn->log_flags, WT_CONN_LOG_RECOVER_ERR) ||
F_ISSET(conn, WT_CONN_READONLY))) {
if (F_ISSET(conn, WT_CONN_READONLY))
WT_ERR_MSG(session, WT_RUN_RECOVERY,
"Read-only database needs recovery");
WT_ERR(WT_RUN_RECOVERY);
}
if (F_ISSET(conn, WT_CONN_READONLY))
goto done;
/*
* Recovery can touch more data than fits in cache, so it relies on
* regular eviction to manage paging. Start eviction threads for
* recovery without LAS cursors.
*/
WT_ERR(__wt_evict_create(session));
eviction_started = true;
/*
* Always run recovery even if it was a clean shutdown only if
* this is not a read-only connection.
* We can consider skipping it in the future.
*/
if (WT_IS_INIT_LSN(&r.ckpt_lsn))
WT_ERR(__wt_log_scan(session, NULL,
WT_LOGSCAN_FIRST | WT_LOGSCAN_RECOVER,
__txn_log_recover, &r));
else {
ret = __wt_log_scan(session, &r.ckpt_lsn,
WT_LOGSCAN_RECOVER, __txn_log_recover, &r);
if (ret == ENOENT)
ret = 0;
WT_ERR(ret);
}
conn->next_file_id = r.max_fileid;
/*
* If recovery ran successfully forcibly log a checkpoint so the next
* open is fast and keep the metadata up to date with the checkpoint
* LSN and archiving.
*/
WT_ERR(session->iface.checkpoint(&session->iface, "force=1"));
done: FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_DONE);
err: WT_TRET(__recovery_free(&r));
__wt_free(session, config);
if (ret != 0)
__wt_err(session, ret, "Recovery failed");
/*
* Destroy the eviction threads that were started in support of
* recovery. They will be restarted once the lookaside table is
* created.
*/
if (eviction_started)
WT_TRET(__wt_evict_destroy(session));
WT_TRET(session->iface.close(&session->iface, NULL));
F_CLR(conn, WT_CONN_RECOVERING);
return (ret);
}
/*
* __wt_txn_recover --
* Run recovery.
*/
int
__wt_txn_recover(WT_SESSION_IMPL *session)
{
WT_CONNECTION_IMPL *conn;
WT_CURSOR *metac;
WT_DECL_RET;
WT_RECOVERY r;
struct WT_RECOVERY_FILE *metafile;
char *config;
int was_backup;
conn = S2C(session);
WT_CLEAR(r);
WT_INIT_LSN(&r.ckpt_lsn);
was_backup = F_ISSET(conn, WT_CONN_WAS_BACKUP) ? 1 : 0;
/* We need a real session for recovery. */
WT_RET(__wt_open_session(conn, NULL, NULL, &session));
F_SET(session, WT_SESSION_NO_LOGGING);
r.session = session;
WT_ERR(__wt_metadata_search(session, WT_METAFILE_URI, &config));
WT_ERR(__recovery_setup_file(&r, WT_METAFILE_URI, config));
WT_ERR(__wt_metadata_cursor(session, NULL, &metac));
metafile = &r.files[WT_METAFILE_ID];
metafile->c = metac;
/*
* If no log was found (including if logging is disabled), or if the
* last checkpoint was done with logging disabled, recovery should not
* run. Scan the metadata to figure out the largest file ID.
*/
if (!FLD_ISSET(S2C(session)->log_flags, WT_CONN_LOG_EXISTED) ||
WT_IS_MAX_LSN(&metafile->ckpt_lsn)) {
WT_ERR(__recovery_file_scan(&r));
conn->next_file_id = r.max_fileid;
goto done;
}
/*
* First, do a pass through the log to recover the metadata, and
* establish the last checkpoint LSN. Skip this when opening a hot
* backup: we already have the correct metadata in that case.
*/
if (!was_backup) {
r.metadata_only = 1;
if (WT_IS_INIT_LSN(&metafile->ckpt_lsn))
WT_ERR(__wt_log_scan(session,
NULL, WT_LOGSCAN_FIRST, __txn_log_recover, &r));
else {
/*
* Start at the last checkpoint LSN referenced in the
* metadata. If we see the end of a checkpoint while
* scanning, we will change the full scan to start from
* there.
*/
r.ckpt_lsn = metafile->ckpt_lsn;
WT_ERR(__wt_log_scan(session,
&metafile->ckpt_lsn, 0, __txn_log_recover, &r));
}
}
/* Scan the metadata to find the live files and their IDs. */
WT_ERR(__recovery_file_scan(&r));
/*
* We no longer need the metadata cursor: close it to avoid pinning any
* resources that could block eviction during recovery.
*/
r.files[0].c = NULL;
WT_ERR(metac->close(metac));
/*
* Now, recover all the files apart from the metadata.
* Pass WT_LOGSCAN_RECOVER so that old logs get truncated.
*/
r.metadata_only = 0;
WT_ERR(__wt_verbose(session, WT_VERB_RECOVERY,
"Main recovery loop: starting at %u/%" PRIuMAX,
r.ckpt_lsn.file, (uintmax_t)r.ckpt_lsn.offset));
if (WT_IS_INIT_LSN(&r.ckpt_lsn))
WT_ERR(__wt_log_scan(session, NULL,
WT_LOGSCAN_FIRST | WT_LOGSCAN_RECOVER,
__txn_log_recover, &r));
else
WT_ERR(__wt_log_scan(session, &r.ckpt_lsn,
WT_LOGSCAN_RECOVER,
__txn_log_recover, &r));
conn->next_file_id = r.max_fileid;
/*
* If recovery ran successfully forcibly log a checkpoint so the next
* open is fast and keep the metadata up to date with the checkpoint
* LSN and archiving.
*/
WT_ERR(session->iface.checkpoint(&session->iface, "force=1"));
done:
err: WT_TRET(__recovery_free(&r));
__wt_free(session, config);
WT_TRET(session->iface.close(&session->iface, NULL));
return (ret);
}
/*
* __wt_curlog_open --
* Initialize a log cursor.
*/
int
__wt_curlog_open(WT_SESSION_IMPL *session,
const char *uri, const char *cfg[], WT_CURSOR **cursorp)
{
WT_CONNECTION_IMPL *conn;
WT_CURSOR_STATIC_INIT(iface,
__wt_cursor_get_key, /* get-key */
__wt_cursor_get_value, /* get-value */
__wt_cursor_set_key, /* set-key */
__wt_cursor_set_value, /* set-value */
__curlog_compare, /* compare */
__wt_cursor_equals, /* equals */
__curlog_next, /* next */
__wt_cursor_notsup, /* prev */
__curlog_reset, /* reset */
__curlog_search, /* search */
__wt_cursor_search_near_notsup, /* search-near */
__wt_cursor_notsup, /* insert */
__wt_cursor_notsup, /* update */
__wt_cursor_notsup, /* remove */
__wt_cursor_reconfigure_notsup, /* reconfigure */
__curlog_close); /* close */
WT_CURSOR *cursor;
WT_CURSOR_LOG *cl;
WT_DECL_RET;
WT_LOG *log;
WT_STATIC_ASSERT(offsetof(WT_CURSOR_LOG, iface) == 0);
conn = S2C(session);
if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED))
WT_RET_MSG(session, EINVAL,
"Cannot open a log cursor without logging enabled");
log = conn->log;
cl = NULL;
WT_RET(__wt_calloc_one(session, &cl));
cursor = &cl->iface;
*cursor = iface;
cursor->session = &session->iface;
WT_ERR(__wt_calloc_one(session, &cl->cur_lsn));
WT_ERR(__wt_calloc_one(session, &cl->next_lsn));
WT_ERR(__wt_scr_alloc(session, 0, &cl->logrec));
WT_ERR(__wt_scr_alloc(session, 0, &cl->opkey));
WT_ERR(__wt_scr_alloc(session, 0, &cl->opvalue));
cursor->key_format = WT_LOGC_KEY_FORMAT;
cursor->value_format = WT_LOGC_VALUE_FORMAT;
WT_INIT_LSN(cl->cur_lsn);
WT_INIT_LSN(cl->next_lsn);
WT_ERR(__wt_cursor_init(cursor, uri, NULL, cfg, cursorp));
/*
* The user may be trying to read a log record they just wrote.
* Log records may be buffered, so force out any now.
*/
WT_ERR(__wt_log_force_write(session, 1));
/* Log cursors block archiving. */
WT_ERR(__wt_readlock(session, log->log_archive_lock));
if (0) {
err: if (F_ISSET(cursor, WT_CURSTD_OPEN))
WT_TRET(cursor->close(cursor));
else {
__wt_free(session, cl->cur_lsn);
__wt_free(session, cl->next_lsn);
__wt_scr_free(session, &cl->logrec);
__wt_scr_free(session, &cl->opkey);
__wt_scr_free(session, &cl->opvalue);
/*
* NOTE: We cannot get on the error path with the
* readlock held. No need to unlock it unless that
* changes above.
*/
__wt_free(session, cl);
}
*cursorp = NULL;
}
return (ret);
}
/*
* __wt_txn_checkpoint_log --
* Write a log record for a checkpoint operation.
*/
int
__wt_txn_checkpoint_log(
WT_SESSION_IMPL *session, bool full, uint32_t flags, WT_LSN *lsnp)
{
WT_CONNECTION_IMPL *conn;
WT_DECL_ITEM(logrec);
WT_DECL_RET;
WT_ITEM *ckpt_snapshot, empty;
WT_LSN *ckpt_lsn;
WT_TXN *txn;
WT_TXN_GLOBAL *txn_global;
uint8_t *end, *p;
size_t recsize;
uint32_t i, rectype;
const char *fmt;
conn = S2C(session);
txn_global = &conn->txn_global;
txn = &session->txn;
ckpt_lsn = &txn->ckpt_lsn;
/*
* If this is a file sync, log it unless there is a full checkpoint in
* progress.
*/
if (!full) {
if (txn->full_ckpt) {
if (lsnp != NULL)
*lsnp = *ckpt_lsn;
return (0);
}
return (__txn_log_file_sync(session, flags, lsnp));
}
switch (flags) {
case WT_TXN_LOG_CKPT_PREPARE:
txn->full_ckpt = true;
if (conn->compat_major >= WT_LOG_V2) {
/*
* Write the system log record containing a checkpoint
* start operation.
*/
rectype = WT_LOGREC_SYSTEM;
fmt = WT_UNCHECKED_STRING(I);
WT_ERR(__wt_struct_size(
session, &recsize, fmt, rectype));
WT_ERR(__wt_logrec_alloc(session, recsize, &logrec));
WT_ERR(__wt_struct_pack(session,
(uint8_t *)logrec->data + logrec->size, recsize,
fmt, rectype));
logrec->size += (uint32_t)recsize;
WT_ERR(__wt_logop_checkpoint_start_pack(
session, logrec));
WT_ERR(__wt_log_write(session, logrec, ckpt_lsn, 0));
} else {
WT_ERR(__wt_log_printf(session,
"CHECKPOINT: Starting record"));
WT_ERR(__wt_log_flush_lsn(session, ckpt_lsn, true));
}
/*
* We take and immediately release the visibility lock.
* Acquiring the write lock guarantees that any transaction
* that has written to the log has also made its transaction
* visible at this time.
*/
__wt_writelock(session, &txn_global->visibility_rwlock);
__wt_writeunlock(session, &txn_global->visibility_rwlock);
/*
* We need to make sure that the log records in the checkpoint
* LSN are on disk. In particular to make sure that the
* current log file exists.
*/
WT_ERR(__wt_log_force_sync(session, ckpt_lsn));
break;
case WT_TXN_LOG_CKPT_START:
/* Take a copy of the transaction snapshot. */
txn->ckpt_nsnapshot = txn->snapshot_count;
recsize = (size_t)txn->ckpt_nsnapshot * WT_INTPACK64_MAXSIZE;
WT_ERR(__wt_scr_alloc(session, recsize, &txn->ckpt_snapshot));
p = txn->ckpt_snapshot->mem;
end = p + recsize;
for (i = 0; i < txn->snapshot_count; i++)
WT_ERR(__wt_vpack_uint(
&p, WT_PTRDIFF(end, p), txn->snapshot[i]));
break;
case WT_TXN_LOG_CKPT_STOP:
/*
* During a clean connection close, we get here without the
* prepare or start steps. In that case, log the current LSN
* as the checkpoint LSN.
*/
if (!txn->full_ckpt) {
txn->ckpt_nsnapshot = 0;
WT_CLEAR(empty);
ckpt_snapshot = ∅
WT_ERR(__wt_log_flush_lsn(session, ckpt_lsn, true));
} else
ckpt_snapshot = txn->ckpt_snapshot;
/* Write the checkpoint log record. */
rectype = WT_LOGREC_CHECKPOINT;
fmt = WT_UNCHECKED_STRING(IIIIu);
WT_ERR(__wt_struct_size(session, &recsize,
fmt, rectype, ckpt_lsn->l.file, ckpt_lsn->l.offset,
txn->ckpt_nsnapshot, ckpt_snapshot));
WT_ERR(__wt_logrec_alloc(session, recsize, &logrec));
WT_ERR(__wt_struct_pack(session,
(uint8_t *)logrec->data + logrec->size, recsize,
fmt, rectype, ckpt_lsn->l.file, ckpt_lsn->l.offset,
txn->ckpt_nsnapshot, ckpt_snapshot));
logrec->size += (uint32_t)recsize;
WT_ERR(__wt_log_write(session, logrec, lsnp,
F_ISSET(conn, WT_CONN_CKPT_SYNC) ?
WT_LOG_FSYNC : 0));
/*
* If this full checkpoint completed successfully and there is
* no hot backup in progress and this is not an unclean
* recovery, tell the logging subsystem the checkpoint LSN so
* that it can archive. Do not update the logging checkpoint
* LSN if this is during a clean connection close, only during
* a full checkpoint. A clean close may not update any
* metadata LSN and we do not want to archive in that case.
*/
if (!conn->hot_backup &&
(!FLD_ISSET(conn->log_flags, WT_CONN_LOG_RECOVER_DIRTY) ||
FLD_ISSET(conn->log_flags, WT_CONN_LOG_FORCE_DOWNGRADE)) &&
txn->full_ckpt)
__wt_log_ckpt(session, ckpt_lsn);
/* FALLTHROUGH */
case WT_TXN_LOG_CKPT_CLEANUP:
/* Cleanup any allocated resources */
WT_INIT_LSN(ckpt_lsn);
txn->ckpt_nsnapshot = 0;
__wt_scr_free(session, &txn->ckpt_snapshot);
txn->full_ckpt = false;
break;
WT_ILLEGAL_VALUE_ERR(session);
}
err: __wt_logrec_free(session, &logrec);
return (ret);
}
/*
* __wt_txn_recover --
* Run recovery.
*/
int
__wt_txn_recover(WT_SESSION_IMPL *session)
{
WT_CONNECTION_IMPL *conn;
WT_CURSOR *metac;
WT_DECL_RET;
WT_RECOVERY r;
WT_RECOVERY_FILE *metafile;
char *config;
bool do_checkpoint, eviction_started, needs_rec, was_backup;
conn = S2C(session);
WT_CLEAR(r);
WT_INIT_LSN(&r.ckpt_lsn);
config = NULL;
do_checkpoint = true;
eviction_started = false;
was_backup = F_ISSET(conn, WT_CONN_WAS_BACKUP);
/* We need a real session for recovery. */
WT_RET(__wt_open_internal_session(conn, "txn-recover",
false, WT_SESSION_NO_LOGGING, &session));
r.session = session;
WT_MAX_LSN(&r.max_ckpt_lsn);
WT_MAX_LSN(&r.max_rec_lsn);
conn->txn_global.recovery_timestamp =
conn->txn_global.meta_ckpt_timestamp = 0;
F_SET(conn, WT_CONN_RECOVERING);
WT_ERR(__wt_metadata_search(session, WT_METAFILE_URI, &config));
WT_ERR(__recovery_setup_file(&r, WT_METAFILE_URI, config));
WT_ERR(__wt_metadata_cursor_open(session, NULL, &metac));
metafile = &r.files[WT_METAFILE_ID];
metafile->c = metac;
/*
* If no log was found (including if logging is disabled), or if the
* last checkpoint was done with logging disabled, recovery should not
* run. Scan the metadata to figure out the largest file ID.
*/
if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_EXISTED) ||
WT_IS_MAX_LSN(&metafile->ckpt_lsn)) {
/*
* Detect if we're going from logging disabled to enabled.
* We need to know this to verify LSNs and start at the correct
* log file later. If someone ran with logging, then disabled
* it and removed all the log files and then turned logging back
* on, we have to start logging in the log file number that is
* larger than any checkpoint LSN we have from the earlier time.
*/
WT_ERR(__recovery_file_scan(&r));
/*
* The array can be re-allocated in recovery_file_scan. Reset
* our pointer after scanning all the files.
*/
metafile = &r.files[WT_METAFILE_ID];
conn->next_file_id = r.max_fileid;
if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED) &&
WT_IS_MAX_LSN(&metafile->ckpt_lsn) &&
!WT_IS_MAX_LSN(&r.max_ckpt_lsn))
WT_ERR(__wt_log_reset(session, r.max_ckpt_lsn.l.file));
else
do_checkpoint = false;
goto done;
}
/*
* First, do a pass through the log to recover the metadata, and
* establish the last checkpoint LSN. Skip this when opening a hot
* backup: we already have the correct metadata in that case.
*
* If we're running with salvage and we hit an error, we ignore it
* and continue. In salvage we want to recover whatever part of the
* data we can from the last checkpoint up until whatever problem we
* detect in the log file. In salvage, we ignore errors from scanning
* the log so recovery can continue. Other errors remain errors.
*/
if (!was_backup) {
r.metadata_only = true;
/*
* If this is a read-only connection, check if the checkpoint
* LSN in the metadata file is up to date, indicating a clean
* shutdown.
*/
if (F_ISSET(conn, WT_CONN_READONLY)) {
WT_ERR(__wt_log_needs_recovery(
session, &metafile->ckpt_lsn, &needs_rec));
if (needs_rec)
WT_ERR_MSG(session, WT_RUN_RECOVERY,
"Read-only database needs recovery");
}
if (WT_IS_INIT_LSN(&metafile->ckpt_lsn))
ret = __wt_log_scan(session,
NULL, WT_LOGSCAN_FIRST, __txn_log_recover, &r);
else {
/*
* Start at the last checkpoint LSN referenced in the
* metadata. If we see the end of a checkpoint while
* scanning, we will change the full scan to start from
* there.
*/
r.ckpt_lsn = metafile->ckpt_lsn;
ret = __wt_log_scan(session,
&metafile->ckpt_lsn, 0, __txn_log_recover, &r);
}
if (F_ISSET(conn, WT_CONN_SALVAGE))
ret = 0;
/*
* If log scan couldn't find a file we expected to be around,
* this indicates a corruption of some sort.
*/
if (ret == ENOENT) {
F_SET(conn, WT_CONN_DATA_CORRUPTION);
ret = WT_ERROR;
}
WT_ERR(ret);
}
/* Scan the metadata to find the live files and their IDs. */
WT_ERR(__recovery_file_scan(&r));
/*
* Clear this out. We no longer need it and it could have been
* re-allocated when scanning the files.
*/
WT_NOT_READ(metafile, NULL);
/*
* We no longer need the metadata cursor: close it to avoid pinning any
* resources that could block eviction during recovery.
*/
r.files[0].c = NULL;
WT_ERR(metac->close(metac));
/*
* Now, recover all the files apart from the metadata.
* Pass WT_LOGSCAN_RECOVER so that old logs get truncated.
*/
r.metadata_only = false;
__wt_verbose(session, WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS,
"Main recovery loop: starting at %" PRIu32 "/%" PRIu32
" to %" PRIu32 "/%" PRIu32, r.ckpt_lsn.l.file, r.ckpt_lsn.l.offset,
r.max_rec_lsn.l.file, r.max_rec_lsn.l.offset);
WT_ERR(__wt_log_needs_recovery(session, &r.ckpt_lsn, &needs_rec));
/*
* Check if the database was shut down cleanly. If not
* return an error if the user does not want automatic
* recovery.
*/
if (needs_rec &&
(FLD_ISSET(conn->log_flags, WT_CONN_LOG_RECOVER_ERR) ||
F_ISSET(conn, WT_CONN_READONLY))) {
if (F_ISSET(conn, WT_CONN_READONLY))
WT_ERR_MSG(session, WT_RUN_RECOVERY,
"Read-only database needs recovery");
WT_ERR_MSG(session, WT_RUN_RECOVERY, "Database needs recovery");
}
if (F_ISSET(conn, WT_CONN_READONLY)) {
do_checkpoint = false;
goto done;
}
/*
* Recovery can touch more data than fits in cache, so it relies on
* regular eviction to manage paging. Start eviction threads for
* recovery without LAS cursors.
*/
WT_ERR(__wt_evict_create(session));
eviction_started = true;
/*
* Always run recovery even if it was a clean shutdown only if
* this is not a read-only connection.
* We can consider skipping it in the future.
*/
if (needs_rec)
FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_DIRTY);
if (WT_IS_INIT_LSN(&r.ckpt_lsn))
ret = __wt_log_scan(session, NULL,
WT_LOGSCAN_FIRST | WT_LOGSCAN_RECOVER,
__txn_log_recover, &r);
else
ret = __wt_log_scan(session, &r.ckpt_lsn,
WT_LOGSCAN_RECOVER, __txn_log_recover, &r);
if (F_ISSET(conn, WT_CONN_SALVAGE))
ret = 0;
WT_ERR(ret);
conn->next_file_id = r.max_fileid;
done: WT_ERR(__recovery_set_checkpoint_timestamp(&r));
if (do_checkpoint)
/*
* Forcibly log a checkpoint so the next open is fast and keep
* the metadata up to date with the checkpoint LSN and
* archiving.
*/
WT_ERR(session->iface.checkpoint(&session->iface, "force=1"));
/*
* If we're downgrading and have newer log files, force an archive,
* no matter what the archive setting is.
*/
if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_FORCE_DOWNGRADE))
WT_ERR(__wt_log_truncate_files(session, NULL, true));
FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_DONE);
err: WT_TRET(__recovery_free(&r));
__wt_free(session, config);
FLD_CLR(conn->log_flags, WT_CONN_LOG_RECOVER_DIRTY);
if (ret != 0) {
FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_FAILED);
__wt_err(session, ret, "Recovery failed");
}
/*
* Destroy the eviction threads that were started in support of
* recovery. They will be restarted once the lookaside table is
* created.
*/
if (eviction_started)
WT_TRET(__wt_evict_destroy(session));
WT_TRET(session->iface.close(&session->iface, NULL));
F_CLR(conn, WT_CONN_RECOVERING);
return (ret);
}
/*
* __wt_curlog_open --
* Initialize a log cursor.
*/
int
__wt_curlog_open(WT_SESSION_IMPL *session,
const char *uri, const char *cfg[], WT_CURSOR **cursorp)
{
WT_CONNECTION_IMPL *conn;
WT_CURSOR_STATIC_INIT(iface,
__wt_cursor_get_key, /* get-key */
__wt_cursor_get_value, /* get-value */
__wt_cursor_set_key, /* set-key */
__wt_cursor_set_value, /* set-value */
__curlog_compare, /* compare */
__wt_cursor_equals, /* equals */
__curlog_next, /* next */
__wt_cursor_notsup, /* prev */
__curlog_reset, /* reset */
__curlog_search, /* search */
__wt_cursor_search_near_notsup, /* search-near */
__wt_cursor_notsup, /* insert */
__wt_cursor_modify_notsup, /* modify */
__wt_cursor_notsup, /* update */
__wt_cursor_notsup, /* remove */
__wt_cursor_notsup, /* reserve */
__wt_cursor_reconfigure_notsup, /* reconfigure */
__wt_cursor_notsup, /* cache */
__wt_cursor_reopen_notsup, /* reopen */
__curlog_close); /* close */
WT_CURSOR *cursor;
WT_CURSOR_LOG *cl;
WT_DECL_RET;
WT_LOG *log;
WT_STATIC_ASSERT(offsetof(WT_CURSOR_LOG, iface) == 0);
conn = S2C(session);
log = conn->log;
WT_RET(__wt_calloc_one(session, &cl));
cursor = (WT_CURSOR *)cl;
*cursor = iface;
cursor->session = (WT_SESSION *)session;
cursor->key_format = WT_LOGC_KEY_FORMAT;
cursor->value_format = WT_LOGC_VALUE_FORMAT;
WT_ERR(__wt_calloc_one(session, &cl->cur_lsn));
WT_ERR(__wt_calloc_one(session, &cl->next_lsn));
WT_ERR(__wt_scr_alloc(session, 0, &cl->logrec));
WT_ERR(__wt_scr_alloc(session, 0, &cl->opkey));
WT_ERR(__wt_scr_alloc(session, 0, &cl->opvalue));
WT_INIT_LSN(cl->cur_lsn);
WT_INIT_LSN(cl->next_lsn);
WT_ERR(__wt_cursor_init(cursor, uri, NULL, cfg, cursorp));
if (log != NULL) {
/*
* The user may be trying to read a log record they just wrote.
* Log records may be buffered, so force out any now.
*/
WT_ERR(__wt_log_force_write(session, 1, NULL));
/* Log cursors block archiving. */
__wt_readlock(session, &log->log_archive_lock);
F_SET(cl, WT_CURLOG_ARCHIVE_LOCK);
(void)__wt_atomic_add32(&conn->log_cursors, 1);
}
if (0) {
err: WT_TRET(__curlog_close(cursor));
*cursorp = NULL;
}
return (ret);
}
/*
* __wt_txn_checkpoint_log --
* Write a log record for a checkpoint operation.
*/
int
__wt_txn_checkpoint_log(
WT_SESSION_IMPL *session, int full, uint32_t flags, WT_LSN *lsnp)
{
WT_DECL_ITEM(logrec);
WT_DECL_RET;
WT_ITEM *ckpt_snapshot, empty;
WT_LSN *ckpt_lsn;
WT_TXN *txn;
uint8_t *end, *p;
size_t recsize;
uint32_t i, rectype = WT_LOGREC_CHECKPOINT;
const char *fmt = WT_UNCHECKED_STRING(IIQIU);
txn = &session->txn;
ckpt_lsn = &txn->ckpt_lsn;
/*
* If this is a file sync, log it unless there is a full checkpoint in
* progress.
*/
if (!full) {
if (txn->full_ckpt) {
if (lsnp != NULL)
*lsnp = *ckpt_lsn;
return (0);
}
return (__txn_log_file_sync(session, flags, lsnp));
}
switch (flags) {
case WT_TXN_LOG_CKPT_PREPARE:
txn->full_ckpt = 1;
WT_ERR(__wt_log_ckpt_lsn(session, ckpt_lsn));
/*
* We need to make sure that the log records in the checkpoint
* LSN are on disk. In particular to make sure that the
* current log file exists.
*/
WT_ERR(__wt_log_force_sync(session, ckpt_lsn));
break;
case WT_TXN_LOG_CKPT_START:
/* Take a copy of the transaction snapshot. */
txn->ckpt_nsnapshot = txn->snapshot_count;
recsize = txn->ckpt_nsnapshot * WT_INTPACK64_MAXSIZE;
WT_ERR(__wt_scr_alloc(session, recsize, &txn->ckpt_snapshot));
p = txn->ckpt_snapshot->mem;
end = p + recsize;
for (i = 0; i < txn->snapshot_count; i++)
WT_ERR(__wt_vpack_uint(
&p, WT_PTRDIFF(end, p), txn->snapshot[i]));
break;
case WT_TXN_LOG_CKPT_STOP:
/*
* During a clean connection close, we get here without the
* prepare or start steps. In that case, log the current LSN
* as the checkpoint LSN.
*/
if (!txn->full_ckpt) {
txn->ckpt_nsnapshot = 0;
WT_CLEAR(empty);
ckpt_snapshot = ∅
WT_ERR(__wt_log_ckpt_lsn(session, ckpt_lsn));
} else
ckpt_snapshot = txn->ckpt_snapshot;
/* Write the checkpoint log record. */
WT_ERR(__wt_struct_size(session, &recsize, fmt,
rectype, ckpt_lsn->file, ckpt_lsn->offset,
txn->ckpt_nsnapshot, ckpt_snapshot));
WT_ERR(__wt_logrec_alloc(session, recsize, &logrec));
WT_ERR(__wt_struct_pack(session,
(uint8_t *)logrec->data + logrec->size, recsize, fmt,
rectype, ckpt_lsn->file, ckpt_lsn->offset,
txn->ckpt_nsnapshot, ckpt_snapshot));
logrec->size += (uint32_t)recsize;
WT_ERR(__wt_log_write(session, logrec, lsnp,
F_ISSET(S2C(session), WT_CONN_CKPT_SYNC) ?
WT_LOG_FSYNC : 0));
/*
* If this full checkpoint completed successfully and there is
* no hot backup in progress, tell the logging subsystem the
* checkpoint LSN so that it can archive.
*/
if (!S2C(session)->hot_backup)
WT_ERR(__wt_log_ckpt(session, ckpt_lsn));
/* FALLTHROUGH */
case WT_TXN_LOG_CKPT_CLEANUP:
/* Cleanup any allocated resources */
WT_INIT_LSN(ckpt_lsn);
txn->ckpt_nsnapshot = 0;
__wt_scr_free(session, &txn->ckpt_snapshot);
txn->full_ckpt = 0;
break;
WT_ILLEGAL_VALUE_ERR(session);
}
err: __wt_logrec_free(session, &logrec);
return (ret);
}