/*
* __curdump_get_key --
* WT_CURSOR->get_key for dump cursors.
*/
static int
__curdump_get_key(WT_CURSOR *cursor, ...)
{
WT_CURSOR *child;
WT_CURSOR_DUMP *cdump;
WT_DECL_RET;
WT_ITEM item, *itemp;
WT_SESSION_IMPL *session;
uint64_t recno;
va_list ap;
cdump = (WT_CURSOR_DUMP *)cursor;
child = cdump->child;
CURSOR_API_CALL(cursor, session, get_key, NULL);
if (WT_CURSOR_RECNO(cursor) && !F_ISSET(cursor, WT_CURSTD_RAW)) {
WT_ERR(child->get_key(child, &recno));
WT_ERR(__wt_buf_fmt(session, &cursor->key, "%" PRIu64, recno));
} else {
WT_ERR(child->get_key(child, &item));
WT_ERR(__raw_to_dump(session, &item,
&cursor->key, F_ISSET(cursor, WT_CURSTD_DUMP_HEX) ? 1 : 0));
}
va_start(ap, cursor);
if (F_ISSET(cursor, WT_CURSTD_RAW)) {
itemp = va_arg(ap, WT_ITEM *);
itemp->data = cursor->key.data;
itemp->size = cursor->key.size;
} else
/*
* __wt_bloom_open --
* Open a Bloom filter object for use by a single session. The filter must
* have been created and finalized.
*/
int
__wt_bloom_open(WT_SESSION_IMPL *session,
const char *uri, uint32_t factor, uint32_t k,
WT_CURSOR *owner, WT_BLOOM **bloomp)
{
WT_BLOOM *bloom;
WT_CURSOR *c;
WT_DECL_RET;
uint64_t size;
WT_RET(__bloom_init(session, uri, NULL, &bloom));
WT_ERR(__bloom_open_cursor(bloom, owner));
c = bloom->c;
/* Find the largest key, to get the size of the filter. */
WT_ERR(c->prev(c));
WT_ERR(c->get_key(c, &size));
WT_ERR(c->reset(c));
WT_ERR(__bloom_setup(bloom, 0, size, factor, k));
*bloomp = bloom;
return (0);
err: (void)__wt_bloom_close(bloom);
return (ret);
}
/*
* __recovery_file_scan --
* Scan the files referenced from the metadata and gather information
* about them for recovery.
*/
static int
__recovery_file_scan(WT_RECOVERY *r)
{
WT_CURSOR *c;
WT_DECL_RET;
int cmp;
const char *uri, *config;
/* Scan through all files in the metadata. */
c = r->files[0].c;
c->set_key(c, "file:");
if ((ret = c->search_near(c, &cmp)) != 0) {
/* Is the metadata empty? */
WT_RET_NOTFOUND_OK(ret);
return (0);
}
if (cmp < 0)
WT_RET_NOTFOUND_OK(c->next(c));
for (; ret == 0; ret = c->next(c)) {
WT_RET(c->get_key(c, &uri));
if (!WT_PREFIX_MATCH(uri, "file:"))
break;
WT_RET(c->get_value(c, &config));
WT_RET(__recovery_setup_file(r, uri, config));
}
WT_RET_NOTFOUND_OK(ret);
return (0);
}
boost::optional<RecordId> WiredTigerRecordStore::oplogStartHack(
OperationContext* txn, const RecordId& startingPosition) const {
if (!_useOplogHack)
return boost::none;
{
WiredTigerRecoveryUnit* wru = WiredTigerRecoveryUnit::get(txn);
_oplogSetStartHack(wru);
}
WiredTigerCursor cursor(_uri, _tableId, true, txn);
WT_CURSOR* c = cursor.get();
int cmp;
c->set_key(c, _makeKey(startingPosition));
int ret = WT_OP_CHECK(c->search_near(c, &cmp));
if (ret == 0 && cmp > 0)
ret = c->prev(c); // landed one higher than startingPosition
if (ret == WT_NOTFOUND)
return RecordId(); // nothing <= startingPosition
invariantWTOK(ret);
int64_t key;
ret = c->get_key(c, &key);
invariantWTOK(ret);
return _fromKey(key);
}
/*
* simple_walk_log --
* A simple walk of the write-ahead log.
* We wrote text messages into the log. Print them.
* This verifies we're decrypting properly.
*/
static void
simple_walk_log(WT_SESSION *session)
{
WT_CURSOR *cursor;
WT_ITEM logrec_key, logrec_value;
uint64_t txnid;
uint32_t fileid, log_file, log_offset, opcount, optype, rectype;
int found, ret;
error_check(session->open_cursor(session, "log:", NULL, NULL, &cursor));
found = 0;
while ((ret = cursor->next(cursor)) == 0) {
error_check(cursor->get_key(
cursor, &log_file, &log_offset, &opcount));
error_check(cursor->get_value(cursor, &txnid,
&rectype, &optype, &fileid, &logrec_key, &logrec_value));
if (rectype == WT_LOGREC_MESSAGE) {
found = 1;
printf("Application Log Record: %s\n",
(char *)logrec_value.data);
}
}
scan_end_check(ret == WT_NOTFOUND);
error_check(cursor->close(cursor));
if (found == 0) {
fprintf(stderr, "Did not find log messages.\n");
exit(EXIT_FAILURE);
}
}
static void
write_and_read_new(WT_SESSION *session)
{
WT_CURSOR *logc;
WT_ITEM logrec_key, logrec_value;
uint64_t txnid;
uint32_t fileid, log_file, log_offset, opcount, optype, rectype;
bool saw_msg;
/*
* Write a log record and force it to disk so we can read it.
*/
printf("Write log_printf record and verify.\n");
testutil_check(session->log_printf(session, "Test Log Record"));
testutil_check(session->log_flush(session, "sync=on"));
testutil_check(
session->open_cursor(session, "log:", NULL, NULL, &logc));
testutil_check(
session->open_cursor(session, "log:", NULL, NULL, &logc));
saw_msg = false;
while (logc->next(logc) == 0) {
/*
* We don't really need to get the key, but in case we want
* the LSN for some message, get it.
*/
testutil_check(logc->get_key(
logc, &log_file, &log_offset, &opcount));
testutil_check(logc->get_value(logc, &txnid,
&rectype, &optype, &fileid, &logrec_key, &logrec_value));
/*
* We should never see a record from us in log file 2. We wrote
* a record there, but then the record in log file 1 was
* truncated to be a partial record, ending the log there.
* So everything after that, including everything in log
* file 2, is invalid until we get to log file 3 which is where
* the post-recovery records will be written.
* The one exception in log file two is the system record for
* the previous log file's LSN. Although it is written by the
* system, we do walk it when using a cursor.
*/
if (log_file == 2 && rectype != WT_LOGREC_SYSTEM)
testutil_die(EINVAL, "Found LSN in Log 2");
#if 0
printf("LSN [%" PRIu32 "][%" PRIu32 "].%" PRIu32
": record type %" PRIu32 " optype %" PRIu32
" txnid %" PRIu64 " fileid %" PRIu32 "\n",
log_file, log_offset, opcount,
rectype, optype, txnid, fileid);
#endif
if (rectype == WT_LOGREC_MESSAGE) {
saw_msg = true;
printf("Application Record: %s\n",
(char *)logrec_value.data);
break;
}
}
testutil_check(logc->close(logc));
if (!saw_msg)
testutil_die(EINVAL, "Did not traverse log printf record");
}
std::vector<std::string> WiredTigerKVEngine::getAllIdents(OperationContext* opCtx) const {
std::vector<std::string> all;
WiredTigerCursor cursor("metadata:", WiredTigerSession::kMetadataTableId, false, opCtx);
WT_CURSOR* c = cursor.get();
if (!c)
return all;
while (c->next(c) == 0) {
const char* raw;
c->get_key(c, &raw);
StringData key(raw);
size_t idx = key.find(':');
if (idx == string::npos)
continue;
StringData type = key.substr(0, idx);
if (type != "table")
continue;
StringData ident = key.substr(idx + 1);
if (ident == "sizeStorer")
continue;
all.push_back(ident.toString());
}
return all;
}
/*! [thread scan] */
void *
scan_thread(void *conn_arg)
{
WT_CONNECTION *conn;
WT_CURSOR *cursor;
WT_SESSION *session;
const char *key, *value;
int ret;
conn = conn_arg;
ret = conn->open_session(conn, NULL, NULL, &session);
ret = session->open_cursor(
session, "table:access", NULL, NULL, &cursor);
/* Show all records. */
while ((ret = cursor->next(cursor)) == 0) {
ret = cursor->get_key(cursor, &key);
ret = cursor->get_value(cursor, &value);
printf("Got record: %s : %s\n", key, value);
}
if (ret != WT_NOTFOUND)
fprintf(stderr,
"WT_CURSOR.next: %s\n", session->strerror(session, ret));
return (NULL);
}
static void
page_init(uint64_t n)
{
WT_CONNECTION *conn;
WT_CURSOR *cursor;
WT_SESSION *session;
uint64_t recno, vrecno;
char buf[64];
conn = opts->conn;
testutil_check(conn->open_session(conn, NULL, NULL, &session));
testutil_check(
session->open_cursor(session, opts->uri, NULL, "append", &cursor));
vrecno = 0;
buf[0] = '\2';
for (recno = 1;; ++recno) {
if (opts->table_type == TABLE_FIX)
cursor->set_value(cursor, buf[0]);
else {
if (recno % 3 == 0)
++vrecno;
testutil_check(__wt_snprintf(buf,
sizeof(buf), "%" PRIu64 " VALUE ------", vrecno));
cursor->set_value(cursor, buf);
}
testutil_check(cursor->insert(cursor));
testutil_check(cursor->get_key(cursor, &opts->max_inserted_id));
if (opts->max_inserted_id >= n)
break;
}
}
void WiredTigerOperationStats::fetchStats(WT_SESSION* session,
const std::string& uri,
const std::string& config) {
invariant(session);
WT_CURSOR* c = nullptr;
const char* cursorConfig = config.empty() ? nullptr : config.c_str();
int ret = session->open_cursor(session, uri.c_str(), nullptr, cursorConfig, &c);
uassert(ErrorCodes::CursorNotFound, "Unable to open statistics cursor", ret == 0);
invariant(c);
ON_BLOCK_EXIT([&] { c->close(c); });
const char* desc;
uint64_t value;
uint64_t key;
while (c->next(c) == 0 && c->get_key(c, &key) == 0) {
fassert(51035, c->get_value(c, &desc, nullptr, &value) == 0);
#if defined(__s390x__)
_stats[key >> 32] = WiredTigerUtil::castStatisticsValue<long long>(value);
#else
_stats[key] = WiredTigerUtil::castStatisticsValue<long long>(value);
#endif // __s390x__
}
// Reset the statistics so that the next fetch gives the recent values.
invariantWTOK(c->reset(c));
}
/*
* simple_walk_log --
* A simple walk of the log.
*/
static int
simple_walk_log(WT_SESSION *session)
{
WT_CURSOR *cursor;
WT_LSN lsn;
WT_ITEM logrec_key, logrec_value;
uint64_t txnid;
uint32_t fileid, opcount, optype, rectype;
int ret;
/*! [log cursor open] */
ret = session->open_cursor(session, "log:", NULL, NULL, &cursor);
/*! [log cursor open] */
while ((ret = cursor->next(cursor)) == 0) {
/*! [log cursor get_key] */
ret = cursor->get_key(cursor, &lsn.file, &lsn.offset, &opcount);
/*! [log cursor get_key] */
/*! [log cursor get_value] */
ret = cursor->get_value(cursor, &txnid,
&rectype, &optype, &fileid, &logrec_key, &logrec_value);
/*! [log cursor get_value] */
print_record(&lsn, opcount,
rectype, optype, txnid, fileid, &logrec_key, &logrec_value);
}
if (ret == WT_NOTFOUND)
ret = 0;
ret = cursor->close(cursor);
return (ret);
}
/*
* Ensure that icount matches the number of records in the
* existing table.
*/
int find_table_count(CONFIG *cfg)
{
WT_CONNECTION *conn;
WT_CURSOR *cursor;
WT_SESSION *session;
char *key;
int ret;
conn = cfg->conn;
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) {
lprintf(cfg, ret, 0,
"open_session failed finding existing table count");
goto err;
}
if ((ret = session->open_cursor(session, cfg->uri,
NULL, NULL, &cursor)) != 0) {
lprintf(cfg, ret, 0,
"open_cursor failed finding existing table count");
goto err;
}
if ((ret = cursor->prev(cursor)) != 0) {
lprintf(cfg, ret, 0,
"cursor prev failed finding existing table count");
goto err;
}
cursor->get_key(cursor, &key);
cfg->icount = (uint32_t)atoi(key);
err: session->close(session, NULL);
return (ret);
}
int
util_backup(WT_SESSION *session, int argc, char *argv[])
{
WT_CURSOR *cursor;
WT_DECL_RET;
int ch;
char *config;
const char *directory, *name;
config = NULL;
while ((ch = util_getopt(argc, argv, "t:")) != EOF)
switch (ch) {
case 't':
if (append_target(util_optarg, &config))
return (1);
break;
case '?':
default:
return (usage());
}
argc -= util_optind;
argv += util_optind;
if (argc != 1) {
(void)usage();
goto err;
}
directory = *argv;
if ((ret = session->open_cursor(
session, "backup:", NULL, config, &cursor)) != 0) {
fprintf(stderr, "%s: cursor open(backup:) failed: %s\n",
progname, wiredtiger_strerror(ret));
goto err;
}
/* Copy the files. */
while (
(ret = cursor->next(cursor)) == 0 &&
(ret = cursor->get_key(cursor, &name)) == 0)
if ((ret = copy(name, directory)) != 0)
goto err;
if (ret == WT_NOTFOUND)
ret = 0;
if (ret != 0) {
fprintf(stderr, "%s: cursor next(backup:) failed: %s\n",
progname, wiredtiger_strerror(ret));
goto err;
}
err: if (config != NULL)
free(config);
if (cbuf != NULL)
free(cbuf);
return (ret);
}
/*
* verify_metadata --
* Verify all the tables expected are in the metadata. We expect all but
* the "corrupt" table name.
*/
static void
verify_metadata(WT_CONNECTION *conn, TABLE_INFO *tables)
{
TABLE_INFO *t;
WT_CURSOR *cursor;
int ret;
const char *kv;
/*
* Open a metadata cursor.
*/
testutil_check(conn->open_session(conn, NULL, NULL, &wt_session));
testutil_check(wt_session->open_cursor(
wt_session, "metadata:", NULL, NULL, &cursor));
reset_verified(tables);
/*
* We have to walk the cursor and walk the tables to match up that
* the expected tables are in the metadata. It is not efficient, but
* the list of tables is small. Walk the cursor once and the array
* of tables each time.
*/
while ((ret = cursor->next(cursor)) == 0) {
testutil_check(cursor->get_key(cursor, &kv));
for (t = tables; t->name != NULL; t++) {
if (strcmp(t->name, kv) == 0) {
testutil_assert(t->verified == false);
t->verified = true;
break;
}
}
}
testutil_assert(ret == WT_NOTFOUND);
testutil_check(cursor->close(cursor));
/*
* Any tables that were salvaged, make sure we can read the data.
* The corrupt table should never be salvaged.
*/
for (t = tables; t->name != NULL; t++) {
if (strcmp(t->name, CORRUPT) == 0 && !test_out_of_sync)
testutil_assert(t->verified == false);
else if (t->verified != true)
printf("%s not seen in metadata\n", t->name);
else {
testutil_check(wt_session->open_cursor(
wt_session, t->name, NULL, NULL, &cursor));
while ((ret = cursor->next(cursor)) == 0) {
testutil_check(cursor->get_value(cursor, &kv));
testutil_assert(strcmp(kv, VALUE) == 0);
}
testutil_assert(ret == WT_NOTFOUND);
testutil_check(cursor->close(cursor));
printf("%s metadata salvaged and data verified\n",
t->name);
}
}
}
boost::optional<Record> next() final {
if (_eof)
return {};
WT_CURSOR* c = _cursor->get();
bool mustAdvance = true;
if (_lastReturnedId.isNull() && !_forward && _rs._isCapped) {
// In this case we need to seek to the highest visible record.
const RecordId reverseCappedInitialSeekPoint =
_readUntilForOplog.isNull() ? _rs.lowestCappedHiddenRecord() : _readUntilForOplog;
if (!reverseCappedInitialSeekPoint.isNull()) {
c->set_key(c, _makeKey(reverseCappedInitialSeekPoint));
int cmp;
int seekRet = WT_OP_CHECK(c->search_near(c, &cmp));
if (seekRet == WT_NOTFOUND) {
_eof = true;
return {};
}
invariantWTOK(seekRet);
// If we landed at or past the lowest hidden record, we must advance to be in
// the visible range.
mustAdvance = _rs.isCappedHidden(reverseCappedInitialSeekPoint)
? (cmp >= 0)
: (cmp > 0); // No longer hidden.
}
}
if (mustAdvance) {
// Nothing after the next line can throw WCEs.
// Note that an unpositioned (or eof) WT_CURSOR returns the first/last entry in the
// table when you call next/prev.
int advanceRet = WT_OP_CHECK(_forward ? c->next(c) : c->prev(c));
if (advanceRet == WT_NOTFOUND) {
_eof = true;
return {};
}
invariantWTOK(advanceRet);
}
int64_t key;
invariantWTOK(c->get_key(c, &key));
const RecordId id = _fromKey(key);
if (!isVisible(id)) {
_eof = true;
return {};
}
WT_ITEM value;
invariantWTOK(c->get_value(c, &value));
_lastReturnedId = id;
return {{id, {static_cast<const char*>(value.data), static_cast<int>(value.size)}}};
}
/*
* __backup_all --
* Backup all objects in the database.
*/
static int
__backup_all(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb)
{
WT_CONFIG_ITEM cval;
WT_CURSOR *cursor;
WT_DECL_RET;
const char *key, *value;
cursor = NULL;
/*
* Open a cursor on the metadata file and copy all of the entries to
* the hot backup file.
*/
WT_ERR(__wt_metadata_cursor(session, NULL, &cursor));
while ((ret = cursor->next(cursor)) == 0) {
WT_ERR(cursor->get_key(cursor, &key));
WT_ERR(cursor->get_value(cursor, &value));
WT_ERR_TEST((fprintf(
cb->bfp, "%s\n%s\n", key, value) < 0), __wt_errno());
/*
* While reading the metadata file, check there are no "sources"
* or "types" which can't support hot backup. This checks for
* a data source that's non-standard, which can't be backed up,
* but is also sanity checking: if there's an entry backed by
* anything other than a file or lsm entry, we're confused.
*/
if ((ret = __wt_config_getones(
session, value, "type", &cval)) == 0 &&
!WT_PREFIX_MATCH_LEN(cval.str, cval.len, "file") &&
!WT_PREFIX_MATCH_LEN(cval.str, cval.len, "lsm"))
WT_ERR_MSG(session, ENOTSUP,
"hot backup is not supported for objects of "
"type %.*s", (int)cval.len, cval.str);
WT_ERR_NOTFOUND_OK(ret);
if ((ret =__wt_config_getones(
session, value, "source", &cval)) == 0 &&
!WT_PREFIX_MATCH_LEN(cval.str, cval.len, "file:") &&
!WT_PREFIX_MATCH_LEN(cval.str, cval.len, "lsm:"))
WT_ERR_MSG(session, ENOTSUP,
"hot backup is not supported for objects of "
"source %.*s", (int)cval.len, cval.str);
WT_ERR_NOTFOUND_OK(ret);
}
WT_ERR_NOTFOUND_OK(ret);
/* Build a list of the file objects that need to be copied. */
WT_WITH_DHANDLE_LOCK(session,
ret = __wt_meta_btree_apply(
session, __backup_list_all_append, NULL));
err: if (cursor != NULL)
WT_TRET(cursor->close(cursor));
return (ret);
}
/*
* Read the index by year and print out who was in office that year.
*/
static void
read_index(WT_SESSION *session)
{
WT_CURSOR *cursor;
int i, ret;
char *first_name, *last_name;
uint16_t rec_year, term_end, term_start, year;
year = 0;
srand((unsigned int)getpid());
error_check(session->open_cursor(
session, "index:presidents:term", NULL, NULL, &cursor));
/*
* Pick 10 random years and read the data.
*/
for (i = 0; i < 10; i++) {
year = (uint16_t)((rand() % YEAR_SPAN) + YEAR_BASE);
printf("Year %" PRIu16 ":\n", year);
cursor->set_key(cursor, year);
error_check(cursor->search(cursor));
error_check(cursor->get_key(cursor, &rec_year));
error_check(cursor->get_value(cursor,
&last_name, &first_name, &term_start, &term_end));
/* Report all presidents that served during the chosen year */
ret = 0;
while (term_start <= year &&
year <= term_end && year == rec_year) {
printf("\t%s %s\n", first_name, last_name);
if ((ret = cursor->next(cursor)) != 0)
break;
error_check(cursor->get_key(cursor, &rec_year));
error_check(cursor->get_value(cursor,
&last_name, &first_name, &term_start, &term_end));
}
scan_end_check(ret == 0 || ret == WT_NOTFOUND);
}
error_check(cursor->close(cursor));
}
int main(void)
{
/*! [access example connection] */
WT_CONNECTION *conn;
WT_CURSOR *cursor;
WT_SESSION *session;
const char *key, *value;
int ret;
if ((ret = wiredtiger_open(home, NULL, "create", &conn)) != 0 ||
(ret = conn->open_session(conn, NULL, NULL, &session)) != 0) {
fprintf(stderr, "Error connecting to %s: %s\n",
home, wiredtiger_strerror(ret));
return (ret);
}
/*! [access example connection] */
/*! [access example table create] */
ret = session->create(session,
"table:access", "key_format=S,value_format=S");
/*! [access example table create] */
/*! [access example cursor open] */
ret = session->open_cursor(session,
"table:access", NULL, NULL, &cursor);
/*! [access example cursor open] */
/*! [access example cursor insert] */
cursor->set_key(cursor, "key1"); /* Insert a record. */
cursor->set_value(cursor, "value1");
ret = cursor->insert(cursor);
/*! [access example cursor insert] */
/*! [access example cursor list] */
ret = cursor->reset(cursor); /* Restart the scan. */
while ((ret = cursor->next(cursor)) == 0) {
ret = cursor->get_key(cursor, &key);
ret = cursor->get_value(cursor, &value);
printf("Got record: %s : %s\n", key, value);
}
/*! [access example cursor list] */
/*! [access example close] */
ret = conn->close(conn, NULL);
/*! [access example close] */
return (ret);
}
/*
* __wt_meta_btree_apply --
* Apply a function to all files listed in the metadata, apart from the
* metadata file.
*/
int
__wt_meta_btree_apply(WT_SESSION_IMPL *session,
int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[])
{
WT_CURSOR *cursor;
WT_DATA_HANDLE *saved_dhandle;
WT_DECL_RET;
const char *uri;
int cmp, tret;
saved_dhandle = session->dhandle;
WT_RET(__wt_metadata_cursor(session, NULL, &cursor));
cursor->set_key(cursor, "file:");
if ((tret = cursor->search_near(cursor, &cmp)) == 0 && cmp < 0)
tret = cursor->next(cursor);
for (; tret == 0; tret = cursor->next(cursor)) {
WT_ERR(cursor->get_key(cursor, &uri));
if (!WT_PREFIX_MATCH(uri, "file:"))
break;
else if (strcmp(uri, WT_METAFILE_URI) == 0)
continue;
/*
* We need to pull the handle into the session handle cache
* and make sure it's referenced to stop other internal code
* dropping the handle (e.g in LSM when cleaning up obsolete
* chunks). Holding the metadata lock isn't enough.
*/
ret = __wt_session_get_btree(session, uri, NULL, NULL, 0);
if (ret == 0) {
WT_SAVE_DHANDLE(session,
ret = func(session, cfg));
if (WT_META_TRACKING(session))
WT_TRET(
__wt_meta_track_handle_lock(session, 0));
else
WT_TRET(__wt_session_release_btree(session));
} else if (ret == EBUSY)
ret = __wt_conn_btree_apply_single(
session, uri, NULL, func, cfg);
WT_ERR(ret);
}
if (tret != WT_NOTFOUND)
WT_TRET(tret);
err: WT_TRET(cursor->close(cursor));
session->dhandle = saved_dhandle;
return (ret);
}
/*
* process --
* Salvage, verify and dump the created file.
*/
void
process(void)
{
FILE *fp;
WT_CONNECTION *conn;
WT_CURSOR *cursor;
const char *key, *value;
WT_SESSION *session;
char config[100];
/* Salvage. */
config[0] = '\0';
if (verbose)
snprintf(config, sizeof(config),
"error_prefix=\"%s\",verbose=[salvage,verify],",
progname);
strcat(config, "log=(enabled=false),");
CHECK(wiredtiger_open(NULL, NULL, config, &conn) == 0);
CHECK(conn->open_session(conn, NULL, NULL, &session) == 0);
CHECK(session->salvage(session, "file:" SLVG, 0) == 0);
CHECK(conn->close(conn, 0) == 0);
/* Verify. */
CHECK(wiredtiger_open(NULL, NULL, config, &conn) == 0);
CHECK(conn->open_session(conn, NULL, NULL, &session) == 0);
CHECK(session->verify(session, "file:" SLVG, 0) == 0);
CHECK(conn->close(conn, 0) == 0);
/* Dump. */
CHECK((fp = fopen(DUMP, "w")) != NULL);
CHECK(wiredtiger_open(NULL, NULL, config, &conn) == 0);
CHECK(conn->open_session(conn, NULL, NULL, &session) == 0);
CHECK(session->open_cursor(
session, "file:" SLVG, NULL, "dump=print", &cursor) == 0);
while (cursor->next(cursor) == 0) {
if (page_type == WT_PAGE_ROW_LEAF) {
CHECK(cursor->get_key(cursor, &key) == 0);
CHECK(fputs(key, fp) >= 0);
CHECK(fputc('\n', fp) >= 0);
}
CHECK(cursor->get_value(cursor, &value) == 0);
CHECK(fputs(value, fp) >= 0);
CHECK(fputc('\n', fp) >= 0);
}
CHECK(conn->close(conn, 0) == 0);
CHECK(fclose(fp) == 0);
}
int
backup(WT_SESSION *session)
{
char buf[1024];
/*! [backup]*/
WT_CURSOR *cursor;
const char *filename;
int ret;
/* Create the backup directory. */
ret = mkdir("/path/database.backup", 077);
/* Open the backup data source. */
ret = session->open_cursor(session, "backup:", NULL, NULL, &cursor);
/* Copy the list of files. */
while (
(ret = cursor->next(cursor)) == 0 &&
(ret = cursor->get_key(cursor, &filename)) == 0) {
(void)snprintf(buf, sizeof(buf),
"cp /path/database/%s /path/database.backup/%s",
filename, filename);
ret = system(buf);
}
if (ret == WT_NOTFOUND)
ret = 0;
if (ret != 0)
fprintf(stderr, "%s: cursor next(backup:) failed: %s\n",
progname, session->strerror(session, ret));
ret = cursor->close(cursor);
/*! [backup]*/
/*! [incremental backup]*/
/* Open the backup data source for incremental backup. */
ret = session->open_cursor(
session, "backup:", NULL, "target=(\"log:\")", &cursor);
/*! [incremental backup]*/
ret = cursor->close(cursor);
/*! [backup of a checkpoint]*/
ret = session->checkpoint(session, "drop=(from=June01),name=June01");
/*! [backup of a checkpoint]*/
return (ret);
}
static void
take_full_backup(WT_SESSION *session, int i)
{
WT_CURSOR *cursor;
int j, ret;
char buf[1024], h[256];
const char *filename, *hdir;
/*
* First time through we take a full backup into the incremental
* directories. Otherwise only into the appropriate full directory.
*/
if (i != 0) {
(void)snprintf(h, sizeof(h), "%s.%d", home_full, i);
hdir = h;
} else
hdir = home_incr;
error_check(
session->open_cursor(session, "backup:", NULL, NULL, &cursor));
while ((ret = cursor->next(cursor)) == 0) {
error_check(cursor->get_key(cursor, &filename));
if (i == 0)
/*
* Take a full backup into each incremental directory.
*/
for (j = 0; j < MAX_ITERATIONS; j++) {
(void)snprintf(h, sizeof(h),
"%s.%d", home_incr, j);
(void)snprintf(buf, sizeof(buf),
"cp %s/%s %s/%s",
home, filename, h, filename);
error_check(system(buf));
}
else {
(void)snprintf(h, sizeof(h), "%s.%d", home_full, i);
(void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s",
home, filename, hdir, filename);
error_check(system(buf));
}
}
scan_end_check(ret == WT_NOTFOUND);
error_check(cursor->close(cursor));
}
static void
backup(WT_SESSION *session)
{
char buf[1024];
/*! [backup]*/
WT_CURSOR *cursor;
const char *filename;
int ret;
/* Create the backup directory. */
error_check(mkdir("/path/database.backup", 077));
/* Open the backup data source. */
error_check(session->open_cursor(
session, "backup:", NULL, NULL, &cursor));
/* Copy the list of files. */
while ((ret = cursor->next(cursor)) == 0) {
error_check(cursor->get_key(cursor, &filename));
(void)snprintf(buf, sizeof(buf),
"cp /path/database/%s /path/database.backup/%s",
filename, filename);
error_check(system(buf));
}
scan_end_check(ret == WT_NOTFOUND);
error_check(cursor->close(cursor));
/*! [backup]*/
/*! [incremental backup]*/
/* Open the backup data source for incremental backup. */
error_check(session->open_cursor(
session, "backup:", NULL, "target=(\"log:\")", &cursor));
/*! [incremental backup]*/
error_check(cursor->close(cursor));
/*! [backup of a checkpoint]*/
error_check(session->checkpoint(
session, "drop=(from=June01),name=June01"));
/*! [backup of a checkpoint]*/
}
/*
* __metadata_load_bulk --
* Create any bulk-loaded file stubs.
*/
static int
__metadata_load_bulk(WT_SESSION_IMPL *session)
{
WT_CURSOR *cursor;
WT_DECL_RET;
uint32_t allocsize;
bool exist;
const char *filecfg[] = {
WT_CONFIG_BASE(session, file_meta), NULL, NULL };
const char *key, *value;
/*
* If a file was being bulk-loaded during the hot backup, it will appear
* in the metadata file, but the file won't exist. Create on demand.
*/
WT_RET(__wt_metadata_cursor(session, &cursor));
while ((ret = cursor->next(cursor)) == 0) {
WT_ERR(cursor->get_key(cursor, &key));
if (!WT_PREFIX_SKIP(key, "file:"))
continue;
/* If the file exists, it's all good. */
WT_ERR(__wt_fs_exist(session, key, &exist));
if (exist)
continue;
/*
* If the file doesn't exist, assume it's a bulk-loaded file;
* retrieve the allocation size and re-create the file.
*/
WT_ERR(cursor->get_value(cursor, &value));
filecfg[1] = value;
WT_ERR(__wt_direct_io_size_check(
session, filecfg, "allocation_size", &allocsize));
WT_ERR(__wt_block_manager_create(session, key, allocsize));
}
WT_ERR_NOTFOUND_OK(ret);
err: WT_TRET(__wt_metadata_cursor_release(session, &cursor));
return (ret);
}
int main(void)
{
int ret;
WT_CONNECTION *conn;
WT_SESSION *session;
WT_CURSOR *cursor;
const char *key, *value;
/*! [configure cache size] */
if ((ret = wiredtiger_open(home, NULL,
"create,cache_size=500M", &conn)) != 0)
fprintf(stderr, "Error connecting to %s: %s\n",
home, wiredtiger_strerror(ret));
/*! [configure cache size] */
/*! [create a table] */
ret = conn->open_session(conn, NULL, NULL, &session);
ret = session->create(session,
"table:access", "key_format=S,value_format=S");
/*! [create a table] */
/*! [transaction] */
ret = session->begin_transaction(session, "priority=100,name=mytxn");
ret = session->open_cursor(session, "config:", NULL, NULL, &cursor);
while ((ret = cursor->next(cursor)) == 0) {
cursor->get_key(cursor, &key);
cursor->get_value(cursor, &value);
printf("configuration value: %s = %s\n", key, value);
}
ret = session->commit_transaction(session, NULL);
/*! [transaction] */
ret = conn->close(conn, NULL);
return (ret);
}
/*! [thread scan] */
void *
scan_thread(void *arg)
{
WT_SESSION *session;
WT_CURSOR *cursor;
const char *key, *value;
int ret;
ret = conn->open_session(conn, NULL, NULL, &session);
ret = session->open_cursor(session, "table:access",
NULL, NULL, &cursor);
/* Show all records. */
while ((ret = cursor->next(cursor)) == 0) {
ret = cursor->get_key(cursor, &key);
ret = cursor->get_value(cursor, &value);
printf("Got record: %s : %s\n", key, value);
}
return (arg);
}
static void
take_incr_backup(WT_SESSION *session, int i)
{
WT_CURSOR *cursor;
int j, ret;
char buf[1024], h[256];
const char *filename;
error_check(session->open_cursor(
session, "backup:", NULL, "target=(\"log:\")", &cursor));
while ((ret = cursor->next(cursor)) == 0) {
error_check(cursor->get_key(cursor, &filename));
/*
* Copy into the 0 incremental directory and then each of the
* incremental directories for this iteration and later.
*/
(void)snprintf(h, sizeof(h), "%s.0", home_incr);
(void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s",
home, filename, h, filename);
error_check(system(buf));
for (j = i; j < MAX_ITERATIONS; j++) {
(void)snprintf(h, sizeof(h), "%s.%d", home_incr, j);
(void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s",
home, filename, h, filename);
error_check(system(buf));
}
}
scan_end_check(ret == WT_NOTFOUND);
/*
* With an incremental cursor, we want to truncate on the backup
* cursor to archive the logs. Only do this if the copy process
* was entirely successful.
*/
error_check(session->truncate(session, "log:", cursor, NULL, NULL));
error_check(cursor->close(cursor));
}
/*
* A thread dedicated to appending records into a table. Works with fixed
* length column stores and variable length column stores.
* One thread (the first thread created by an application) checks for a
* terminating condition after each insert.
*/
WT_THREAD_RET
thread_append(void *arg)
{
TEST_OPTS *opts;
WT_CONNECTION *conn;
WT_CURSOR *cursor;
WT_SESSION *session;
uint64_t id, recno;
char buf[64];
opts = (TEST_OPTS *)arg;
conn = opts->conn;
id = __wt_atomic_fetch_addv64(&opts->next_threadid, 1);
testutil_check(conn->open_session(conn, NULL, NULL, &session));
testutil_check(
session->open_cursor(session, opts->uri, NULL, "append", &cursor));
buf[0] = '\2';
for (recno = 1; opts->running; ++recno) {
if (opts->table_type == TABLE_FIX)
cursor->set_value(cursor, buf[0]);
else {
testutil_check(__wt_snprintf(buf, sizeof(buf),
"%" PRIu64 " VALUE ------", recno));
cursor->set_value(cursor, buf);
}
testutil_check(cursor->insert(cursor));
if (id == 0) {
testutil_check(
cursor->get_key(cursor, &opts->max_inserted_id));
if (opts->max_inserted_id >= opts->nrecords)
opts->running = false;
}
}
return (WT_THREAD_RET_VALUE);
}
void WiredTigerSizeStorer::loadFrom( WiredTigerSession* session,
const std::string& uri ) {
_checkMagic();
Map m;
{
WT_SESSION* s = session->getSession();
WT_CURSOR* c = NULL;
int ret = s->open_cursor( s, uri.c_str(), NULL, NULL, &c );
if ( ret == ENOENT ) {
// doesn't exist, we'll create later
return;
}
invariantWTOK( ret );
while ( c->next(c) == 0 ) {
WT_ITEM key;
WT_ITEM value;
invariantWTOK( c->get_key(c, &key ) );
invariantWTOK( c->get_value(c, &value ) );
std::string uriKey( reinterpret_cast<const char*>( key.data ), key.size );
BSONObj data( reinterpret_cast<const char*>( value.data ) );
LOG(2) << "WiredTigerSizeStorer::loadFrom " << uriKey << " -> " << data;
Entry& e = m[uriKey];
e.numRecords = data["numRecords"].safeNumberLong();
e.dataSize = data["dataSize"].safeNumberLong();
e.dirty = false;
e.rs = NULL;
}
invariantWTOK( c->close(c) );
}
boost::mutex::scoped_lock lk( _entriesMutex );
_entries = m;
}
/*
* __lsm_bloom_create --
* Create a bloom filter for a chunk of the LSM tree that has been
* checkpointed but not yet been merged.
*/
static int
__lsm_bloom_create(WT_SESSION_IMPL *session,
WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK *chunk, u_int chunk_off)
{
WT_BLOOM *bloom;
WT_CURSOR *src;
WT_DECL_RET;
WT_ITEM key;
uint64_t insert_count;
WT_RET(__wt_lsm_tree_setup_bloom(session, lsm_tree, chunk));
bloom = NULL;
/*
* This is merge-like activity, and we don't want compacts to give up
* because we are creating a bunch of bloom filters before merging.
*/
++lsm_tree->merge_progressing;
WT_RET(__wt_bloom_create(session, chunk->bloom_uri,
lsm_tree->bloom_config, chunk->count,
lsm_tree->bloom_bit_count, lsm_tree->bloom_hash_count, &bloom));
/* Open a special merge cursor just on this chunk. */
WT_ERR(__wt_open_cursor(session, lsm_tree->name, NULL, NULL, &src));
F_SET(src, WT_CURSTD_RAW);
WT_ERR(__wt_clsm_init_merge(src, chunk_off, chunk->id, 1));
/*
* Setup so that we don't hold pages we read into cache, and so
* that we don't get stuck if the cache is full. If we allow
* ourselves to get stuck creating bloom filters, the entire tree
* can stall since there may be no worker threads available to flush.
*/
F_SET(session, WT_SESSION_NO_CACHE | WT_SESSION_NO_EVICTION);
for (insert_count = 0; (ret = src->next(src)) == 0; insert_count++) {
WT_ERR(src->get_key(src, &key));
WT_ERR(__wt_bloom_insert(bloom, &key));
}
WT_ERR_NOTFOUND_OK(ret);
WT_TRET(src->close(src));
WT_TRET(__wt_bloom_finalize(bloom));
WT_ERR(ret);
F_CLR(session, WT_SESSION_NO_CACHE);
/* Load the new Bloom filter into cache. */
WT_CLEAR(key);
WT_ERR_NOTFOUND_OK(__wt_bloom_get(bloom, &key));
WT_ERR(__wt_verbose(session, WT_VERB_LSM,
"LSM worker created bloom filter %s. "
"Expected %" PRIu64 " items, got %" PRIu64,
chunk->bloom_uri, chunk->count, insert_count));
/* Ensure the bloom filter is in the metadata. */
WT_ERR(__wt_lsm_tree_writelock(session, lsm_tree));
F_SET(chunk, WT_LSM_CHUNK_BLOOM);
ret = __wt_lsm_meta_write(session, lsm_tree);
++lsm_tree->dsk_gen;
WT_TRET(__wt_lsm_tree_writeunlock(session, lsm_tree));
if (ret != 0)
WT_ERR_MSG(session, ret, "LSM bloom worker metadata write");
err: if (bloom != NULL)
WT_TRET(__wt_bloom_close(bloom));
F_CLR(session, WT_SESSION_NO_CACHE | WT_SESSION_NO_EVICTION);
return (ret);
}
