/*
* thread_ckpt_run --
* Runner function for the checkpoint thread.
*/
static WT_THREAD_RET
thread_ckpt_run(void *arg)
{
FILE *fp;
WT_RAND_STATE rnd;
WT_SESSION *session;
WT_THREAD_DATA *td;
uint64_t ts;
uint32_t sleep_time;
int i, ret;
bool first_ckpt;
__wt_random_init(&rnd);
td = (WT_THREAD_DATA *)arg;
/*
* Keep a separate file with the records we wrote for checking.
*/
(void)unlink(ckpt_file);
if ((ret = td->conn->open_session(td->conn, NULL, NULL, &session)) != 0)
testutil_die(ret, "WT_CONNECTION:open_session");
first_ckpt = true;
ts = 0;
for (i = 0; ;++i) {
sleep_time = __wt_random(&rnd) % MAX_CKPT_INTERVAL;
sleep(sleep_time);
if (use_ts)
ts = global_ts;
/*
* Since this is the default, send in this string even if
* running without timestamps.
*/
testutil_check(session->checkpoint(
session, "use_timestamp=true"));
printf("Checkpoint %d complete. Minimum ts %" PRIu64 "\n",
i, ts);
fflush(stdout);
/*
* Create the checkpoint file so that the parent process knows
* at least one checkpoint has finished and can start its
* timer.
*/
if (first_ckpt) {
testutil_checksys((fp = fopen(ckpt_file, "w")) == NULL);
first_ckpt = false;
testutil_checksys(fclose(fp) != 0);
}
}
/* NOTREACHED */
}
/*
* thread_ckpt_run --
* Runner function for the checkpoint thread.
*/
static WT_THREAD_RET
thread_ckpt_run(void *arg)
{
FILE *fp;
WT_RAND_STATE rnd;
WT_SESSION *session;
THREAD_DATA *td;
uint64_t stable;
uint32_t sleep_time;
int i;
bool first_ckpt;
char ts_string[WT_TS_HEX_STRING_SIZE];
__wt_random_init(&rnd);
td = (THREAD_DATA *)arg;
/*
* Keep a separate file with the records we wrote for checking.
*/
(void)unlink(ckpt_file);
testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session));
first_ckpt = true;
for (i = 0; ;++i) {
sleep_time = __wt_random(&rnd) % MAX_CKPT_INVL;
sleep(sleep_time);
/*
* Since this is the default, send in this string even if
* running without timestamps.
*/
testutil_check(session->checkpoint(
session, "use_timestamp=true"));
testutil_check(td->conn->query_timestamp(
td->conn, ts_string, "get=last_checkpoint"));
testutil_assert(sscanf(ts_string, "%" SCNx64, &stable) == 1);
printf("Checkpoint %d complete at stable %"
PRIu64 ".\n", i, stable);
fflush(stdout);
/*
* Create the checkpoint file so that the parent process knows
* at least one checkpoint has finished and can start its
* timer.
*/
if (first_ckpt) {
testutil_checksys((fp = fopen(ckpt_file, "w")) == NULL);
first_ckpt = false;
testutil_checksys(fclose(fp) != 0);
}
}
/* NOTREACHED */
}
int
main(int argc, char *argv[])
{
struct stat sb;
FILE *fp;
WT_CONNECTION *conn;
WT_CURSOR *cur_coll, *cur_local, *cur_oplog, *cur_stable;
WT_RAND_STATE rnd;
WT_SESSION *session;
pid_t pid;
uint64_t absent_coll, absent_local, absent_oplog, count, key, last_key;
uint64_t first_miss, middle_coll, middle_local, middle_oplog;
uint64_t stable_fp, stable_val, val[MAX_TH+1];
uint32_t i, nth, timeout;
int ch, status, ret;
const char *working_dir;
char buf[512], fname[64], kname[64], statname[1024];
bool fatal, rand_th, rand_time, verify_only;
(void)testutil_set_progname(argv);
compat = inmem = false;
use_ts = true;
nth = MIN_TH;
rand_th = rand_time = true;
timeout = MIN_TIME;
verify_only = false;
working_dir = "WT_TEST.timestamp-abort";
while ((ch = __wt_getopt(progname, argc, argv, "Ch:mT:t:vz")) != EOF)
switch (ch) {
case 'C':
compat = true;
break;
case 'h':
working_dir = __wt_optarg;
break;
case 'm':
inmem = true;
break;
case 'T':
rand_th = false;
nth = (uint32_t)atoi(__wt_optarg);
break;
case 't':
rand_time = false;
timeout = (uint32_t)atoi(__wt_optarg);
break;
case 'v':
verify_only = true;
break;
case 'z':
use_ts = false;
break;
default:
usage();
}
argc -= __wt_optind;
argv += __wt_optind;
if (argc != 0)
usage();
testutil_work_dir_from_path(home, sizeof(home), working_dir);
/*
* If the user wants to verify they need to tell us how many threads
* there were so we can find the old record files.
*/
if (verify_only && rand_th) {
fprintf(stderr,
"Verify option requires specifying number of threads\n");
exit (EXIT_FAILURE);
}
if (!verify_only) {
testutil_make_work_dir(home);
__wt_random_init_seed(NULL, &rnd);
if (rand_time) {
timeout = __wt_random(&rnd) % MAX_TIME;
if (timeout < MIN_TIME)
timeout = MIN_TIME;
}
if (rand_th) {
nth = __wt_random(&rnd) % MAX_TH;
if (nth < MIN_TH)
nth = MIN_TH;
}
printf("Parent: compatibility: %s, "
"in-mem log sync: %s, timestamp in use: %s\n",
compat ? "true" : "false",
inmem ? "true" : "false",
use_ts ? "true" : "false");
printf("Parent: Create %" PRIu32
" threads; sleep %" PRIu32 " seconds\n", nth, timeout);
/*
* Fork a child to insert as many items. We will then randomly
* kill the child, run recovery and make sure all items we wrote
* exist after recovery runs.
*/
testutil_checksys((pid = fork()) < 0);
if (pid == 0) { /* child */
run_workload(nth);
return (EXIT_SUCCESS);
}
/* parent */
/*
* Sleep for the configured amount of time before killing
* the child. Start the timeout from the time we notice that
* the file has been created. That allows the test to run
* correctly on really slow machines. Verify the process ID
* still exists in case the child aborts for some reason we
* don't stay in this loop forever.
*/
testutil_check(__wt_snprintf(
statname, sizeof(statname), "%s/%s", home, ckpt_file));
while (stat(statname, &sb) != 0 && kill(pid, 0) == 0)
sleep(1);
sleep(timeout);
/*
* !!! It should be plenty long enough to make sure more than
* one log file exists. If wanted, that check would be added
* here.
*/
printf("Kill child\n");
testutil_checksys(kill(pid, SIGKILL) != 0);
testutil_checksys(waitpid(pid, &status, 0) == -1);
}
/*
* !!! If we wanted to take a copy of the directory before recovery,
* this is the place to do it.
*/
if (chdir(home) != 0)
testutil_die(errno, "parent chdir: %s", home);
testutil_check(__wt_snprintf(buf, sizeof(buf),
"rm -rf ../%s.SAVE && mkdir ../%s.SAVE && "
"cp -p WiredTigerLog.* ../%s.SAVE",
home, home, home));
(void)system(buf);
printf("Open database, run recovery and verify content\n");
/*
* Open the connection which forces recovery to be run.
*/
if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn)) != 0)
testutil_die(ret, "wiredtiger_open");
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
testutil_die(ret, "WT_CONNECTION:open_session");
/*
* Open a cursor on all the tables.
*/
if ((ret = session->open_cursor(session,
uri_collection, NULL, NULL, &cur_coll)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_collection);
if ((ret = session->open_cursor(session,
uri_local, NULL, NULL, &cur_local)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_local);
if ((ret = session->open_cursor(session,
uri_oplog, NULL, NULL, &cur_oplog)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_oplog);
if ((ret = session->open_cursor(session,
stable_store, NULL, NULL, &cur_stable)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", stable_store);
/*
* Find the biggest stable timestamp value that was saved.
*/
stable_val = 0;
memset(val, 0, sizeof(val));
while (cur_stable->next(cur_stable) == 0) {
cur_stable->get_key(cur_stable, &key);
cur_stable->get_value(cur_stable, &val[key]);
if (val[key] > stable_val)
stable_val = val[key];
if (use_ts)
printf("Stable: key %" PRIu64 " value %" PRIu64 "\n",
key, val[key]);
}
if (use_ts)
printf("Got stable_val %" PRIu64 "\n", stable_val);
count = 0;
absent_coll = absent_local = absent_oplog = 0;
fatal = false;
for (i = 1; i <= nth; ++i) {
first_miss = middle_coll = middle_local = middle_oplog = 0;
testutil_check(__wt_snprintf(
fname, sizeof(fname), RECORDS_FILE, i));
if ((fp = fopen(fname, "r")) == NULL)
testutil_die(errno, "fopen: %s", fname);
/*
* For every key in the saved file, verify that the key exists
* in the table after recovery. If we're doing in-memory
* log buffering we never expect a record missing in the middle,
* but records may be missing at the end. If we did
* write-no-sync, we expect every key to have been recovered.
*/
for (last_key = UINT64_MAX;; ++count, last_key = key) {
ret = fscanf(fp, "%" SCNu64 "%" SCNu64 "\n",
&stable_fp, &key);
if (ret != EOF && ret != 2) {
/*
* If we find a partial line, consider it
* like an EOF.
*/
if (ret == 1 || ret == 0)
break;
testutil_die(errno, "fscanf");
}
if (ret == EOF)
break;
/*
* If we're unlucky, the last line may be a partially
* written key at the end that can result in a false
* negative error for a missing record. Detect it.
*/
if (last_key != UINT64_MAX && key != last_key + 1) {
printf("%s: Ignore partial record %" PRIu64
" last valid key %" PRIu64 "\n",
fname, key, last_key);
break;
}
testutil_check(__wt_snprintf(
kname, sizeof(kname), "%" PRIu64, key));
cur_coll->set_key(cur_coll, kname);
cur_local->set_key(cur_local, kname);
cur_oplog->set_key(cur_oplog, kname);
/*
* The collection table should always only have the
* data as of the checkpoint.
*/
if ((ret = cur_coll->search(cur_coll)) != 0) {
if (ret != WT_NOTFOUND)
testutil_die(ret, "search");
/*
* If we don't find a record, the stable
* timestamp written to our file better be
* larger than the saved one.
*/
if (!inmem &&
stable_fp != 0 && stable_fp <= val[i]) {
printf("%s: COLLECTION no record with "
"key %" PRIu64 " record ts %" PRIu64
" <= stable ts %" PRIu64 "\n",
fname, key, stable_fp, val[i]);
absent_coll++;
}
if (middle_coll == 0)
first_miss = key;
middle_coll = key;
} else if (middle_coll != 0) {
/*
* We should never find an existing key after
* we have detected one missing.
*/
printf("%s: COLLECTION after absent records %"
PRIu64 "-%" PRIu64 " key %" PRIu64
" exists\n",
fname, first_miss, middle_coll, key);
fatal = true;
}
/*
* The local table should always have all data.
*/
if ((ret = cur_local->search(cur_local)) != 0) {
if (ret != WT_NOTFOUND)
testutil_die(ret, "search");
if (!inmem)
printf("%s: LOCAL no record with key %"
PRIu64 "\n", fname, key);
absent_local++;
middle_local = key;
} else if (middle_local != 0) {
/*
* We should never find an existing key after
* we have detected one missing.
*/
printf("%s: LOCAL after absent record at %"
PRIu64 " key %" PRIu64 " exists\n",
fname, middle_local, key);
fatal = true;
}
/*
* The oplog table should always have all data.
*/
if ((ret = cur_oplog->search(cur_oplog)) != 0) {
if (ret != WT_NOTFOUND)
testutil_die(ret, "search");
if (!inmem)
printf("%s: OPLOG no record with key %"
PRIu64 "\n", fname, key);
absent_oplog++;
middle_oplog = key;
} else if (middle_oplog != 0) {
/*
* We should never find an existing key after
* we have detected one missing.
*/
printf("%s: OPLOG after absent record at %"
PRIu64 " key %" PRIu64 " exists\n",
fname, middle_oplog, key);
fatal = true;
}
}
testutil_checksys(fclose(fp) != 0);
}
if ((ret = conn->close(conn, NULL)) != 0)
testutil_die(ret, "WT_CONNECTION:close");
if (fatal)
return (EXIT_FAILURE);
if (!inmem && absent_coll) {
printf("COLLECTION: %" PRIu64
" record(s) absent from %" PRIu64 "\n",
absent_coll, count);
fatal = true;
}
if (!inmem && absent_local) {
printf("LOCAL: %" PRIu64 " record(s) absent from %" PRIu64 "\n",
absent_local, count);
fatal = true;
}
if (!inmem && absent_oplog) {
printf("OPLOG: %" PRIu64 " record(s) absent from %" PRIu64 "\n",
absent_oplog, count);
fatal = true;
}
if (fatal)
return (EXIT_FAILURE);
printf("%" PRIu64 " records verified\n", count);
return (EXIT_SUCCESS);
}
/*
* thread_run --
* Runner function for the worker threads.
*/
static WT_THREAD_RET
thread_run(void *arg)
{
FILE *fp;
WT_CURSOR *cur_coll, *cur_local, *cur_oplog, *cur_stable;
WT_ITEM data;
WT_RAND_STATE rnd;
WT_SESSION *session;
WT_THREAD_DATA *td;
uint64_t i, stable_ts;
int ret;
char cbuf[MAX_VAL], lbuf[MAX_VAL], obuf[MAX_VAL];
char kname[64], tscfg[64];
__wt_random_init(&rnd);
memset(cbuf, 0, sizeof(cbuf));
memset(lbuf, 0, sizeof(lbuf));
memset(obuf, 0, sizeof(obuf));
memset(kname, 0, sizeof(kname));
td = (WT_THREAD_DATA *)arg;
/*
* Set up the separate file for checking.
*/
testutil_check(__wt_snprintf(cbuf, sizeof(cbuf), RECORDS_FILE, td->id));
(void)unlink(cbuf);
testutil_checksys((fp = fopen(cbuf, "w")) == NULL);
/*
* Set to line buffering. But that is advisory only. We've seen
* cases where the result files end up with partial lines.
*/
__wt_stream_set_line_buffer(fp);
if ((ret = td->conn->open_session(td->conn, NULL, NULL, &session)) != 0)
testutil_die(ret, "WT_CONNECTION:open_session");
/*
* Open a cursor to each table.
*/
if ((ret = session->open_cursor(session,
uri_collection, NULL, NULL, &cur_coll)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_collection);
if ((ret = session->open_cursor(session,
uri_local, NULL, NULL, &cur_local)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_local);
if ((ret = session->open_cursor(session,
uri_oplog, NULL, NULL, &cur_oplog)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_oplog);
if ((ret = session->open_cursor(
session, stable_store, NULL, NULL, &cur_stable)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", stable_store);
/*
* Write our portion of the key space until we're killed.
*/
printf("Thread %" PRIu32 " starts at %" PRIu64 "\n", td->id, td->start);
for (i = td->start; ; ++i) {
if (use_ts)
stable_ts = global_ts++;
else
stable_ts = 0;
testutil_check(__wt_snprintf(
kname, sizeof(kname), "%" PRIu64, i));
testutil_check(session->begin_transaction(session, NULL));
cur_coll->set_key(cur_coll, kname);
cur_local->set_key(cur_local, kname);
cur_oplog->set_key(cur_oplog, kname);
/*
* Put an informative string into the value so that it
* can be viewed well in a binary dump.
*/
testutil_check(__wt_snprintf(cbuf, sizeof(cbuf),
"COLL: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
td->id, stable_ts, i));
testutil_check(__wt_snprintf(lbuf, sizeof(lbuf),
"LOCAL: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
td->id, stable_ts, i));
testutil_check(__wt_snprintf(obuf, sizeof(obuf),
"OPLOG: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
td->id, stable_ts, i));
data.size = __wt_random(&rnd) % MAX_VAL;
data.data = cbuf;
cur_coll->set_value(cur_coll, &data);
if ((ret = cur_coll->insert(cur_coll)) != 0)
testutil_die(ret, "WT_CURSOR.insert");
data.size = __wt_random(&rnd) % MAX_VAL;
data.data = obuf;
cur_oplog->set_value(cur_oplog, &data);
if ((ret = cur_oplog->insert(cur_oplog)) != 0)
testutil_die(ret, "WT_CURSOR.insert");
if (use_ts) {
testutil_check(__wt_snprintf(tscfg, sizeof(tscfg),
"commit_timestamp=%" PRIx64, stable_ts));
testutil_check(
session->commit_transaction(session, tscfg));
} else
testutil_check(
session->commit_transaction(session, NULL));
/*
* Insert into the local table outside the timestamp txn.
*/
data.size = __wt_random(&rnd) % MAX_VAL;
data.data = lbuf;
cur_local->set_value(cur_local, &data);
if ((ret = cur_local->insert(cur_local)) != 0)
testutil_die(ret, "WT_CURSOR.insert");
/*
* Every N records we will record our stable timestamp into the
* stable table. That will define our threshold where we
* expect to find records after recovery.
*/
if (i % STABLE_PERIOD == 0) {
if (use_ts) {
/*
* Set both the oldest and stable timestamp
* so that we don't need to maintain read
* availability at older timestamps.
*/
testutil_check(__wt_snprintf(
tscfg, sizeof(tscfg),
"oldest_timestamp=%" PRIx64
",stable_timestamp=%" PRIx64,
stable_ts, stable_ts));
testutil_check(
td->conn->set_timestamp(td->conn, tscfg));
}
cur_stable->set_key(cur_stable, td->id);
cur_stable->set_value(cur_stable, stable_ts);
testutil_check(cur_stable->insert(cur_stable));
}
/*
* Save the timestamp and key separately for checking later.
*/
if (fprintf(fp,
"%" PRIu64 " %" PRIu64 "\n", stable_ts, i) < 0)
testutil_die(EIO, "fprintf");
}
/* NOTREACHED */
}
int
main(int argc, char *argv[])
{
struct sigaction sa;
struct stat sb;
FILE *fp;
REPORT c_rep[MAX_TH], l_rep[MAX_TH], o_rep[MAX_TH];
WT_CONNECTION *conn;
WT_CURSOR *cur_coll, *cur_local, *cur_oplog;
WT_RAND_STATE rnd;
WT_SESSION *session;
pid_t pid;
uint64_t absent_coll, absent_local, absent_oplog, count, key, last_key;
uint64_t stable_fp, stable_val;
uint32_t i, nth, timeout;
int ch, status, ret;
const char *working_dir;
char buf[512], fname[64], kname[64], statname[1024];
char ts_string[WT_TS_HEX_STRING_SIZE];
bool fatal, rand_th, rand_time, verify_only;
(void)testutil_set_progname(argv);
compat = inmem = false;
use_ts = true;
nth = MIN_TH;
rand_th = rand_time = true;
timeout = MIN_TIME;
verify_only = false;
working_dir = "WT_TEST.timestamp-abort";
while ((ch = __wt_getopt(progname, argc, argv, "Ch:LmT:t:vz")) != EOF)
switch (ch) {
case 'C':
compat = true;
break;
case 'h':
working_dir = __wt_optarg;
break;
case 'L':
table_pfx = "lsm";
break;
case 'm':
inmem = true;
break;
case 'T':
rand_th = false;
nth = (uint32_t)atoi(__wt_optarg);
if (nth > MAX_TH) {
fprintf(stderr,
"Number of threads is larger than the"
" maximum %" PRId32 "\n", MAX_TH);
return (EXIT_FAILURE);
}
break;
case 't':
rand_time = false;
timeout = (uint32_t)atoi(__wt_optarg);
break;
case 'v':
verify_only = true;
break;
case 'z':
use_ts = false;
break;
default:
usage();
}
argc -= __wt_optind;
if (argc != 0)
usage();
testutil_work_dir_from_path(home, sizeof(home), working_dir);
testutil_check(pthread_rwlock_init(&ts_lock, NULL));
/*
* If the user wants to verify they need to tell us how many threads
* there were so we can find the old record files.
*/
if (verify_only && rand_th) {
fprintf(stderr,
"Verify option requires specifying number of threads\n");
exit (EXIT_FAILURE);
}
if (!verify_only) {
testutil_make_work_dir(home);
__wt_random_init_seed(NULL, &rnd);
if (rand_time) {
timeout = __wt_random(&rnd) % MAX_TIME;
if (timeout < MIN_TIME)
timeout = MIN_TIME;
}
if (rand_th) {
nth = __wt_random(&rnd) % MAX_TH;
if (nth < MIN_TH)
nth = MIN_TH;
}
printf("Parent: compatibility: %s, "
"in-mem log sync: %s, timestamp in use: %s\n",
compat ? "true" : "false",
inmem ? "true" : "false",
use_ts ? "true" : "false");
printf("Parent: Create %" PRIu32
" threads; sleep %" PRIu32 " seconds\n", nth, timeout);
printf("CONFIG: %s%s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n",
progname,
compat ? " -C" : "",
inmem ? " -m" : "",
!use_ts ? " -z" : "",
working_dir, nth, timeout);
/*
* Fork a child to insert as many items. We will then randomly
* kill the child, run recovery and make sure all items we wrote
* exist after recovery runs.
*/
memset(&sa, 0, sizeof(sa));
sa.sa_handler = handler;
testutil_checksys(sigaction(SIGCHLD, &sa, NULL));
testutil_checksys((pid = fork()) < 0);
if (pid == 0) { /* child */
run_workload(nth);
return (EXIT_SUCCESS);
}
/* parent */
/*
* Sleep for the configured amount of time before killing
* the child. Start the timeout from the time we notice that
* the file has been created. That allows the test to run
* correctly on really slow machines.
*/
testutil_check(__wt_snprintf(
statname, sizeof(statname), "%s/%s", home, ckpt_file));
while (stat(statname, &sb) != 0)
testutil_sleep_wait(1, pid);
sleep(timeout);
sa.sa_handler = SIG_DFL;
testutil_checksys(sigaction(SIGCHLD, &sa, NULL));
/*
* !!! It should be plenty long enough to make sure more than
* one log file exists. If wanted, that check would be added
* here.
*/
printf("Kill child\n");
testutil_checksys(kill(pid, SIGKILL) != 0);
testutil_checksys(waitpid(pid, &status, 0) == -1);
}
/*
* !!! If we wanted to take a copy of the directory before recovery,
* this is the place to do it. Don't do it all the time because
* it can use a lot of disk space, which can cause test machine
* issues.
*/
if (chdir(home) != 0)
testutil_die(errno, "parent chdir: %s", home);
/*
* The tables can get very large, so while we'd ideally like to
* copy the entire database, we only copy the log files for now.
* Otherwise it can take far too long to run the test, particularly
* in automated testing.
*/
testutil_check(__wt_snprintf(buf, sizeof(buf),
"rm -rf ../%s.SAVE && mkdir ../%s.SAVE && "
"cp -p * ../%s.SAVE",
home, home, home));
if ((status = system(buf)) < 0)
testutil_die(status, "system: %s", buf);
printf("Open database, run recovery and verify content\n");
/*
* Open the connection which forces recovery to be run.
*/
testutil_check(wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn));
testutil_check(conn->open_session(conn, NULL, NULL, &session));
/*
* Open a cursor on all the tables.
*/
testutil_check(__wt_snprintf(
buf, sizeof(buf), "%s:%s", table_pfx, uri_collection));
testutil_check(session->open_cursor(session,
buf, NULL, NULL, &cur_coll));
testutil_check(__wt_snprintf(
buf, sizeof(buf), "%s:%s", table_pfx, uri_local));
testutil_check(session->open_cursor(session,
buf, NULL, NULL, &cur_local));
testutil_check(__wt_snprintf(
buf, sizeof(buf), "%s:%s", table_pfx, uri_oplog));
testutil_check(session->open_cursor(session,
buf, NULL, NULL, &cur_oplog));
/*
* Find the biggest stable timestamp value that was saved.
*/
stable_val = 0;
if (use_ts) {
testutil_check(
conn->query_timestamp(conn, ts_string, "get=recovery"));
testutil_assert(
sscanf(ts_string, "%" SCNx64, &stable_val) == 1);
printf("Got stable_val %" PRIu64 "\n", stable_val);
}
count = 0;
absent_coll = absent_local = absent_oplog = 0;
fatal = false;
for (i = 0; i < nth; ++i) {
initialize_rep(&c_rep[i]);
initialize_rep(&l_rep[i]);
initialize_rep(&o_rep[i]);
testutil_check(__wt_snprintf(
fname, sizeof(fname), RECORDS_FILE, i));
if ((fp = fopen(fname, "r")) == NULL)
testutil_die(errno, "fopen: %s", fname);
/*
* For every key in the saved file, verify that the key exists
* in the table after recovery. If we're doing in-memory
* log buffering we never expect a record missing in the middle,
* but records may be missing at the end. If we did
* write-no-sync, we expect every key to have been recovered.
*/
for (last_key = INVALID_KEY;; ++count, last_key = key) {
ret = fscanf(fp, "%" SCNu64 "%" SCNu64 "\n",
&stable_fp, &key);
if (last_key == INVALID_KEY) {
c_rep[i].first_key = key;
l_rep[i].first_key = key;
o_rep[i].first_key = key;
}
if (ret != EOF && ret != 2) {
/*
* If we find a partial line, consider it
* like an EOF.
*/
if (ret == 1 || ret == 0)
break;
testutil_die(errno, "fscanf");
}
if (ret == EOF)
break;
/*
* If we're unlucky, the last line may be a partially
* written key at the end that can result in a false
* negative error for a missing record. Detect it.
*/
if (last_key != INVALID_KEY && key != last_key + 1) {
printf("%s: Ignore partial record %" PRIu64
" last valid key %" PRIu64 "\n",
fname, key, last_key);
break;
}
testutil_check(__wt_snprintf(
kname, sizeof(kname), "%" PRIu64, key));
cur_coll->set_key(cur_coll, kname);
cur_local->set_key(cur_local, kname);
cur_oplog->set_key(cur_oplog, kname);
/*
* The collection table should always only have the
* data as of the checkpoint.
*/
if ((ret = cur_coll->search(cur_coll)) != 0) {
if (ret != WT_NOTFOUND)
testutil_die(ret, "search");
/*
* If we don't find a record, the stable
* timestamp written to our file better be
* larger than the saved one.
*/
if (!inmem &&
stable_fp != 0 && stable_fp <= stable_val) {
printf("%s: COLLECTION no record with "
"key %" PRIu64 " record ts %" PRIu64
" <= stable ts %" PRIu64 "\n",
fname, key, stable_fp, stable_val);
absent_coll++;
}
if (c_rep[i].first_miss == INVALID_KEY)
c_rep[i].first_miss = key;
c_rep[i].absent_key = key;
} else if (c_rep[i].absent_key != INVALID_KEY &&
c_rep[i].exist_key == INVALID_KEY) {
/*
* If we get here we found a record that exists
* after absent records, a hole in our data.
*/
c_rep[i].exist_key = key;
fatal = true;
} else if (!inmem &&
stable_fp != 0 && stable_fp > stable_val) {
/*
* If we found a record, the stable timestamp
* written to our file better be no larger
* than the checkpoint one.
*/
printf("%s: COLLECTION record with "
"key %" PRIu64 " record ts %" PRIu64
" > stable ts %" PRIu64 "\n",
fname, key, stable_fp, stable_val);
fatal = true;
}
/*
* The local table should always have all data.
*/
if ((ret = cur_local->search(cur_local)) != 0) {
if (ret != WT_NOTFOUND)
testutil_die(ret, "search");
if (!inmem)
printf("%s: LOCAL no record with key %"
PRIu64 "\n", fname, key);
absent_local++;
if (l_rep[i].first_miss == INVALID_KEY)
l_rep[i].first_miss = key;
l_rep[i].absent_key = key;
} else if (l_rep[i].absent_key != INVALID_KEY &&
l_rep[i].exist_key == INVALID_KEY) {
/*
* We should never find an existing key after
* we have detected one missing.
*/
l_rep[i].exist_key = key;
fatal = true;
}
/*
* The oplog table should always have all data.
*/
if ((ret = cur_oplog->search(cur_oplog)) != 0) {
if (ret != WT_NOTFOUND)
testutil_die(ret, "search");
if (!inmem)
printf("%s: OPLOG no record with key %"
PRIu64 "\n", fname, key);
absent_oplog++;
if (o_rep[i].first_miss == INVALID_KEY)
o_rep[i].first_miss = key;
o_rep[i].absent_key = key;
} else if (o_rep[i].absent_key != INVALID_KEY &&
o_rep[i].exist_key == INVALID_KEY) {
/*
* We should never find an existing key after
* we have detected one missing.
*/
o_rep[i].exist_key = key;
fatal = true;
}
}
c_rep[i].last_key = last_key;
l_rep[i].last_key = last_key;
o_rep[i].last_key = last_key;
testutil_checksys(fclose(fp) != 0);
print_missing(&c_rep[i], fname, "COLLECTION");
print_missing(&l_rep[i], fname, "LOCAL");
print_missing(&o_rep[i], fname, "OPLOG");
}
testutil_check(conn->close(conn, NULL));
if (!inmem && absent_coll) {
printf("COLLECTION: %" PRIu64
" record(s) absent from %" PRIu64 "\n",
absent_coll, count);
fatal = true;
}
if (!inmem && absent_local) {
printf("LOCAL: %" PRIu64 " record(s) absent from %" PRIu64 "\n",
absent_local, count);
fatal = true;
}
if (!inmem && absent_oplog) {
printf("OPLOG: %" PRIu64 " record(s) absent from %" PRIu64 "\n",
absent_oplog, count);
fatal = true;
}
testutil_check(pthread_rwlock_destroy(&ts_lock));
if (fatal)
return (EXIT_FAILURE);
printf("%" PRIu64 " records verified\n", count);
return (EXIT_SUCCESS);
}
/*
* thread_run --
* Runner function for the worker threads.
*/
static WT_THREAD_RET
thread_run(void *arg)
{
FILE *fp;
WT_CURSOR *cur_coll, *cur_local, *cur_oplog;
WT_ITEM data;
WT_RAND_STATE rnd;
WT_SESSION *prepared_session, *session;
THREAD_DATA *td;
uint64_t i, active_ts;
char cbuf[MAX_VAL], lbuf[MAX_VAL], obuf[MAX_VAL];
char kname[64], tscfg[64], uri[128];
bool use_prep;
__wt_random_init(&rnd);
memset(cbuf, 0, sizeof(cbuf));
memset(lbuf, 0, sizeof(lbuf));
memset(obuf, 0, sizeof(obuf));
memset(kname, 0, sizeof(kname));
prepared_session = NULL;
td = (THREAD_DATA *)arg;
/*
* Set up the separate file for checking.
*/
testutil_check(__wt_snprintf(
cbuf, sizeof(cbuf), RECORDS_FILE, td->info));
(void)unlink(cbuf);
testutil_checksys((fp = fopen(cbuf, "w")) == NULL);
/*
* Set to line buffering. But that is advisory only. We've seen
* cases where the result files end up with partial lines.
*/
__wt_stream_set_line_buffer(fp);
/*
* Have 10% of the threads use prepared transactions if timestamps
* are in use. Thread numbers start at 0 so we're always guaranteed
* that at least one thread is using prepared transactions.
*/
use_prep = (use_ts && td->info % PREPARE_PCT == 0) ? true : false;
/*
* For the prepared case we have two sessions so that the oplog session
* can have its own transaction in parallel with the collection session
* We need this because prepared transactions cannot have any operations
* that modify a table that is logged. But we also want to test mixed
* logged and not-logged transactions.
*/
testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session));
if (use_prep)
testutil_check(td->conn->open_session(
td->conn, NULL, NULL, &prepared_session));
/*
* Open a cursor to each table.
*/
testutil_check(__wt_snprintf(
uri, sizeof(uri), "%s:%s", table_pfx, uri_collection));
if (use_prep)
testutil_check(prepared_session->open_cursor(prepared_session,
uri, NULL, NULL, &cur_coll));
else
testutil_check(session->open_cursor(session,
uri, NULL, NULL, &cur_coll));
testutil_check(__wt_snprintf(
uri, sizeof(uri), "%s:%s", table_pfx, uri_local));
if (use_prep)
testutil_check(prepared_session->open_cursor(prepared_session,
uri, NULL, NULL, &cur_local));
else
testutil_check(session->open_cursor(session,
uri, NULL, NULL, &cur_local));
testutil_check(__wt_snprintf(
uri, sizeof(uri), "%s:%s", table_pfx, uri_oplog));
testutil_check(session->open_cursor(session,
uri, NULL, NULL, &cur_oplog));
/*
* Write our portion of the key space until we're killed.
*/
printf("Thread %" PRIu32 " starts at %" PRIu64 "\n",
td->info, td->start);
active_ts = 0;
for (i = td->start;; ++i) {
testutil_check(__wt_snprintf(
kname, sizeof(kname), "%" PRIu64, i));
testutil_check(session->begin_transaction(session, NULL));
if (use_prep)
testutil_check(prepared_session->begin_transaction(
prepared_session, NULL));
if (use_ts) {
testutil_check(pthread_rwlock_rdlock(&ts_lock));
active_ts = __wt_atomic_addv64(&global_ts, 1);
testutil_check(__wt_snprintf(tscfg,
sizeof(tscfg), "commit_timestamp=%" PRIx64,
active_ts));
/*
* Set the transaction's timestamp now before performing
* the operation. If we are using prepared transactions,
* set the timestamp for the session used for oplog. The
* collection session in that case would continue to use
* this timestamp.
*/
testutil_check(session->timestamp_transaction(
session, tscfg));
testutil_check(pthread_rwlock_unlock(&ts_lock));
}
cur_coll->set_key(cur_coll, kname);
cur_local->set_key(cur_local, kname);
cur_oplog->set_key(cur_oplog, kname);
/*
* Put an informative string into the value so that it
* can be viewed well in a binary dump.
*/
testutil_check(__wt_snprintf(cbuf, sizeof(cbuf),
"COLL: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
td->info, active_ts, i));
testutil_check(__wt_snprintf(lbuf, sizeof(lbuf),
"LOCAL: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
td->info, active_ts, i));
testutil_check(__wt_snprintf(obuf, sizeof(obuf),
"OPLOG: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
td->info, active_ts, i));
data.size = __wt_random(&rnd) % MAX_VAL;
data.data = cbuf;
cur_coll->set_value(cur_coll, &data);
testutil_check(cur_coll->insert(cur_coll));
data.size = __wt_random(&rnd) % MAX_VAL;
data.data = obuf;
cur_oplog->set_value(cur_oplog, &data);
testutil_check(cur_oplog->insert(cur_oplog));
if (use_prep) {
/*
* Run with prepare every once in a while. And also
* yield after prepare sometimes too. This is only done
* on the collection session.
*/
if (i % PREPARE_FREQ == 0) {
testutil_check(__wt_snprintf(tscfg,
sizeof(tscfg), "prepare_timestamp=%"
PRIx64, active_ts));
testutil_check(
prepared_session->prepare_transaction(
prepared_session, tscfg));
if (i % PREPARE_YIELD == 0)
__wt_yield();
testutil_check(
__wt_snprintf(tscfg, sizeof(tscfg),
"commit_timestamp=%" PRIx64
",durable_timestamp=%" PRIx64,
active_ts, active_ts));
} else
testutil_check(
__wt_snprintf(tscfg, sizeof(tscfg),
"commit_timestamp=%" PRIx64, active_ts));
testutil_check(
prepared_session->commit_transaction(
prepared_session, tscfg));
}
testutil_check(
session->commit_transaction(session, NULL));
/*
* Insert into the local table outside the timestamp txn.
*/
data.size = __wt_random(&rnd) % MAX_VAL;
data.data = lbuf;
cur_local->set_value(cur_local, &data);
testutil_check(cur_local->insert(cur_local));
/*
* Save the timestamp and key separately for checking later.
*/
if (fprintf(fp,
"%" PRIu64 " %" PRIu64 "\n", active_ts, i) < 0)
testutil_die(EIO, "fprintf");
}
/* NOTREACHED */
}
