static int
compare_int_items(WT_ITEM *itema, WT_ITEM *itemb)
{
testutil_assert(itema->size == sizeof(int32_t));
testutil_assert(itemb->size == sizeof(int32_t));
return (compare_int(item_to_int(itema), item_to_int(itemb)));
}
/*
* verify_metadata --
* Verify all the tables expected are in the metadata. We expect all but
* the "corrupt" table name.
*/
static void
verify_metadata(WT_CONNECTION *conn, TABLE_INFO *tables)
{
TABLE_INFO *t;
WT_CURSOR *cursor;
int ret;
const char *kv;
/*
* Open a metadata cursor.
*/
testutil_check(conn->open_session(conn, NULL, NULL, &wt_session));
testutil_check(wt_session->open_cursor(
wt_session, "metadata:", NULL, NULL, &cursor));
reset_verified(tables);
/*
* We have to walk the cursor and walk the tables to match up that
* the expected tables are in the metadata. It is not efficient, but
* the list of tables is small. Walk the cursor once and the array
* of tables each time.
*/
while ((ret = cursor->next(cursor)) == 0) {
testutil_check(cursor->get_key(cursor, &kv));
for (t = tables; t->name != NULL; t++) {
if (strcmp(t->name, kv) == 0) {
testutil_assert(t->verified == false);
t->verified = true;
break;
}
}
}
testutil_assert(ret == WT_NOTFOUND);
testutil_check(cursor->close(cursor));
/*
* Any tables that were salvaged, make sure we can read the data.
* The corrupt table should never be salvaged.
*/
for (t = tables; t->name != NULL; t++) {
if (strcmp(t->name, CORRUPT) == 0 && !test_out_of_sync)
testutil_assert(t->verified == false);
else if (t->verified != true)
printf("%s not seen in metadata\n", t->name);
else {
testutil_check(wt_session->open_cursor(
wt_session, t->name, NULL, NULL, &cursor));
while ((ret = cursor->next(cursor)) == 0) {
testutil_check(cursor->get_value(cursor, &kv));
testutil_assert(strcmp(kv, VALUE) == 0);
}
testutil_assert(ret == WT_NOTFOUND);
testutil_check(cursor->close(cursor));
printf("%s metadata salvaged and data verified\n",
t->name);
}
}
}
/*
* subtest_main --
* The main program for the subtest
*/
static void
subtest_main(int argc, char *argv[], bool close_test)
{
struct rlimit rlim;
TEST_OPTS *opts, _opts;
WT_SESSION *session;
char config[1024], filename[1024];
opts = &_opts;
memset(opts, 0, sizeof(*opts));
memset(&rlim, 0, sizeof(rlim));
/* No core files during fault injection tests. */
testutil_check(setrlimit(RLIMIT_CORE, &rlim));
testutil_check(testutil_parse_opts(argc, argv, opts));
testutil_make_work_dir(opts->home);
/* Redirect stderr, stdout. */
testutil_check(__wt_snprintf(
filename, sizeof(filename), "%s/%s", opts->home, STDERR_FILE));
testutil_assert(freopen(filename, "a", stderr) != NULL);
testutil_check(__wt_snprintf(
filename, sizeof(filename), "%s/%s", opts->home, STDOUT_FILE));
testutil_assert(freopen(filename, "a", stdout) != NULL);
testutil_check(__wt_snprintf(config, sizeof(config),
"create,cache_size=250M,log=(enabled),"
"transaction_sync=(enabled,method=none),extensions=("
WT_FAIL_FS_LIB
"=(early_load,config={environment=true,verbose=true})]"));
testutil_check(
wiredtiger_open(opts->home, &event_handler, config, &opts->conn));
testutil_check(
opts->conn->open_session(opts->conn, NULL, NULL, &session));
testutil_check(session->create(session, "table:subtest",
"key_format=i,value_format=iiiS,"
"columns=(id,v0,v1,v2,big)"));
testutil_check(session->create(session, "table:subtest2",
"key_format=i,value_format=i"));
testutil_check(session->create(session, "index:subtest:v0",
"columns=(v0)"));
testutil_check(session->create(session, "index:subtest:v1",
"columns=(v1)"));
testutil_check(session->create(session, "index:subtest:v2",
"columns=(v2)"));
testutil_check(session->close(session, NULL));
subtest_populate(opts, close_test);
testutil_cleanup(opts);
}
/*
* run_check_subtest_range --
*
* Run successive tests via binary search that determines the approximate
* crossover point between when data is recoverable or not. Once that is
* determined, run the subtest in a range near that crossover point.
*
* The theory is that running at the crossover point will tend to trigger
* "interesting" failures at the borderline when the checkpoint is about to,
* or has, succeeded. If any of those failures creates a WT home directory
* that cannot be recovered, the top level test will fail.
*/
static void
run_check_subtest_range(TEST_OPTS *opts, const char *debugger, bool close_test)
{
uint64_t cutoff, high, low, mid, nops, nresults;
int i;
bool got_failure, got_success;
if (opts->verbose)
printf("Determining best range of operations until failure, "
"with close_test %s.\n",
(close_test ? "enabled" : "disabled"));
run_check_subtest(opts, debugger, 1, close_test, &cutoff);
low = 0;
high = MAX_OP_RANGE;
mid = (low + high) / 2;
while (mid != low) {
run_check_subtest(opts, debugger, mid, close_test,
&nresults);
if (nresults > cutoff)
high = mid;
else
low = mid;
mid = (low + high) / 2;
}
/*
* mid is the number of ops that is the crossover point.
* Run some tests near that point to try to trigger weird
* failures. If mid is too low or too high, it indicates
* there is a fundamental problem with the test.
*/
testutil_assert(mid > 1 && mid < MAX_OP_RANGE - 1);
if (opts->verbose)
printf("Retesting around %" PRIu64 " operations.\n",
mid);
got_failure = false;
got_success = false;
for (nops = mid - 10; nops < mid + 10; nops++) {
for (i = 0; i < TESTS_PER_OP_VALUE; i++) {
run_check_subtest(opts, debugger, nops,
close_test, &nresults);
if (nresults > cutoff)
got_failure = true;
else
got_success = true;
}
}
/*
* Check that it really ran with a crossover point.
*/
testutil_assert(got_failure);
testutil_assert(got_success);
}
/*
* check_values --
* Check that the values in the cursor match the given values.
*/
static void
check_values(WT_CURSOR *cursor, int v0, int v1, int v2, char *big)
{
int v0_got, v1_got, v2_got;
char *big_got;
testutil_check(cursor->get_value(cursor, &v0_got, &v1_got, &v2_got,
&big_got));
testutil_assert(v0 == v0_got);
testutil_assert(v1 == v1_got);
testutil_assert(v2 == v2_got);
testutil_assert(strcmp(big, big_got) == 0);
}
static void
wt_open_corrupt(const char *sfx)
{
WT_CONNECTION *conn;
int ret;
char buf[1024];
#ifdef HAVE_ATTACH
WT_UNUSED(buf);
WT_UNUSED(conn);
WT_UNUSED(ret);
WT_UNUSED(sfx);
#else
conn = NULL;
if (sfx != NULL)
testutil_check(__wt_snprintf(buf, sizeof(buf),
"%s.%s", home, sfx));
else
testutil_check(__wt_snprintf(buf, sizeof(buf), "%s", home));
ret = wiredtiger_open(buf, &event_handler, NULL, &conn);
/*
* Not all out of sync combinations lead to corruption. We keep
* the previous checkpoint in the file so some combinations of
* future or old turtle files and metadata files will succeed.
*/
if (ret != WT_TRY_SALVAGE && ret != 0)
fprintf(stderr,
"OPEN_CORRUPT: wiredtiger_open returned %d\n", ret);
testutil_assert(ret == WT_TRY_SALVAGE || ret == 0);
#endif
exit (EXIT_SUCCESS);
}
static void
print_int_item(const char *str, const WT_ITEM *item)
{
if (item->size > 0) {
testutil_assert(item->size == sizeof(int32_t));
printf("%s%" PRId32, str, *(int32_t *)item->data);
} else
printf("%s<empty>", str);
}
/*
* main --
* The main program for the test. When invoked with "subtest"
* argument, run the subtest. Otherwise, run a separate process
* for each needed subtest, and check the results.
*/
int
main(int argc, char *argv[])
{
TEST_OPTS *opts, _opts;
uint64_t nresults;
const char *debugger;
/* Ignore unless requested */
if (!testutil_is_flag_set("TESTUTIL_ENABLE_LONG_TESTS"))
return (EXIT_SUCCESS);
opts = &_opts;
memset(opts, 0, sizeof(*opts));
debugger = NULL;
testutil_check(testutil_parse_opts(argc, argv, opts));
argc -= __wt_optind;
argv += __wt_optind;
if (opts->nrecords == 0)
opts->nrecords = 50000;
while (argc > 0) {
if (strcmp(argv[0], "subtest") == 0) {
subtest_main(argc, argv, false);
return (0);
} else if (strcmp(argv[0], "subtest_close") == 0) {
subtest_main(argc, argv, true);
return (0);
} else if (strcmp(argv[0], "gdb") == 0)
debugger = "/usr/bin/gdb";
else
testutil_assert(false);
argc--;
argv++;
}
if (opts->verbose) {
printf("Number of operations until failure: %" PRIu64
" (change with -o N)\n", opts->nops);
printf("Number of records: %" PRIu64
" (change with -n N)\n", opts->nrecords);
}
if (opts->nops == 0) {
run_check_subtest_range(opts, debugger, false);
run_check_subtest_range(opts, debugger, true);
} else
run_check_subtest(opts, debugger, opts->nops,
opts->nrecords, &nresults);
testutil_clean_work_dir(opts->home);
testutil_cleanup(opts);
return (0);
}
/*
* cursor_count_items --
* Count the number of items in the table by traversing
* through the cursor.
*/
static void
cursor_count_items(WT_CURSOR *cursor, uint64_t *countp)
{
int ret;
*countp = 0;
testutil_check(cursor->reset(cursor));
while ((ret = cursor->next(cursor)) == 0)
(*countp)++;
testutil_assert(ret == WT_NOTFOUND);
}
/*
* thread_ckpt_run --
* Runner function for the checkpoint thread.
*/
static WT_THREAD_RET
thread_ckpt_run(void *arg)
{
FILE *fp;
WT_RAND_STATE rnd;
WT_SESSION *session;
THREAD_DATA *td;
uint64_t stable;
uint32_t sleep_time;
int i;
bool first_ckpt;
char ts_string[WT_TS_HEX_STRING_SIZE];
__wt_random_init(&rnd);
td = (THREAD_DATA *)arg;
/*
* Keep a separate file with the records we wrote for checking.
*/
(void)unlink(ckpt_file);
testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session));
first_ckpt = true;
for (i = 0; ;++i) {
sleep_time = __wt_random(&rnd) % MAX_CKPT_INVL;
sleep(sleep_time);
/*
* Since this is the default, send in this string even if
* running without timestamps.
*/
testutil_check(session->checkpoint(
session, "use_timestamp=true"));
testutil_check(td->conn->query_timestamp(
td->conn, ts_string, "get=last_checkpoint"));
testutil_assert(sscanf(ts_string, "%" SCNx64, &stable) == 1);
printf("Checkpoint %d complete at stable %"
PRIu64 ".\n", i, stable);
fflush(stdout);
/*
* Create the checkpoint file so that the parent process knows
* at least one checkpoint has finished and can start its
* timer.
*/
if (first_ckpt) {
testutil_checksys((fp = fopen(ckpt_file, "w")) == NULL);
first_ckpt = false;
testutil_checksys(fclose(fp) != 0);
}
}
/* NOTREACHED */
}
static void
open_with_salvage(const char *sfx, TABLE_INFO *table_data)
{
WT_CONNECTION *conn;
char buf[1024];
printf("=== wt_open with salvage ===\n");
/*
* Then call wiredtiger_open with the salvage configuration setting.
* That should succeed. We should be able to then verify the contents
* of the metadata file.
*/
test_abort = true;
if (sfx != NULL)
testutil_check(__wt_snprintf(buf, sizeof(buf),
"%s.%s", home, sfx));
else
testutil_check(__wt_snprintf(buf, sizeof(buf), "%s", home));
testutil_check(wiredtiger_open(buf,
&event_handler, "salvage=true", &conn));
testutil_assert(conn != NULL);
if (sfx != NULL)
testutil_check(__wt_snprintf(buf, sizeof(buf),
"%s.%s/%s", home, sfx, WT_METAFILE_SLVG));
else
testutil_check(__wt_snprintf(buf, sizeof(buf),
"%s/%s", home, WT_METAFILE_SLVG));
testutil_assert(file_exists(buf));
/*
* Confirm we salvaged the metadata file by looking for the saved
* copy of the original metadata.
*/
printf("verify with salvaged connection\n");
verify_metadata(conn, &table_data[0]);
testutil_check(conn->close(conn, NULL));
}
/*
* compare --
* Compare two results.
*/
static void
compare(WT_ITEM *local, WT_ITEM *library)
{
#if DEBUG
if (local->size != library->size ||
memcmp(local->data, library->data, local->size) != 0) {
fprintf(stderr, "results differ\n");
show(local, "local results");
show(library, "library results");
}
#endif
testutil_assert(
local->size == library->size && memcmp(
local->data, library->data, local->size) == 0);
}
/*
* run_check_subtest --
* Run the subtest with the given parameters and check the results.
*/
static void
run_check_subtest(TEST_OPTS *opts, const char *debugger, uint64_t nops,
bool close_test, uint64_t *nresultsp)
{
int estatus, narg;
char rarg[20], sarg[20], *subtest_args[MAX_ARGS];
narg = 0;
if (debugger != NULL) {
subtest_args[narg++] = (char *)debugger;
subtest_args[narg++] = (char *)"--";
}
subtest_args[narg++] = (char *)opts->progname;
/* "subtest" must appear before arguments */
if (close_test)
subtest_args[narg++] = (char *)"subtest_close";
else
subtest_args[narg++] = (char *)"subtest";
subtest_args[narg++] = (char *)"-h";
subtest_args[narg++] = opts->home;
subtest_args[narg++] = (char *)"-v"; /* subtest is always verbose */
subtest_args[narg++] = (char *)"-p";
subtest_args[narg++] = (char *)"-o";
testutil_check(__wt_snprintf(sarg, sizeof(sarg), "%" PRIu64, nops));
subtest_args[narg++] = sarg; /* number of operations */
subtest_args[narg++] = (char *)"-n";
testutil_check(__wt_snprintf(
rarg, sizeof(rarg), "%" PRIu64, opts->nrecords));
subtest_args[narg++] = rarg; /* number of records */
subtest_args[narg++] = NULL;
testutil_assert(narg <= MAX_ARGS);
if (opts->verbose)
printf("running a separate process with %" PRIu64
" operations until fail...\n", nops);
testutil_clean_work_dir(opts->home);
testutil_check(run_process(
opts, debugger != NULL ? debugger : opts->progname,
subtest_args, &estatus));
if (opts->verbose)
printf("process exited %d\n", estatus);
/*
* Verify results in parent process.
*/
testutil_check(check_results(opts, nresultsp));
}
/*
* corrupt_metadata --
* Corrupt the metadata by scribbling on the "corrupt" URI string.
*/
static void
corrupt_metadata(void)
{
FILE *fp;
struct stat sb;
long off;
size_t meta_size;
bool corrupted;
uint8_t *buf, *corrupt;
char path[256];
/*
* Open the file, read its contents. Find the string "corrupt" and
* modify one byte at that offset. That will cause a checksum error
* when WiredTiger next reads it.
*/
testutil_check(__wt_snprintf(
path, sizeof(path), "%s/%s", home, WT_METAFILE));
if ((fp = fopen(path, "r+")) == NULL)
testutil_die(errno, "fopen: %s", path);
testutil_check(fstat(fileno(fp), &sb));
meta_size = (size_t)sb.st_size;
buf = dcalloc(meta_size, 1);
if (fread(buf, 1, meta_size, fp) != meta_size)
testutil_die(errno, "fread: %" WT_SIZET_FMT, meta_size);
corrupted = false;
/*
* Corrupt all occurrences of the string in the file.
*/
while ((corrupt = byte_str(buf, meta_size, CORRUPT)) != NULL) {
corrupted = true;
testutil_assert(*(char *)corrupt != 'X');
*(char *)corrupt = 'X';
off = (long)(corrupt - buf);
if (fseek(fp, off, SEEK_SET) != 0)
testutil_die(errno, "fseek: %ld", off);
if (fwrite("X", 1, 1, fp) != 1)
testutil_die(errno, "fwrite");
}
if (!corrupted)
testutil_die(errno, "corrupt string did not occur");
if (fclose(fp) != 0)
testutil_die(errno, "fclose");
free(buf);
}
/*
* thread_ts_run --
* Runner function for a timestamp thread.
*/
static WT_THREAD_RET
thread_ts_run(void *arg)
{
WT_DECL_RET;
WT_SESSION *session;
THREAD_DATA *td;
char tscfg[64], ts_string[WT_TS_HEX_STRING_SIZE];
td = (THREAD_DATA *)arg;
testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session));
/* Update the oldest timestamp every 1 millisecond. */
for (;;) {
/*
* We get the last committed timestamp periodically in order to
* update the oldest timestamp, that requires locking out
* transactional ops that set or query a timestamp.
*/
testutil_check(pthread_rwlock_wrlock(&ts_lock));
ret = td->conn->query_timestamp(
td->conn, ts_string, "get=all_committed");
testutil_check(pthread_rwlock_unlock(&ts_lock));
testutil_assert(ret == 0 || ret == WT_NOTFOUND);
if (ret == 0) {
/*
* Set both the oldest and stable timestamp so that we
* don't need to maintain read availability at older
* timestamps.
*/
testutil_check(__wt_snprintf(
tscfg, sizeof(tscfg),
"oldest_timestamp=%s,stable_timestamp=%s",
ts_string, ts_string));
testutil_check(
td->conn->set_timestamp(td->conn, tscfg));
}
__wt_sleep(0, 1000);
}
/* NOTREACHED */
}
int
main(int argc, char *argv[])
{
struct sigaction sa;
struct stat sb;
FILE *fp;
REPORT c_rep[MAX_TH], l_rep[MAX_TH], o_rep[MAX_TH];
WT_CONNECTION *conn;
WT_CURSOR *cur_coll, *cur_local, *cur_oplog;
WT_RAND_STATE rnd;
WT_SESSION *session;
pid_t pid;
uint64_t absent_coll, absent_local, absent_oplog, count, key, last_key;
uint64_t stable_fp, stable_val;
uint32_t i, nth, timeout;
int ch, status, ret;
const char *working_dir;
char buf[512], fname[64], kname[64], statname[1024];
char ts_string[WT_TS_HEX_STRING_SIZE];
bool fatal, rand_th, rand_time, verify_only;
(void)testutil_set_progname(argv);
compat = inmem = false;
use_ts = true;
nth = MIN_TH;
rand_th = rand_time = true;
timeout = MIN_TIME;
verify_only = false;
working_dir = "WT_TEST.timestamp-abort";
while ((ch = __wt_getopt(progname, argc, argv, "Ch:LmT:t:vz")) != EOF)
switch (ch) {
case 'C':
compat = true;
break;
case 'h':
working_dir = __wt_optarg;
break;
case 'L':
table_pfx = "lsm";
break;
case 'm':
inmem = true;
break;
case 'T':
rand_th = false;
nth = (uint32_t)atoi(__wt_optarg);
if (nth > MAX_TH) {
fprintf(stderr,
"Number of threads is larger than the"
" maximum %" PRId32 "\n", MAX_TH);
return (EXIT_FAILURE);
}
break;
case 't':
rand_time = false;
timeout = (uint32_t)atoi(__wt_optarg);
break;
case 'v':
verify_only = true;
break;
case 'z':
use_ts = false;
break;
default:
usage();
}
argc -= __wt_optind;
if (argc != 0)
usage();
testutil_work_dir_from_path(home, sizeof(home), working_dir);
testutil_check(pthread_rwlock_init(&ts_lock, NULL));
/*
* If the user wants to verify they need to tell us how many threads
* there were so we can find the old record files.
*/
if (verify_only && rand_th) {
fprintf(stderr,
"Verify option requires specifying number of threads\n");
exit (EXIT_FAILURE);
}
if (!verify_only) {
testutil_make_work_dir(home);
__wt_random_init_seed(NULL, &rnd);
if (rand_time) {
timeout = __wt_random(&rnd) % MAX_TIME;
if (timeout < MIN_TIME)
timeout = MIN_TIME;
}
if (rand_th) {
nth = __wt_random(&rnd) % MAX_TH;
if (nth < MIN_TH)
nth = MIN_TH;
}
printf("Parent: compatibility: %s, "
"in-mem log sync: %s, timestamp in use: %s\n",
compat ? "true" : "false",
inmem ? "true" : "false",
use_ts ? "true" : "false");
printf("Parent: Create %" PRIu32
" threads; sleep %" PRIu32 " seconds\n", nth, timeout);
printf("CONFIG: %s%s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n",
progname,
compat ? " -C" : "",
inmem ? " -m" : "",
!use_ts ? " -z" : "",
working_dir, nth, timeout);
/*
* Fork a child to insert as many items. We will then randomly
* kill the child, run recovery and make sure all items we wrote
* exist after recovery runs.
*/
memset(&sa, 0, sizeof(sa));
sa.sa_handler = handler;
testutil_checksys(sigaction(SIGCHLD, &sa, NULL));
testutil_checksys((pid = fork()) < 0);
if (pid == 0) { /* child */
run_workload(nth);
return (EXIT_SUCCESS);
}
/* parent */
/*
* Sleep for the configured amount of time before killing
* the child. Start the timeout from the time we notice that
* the file has been created. That allows the test to run
* correctly on really slow machines.
*/
testutil_check(__wt_snprintf(
statname, sizeof(statname), "%s/%s", home, ckpt_file));
while (stat(statname, &sb) != 0)
testutil_sleep_wait(1, pid);
sleep(timeout);
sa.sa_handler = SIG_DFL;
testutil_checksys(sigaction(SIGCHLD, &sa, NULL));
/*
* !!! It should be plenty long enough to make sure more than
* one log file exists. If wanted, that check would be added
* here.
*/
printf("Kill child\n");
testutil_checksys(kill(pid, SIGKILL) != 0);
testutil_checksys(waitpid(pid, &status, 0) == -1);
}
/*
* !!! If we wanted to take a copy of the directory before recovery,
* this is the place to do it. Don't do it all the time because
* it can use a lot of disk space, which can cause test machine
* issues.
*/
if (chdir(home) != 0)
testutil_die(errno, "parent chdir: %s", home);
/*
* The tables can get very large, so while we'd ideally like to
* copy the entire database, we only copy the log files for now.
* Otherwise it can take far too long to run the test, particularly
* in automated testing.
*/
testutil_check(__wt_snprintf(buf, sizeof(buf),
"rm -rf ../%s.SAVE && mkdir ../%s.SAVE && "
"cp -p * ../%s.SAVE",
home, home, home));
if ((status = system(buf)) < 0)
testutil_die(status, "system: %s", buf);
printf("Open database, run recovery and verify content\n");
/*
* Open the connection which forces recovery to be run.
*/
testutil_check(wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn));
testutil_check(conn->open_session(conn, NULL, NULL, &session));
/*
* Open a cursor on all the tables.
*/
testutil_check(__wt_snprintf(
buf, sizeof(buf), "%s:%s", table_pfx, uri_collection));
testutil_check(session->open_cursor(session,
buf, NULL, NULL, &cur_coll));
testutil_check(__wt_snprintf(
buf, sizeof(buf), "%s:%s", table_pfx, uri_local));
testutil_check(session->open_cursor(session,
buf, NULL, NULL, &cur_local));
testutil_check(__wt_snprintf(
buf, sizeof(buf), "%s:%s", table_pfx, uri_oplog));
testutil_check(session->open_cursor(session,
buf, NULL, NULL, &cur_oplog));
/*
* Find the biggest stable timestamp value that was saved.
*/
stable_val = 0;
if (use_ts) {
testutil_check(
conn->query_timestamp(conn, ts_string, "get=recovery"));
testutil_assert(
sscanf(ts_string, "%" SCNx64, &stable_val) == 1);
printf("Got stable_val %" PRIu64 "\n", stable_val);
}
count = 0;
absent_coll = absent_local = absent_oplog = 0;
fatal = false;
for (i = 0; i < nth; ++i) {
initialize_rep(&c_rep[i]);
initialize_rep(&l_rep[i]);
initialize_rep(&o_rep[i]);
testutil_check(__wt_snprintf(
fname, sizeof(fname), RECORDS_FILE, i));
if ((fp = fopen(fname, "r")) == NULL)
testutil_die(errno, "fopen: %s", fname);
/*
* For every key in the saved file, verify that the key exists
* in the table after recovery. If we're doing in-memory
* log buffering we never expect a record missing in the middle,
* but records may be missing at the end. If we did
* write-no-sync, we expect every key to have been recovered.
*/
for (last_key = INVALID_KEY;; ++count, last_key = key) {
ret = fscanf(fp, "%" SCNu64 "%" SCNu64 "\n",
&stable_fp, &key);
if (last_key == INVALID_KEY) {
c_rep[i].first_key = key;
l_rep[i].first_key = key;
o_rep[i].first_key = key;
}
if (ret != EOF && ret != 2) {
/*
* If we find a partial line, consider it
* like an EOF.
*/
if (ret == 1 || ret == 0)
break;
testutil_die(errno, "fscanf");
}
if (ret == EOF)
break;
/*
* If we're unlucky, the last line may be a partially
* written key at the end that can result in a false
* negative error for a missing record. Detect it.
*/
if (last_key != INVALID_KEY && key != last_key + 1) {
printf("%s: Ignore partial record %" PRIu64
" last valid key %" PRIu64 "\n",
fname, key, last_key);
break;
}
testutil_check(__wt_snprintf(
kname, sizeof(kname), "%" PRIu64, key));
cur_coll->set_key(cur_coll, kname);
cur_local->set_key(cur_local, kname);
cur_oplog->set_key(cur_oplog, kname);
/*
* The collection table should always only have the
* data as of the checkpoint.
*/
if ((ret = cur_coll->search(cur_coll)) != 0) {
if (ret != WT_NOTFOUND)
testutil_die(ret, "search");
/*
* If we don't find a record, the stable
* timestamp written to our file better be
* larger than the saved one.
*/
if (!inmem &&
stable_fp != 0 && stable_fp <= stable_val) {
printf("%s: COLLECTION no record with "
"key %" PRIu64 " record ts %" PRIu64
" <= stable ts %" PRIu64 "\n",
fname, key, stable_fp, stable_val);
absent_coll++;
}
if (c_rep[i].first_miss == INVALID_KEY)
c_rep[i].first_miss = key;
c_rep[i].absent_key = key;
} else if (c_rep[i].absent_key != INVALID_KEY &&
c_rep[i].exist_key == INVALID_KEY) {
/*
* If we get here we found a record that exists
* after absent records, a hole in our data.
*/
c_rep[i].exist_key = key;
fatal = true;
} else if (!inmem &&
stable_fp != 0 && stable_fp > stable_val) {
/*
* If we found a record, the stable timestamp
* written to our file better be no larger
* than the checkpoint one.
*/
printf("%s: COLLECTION record with "
"key %" PRIu64 " record ts %" PRIu64
" > stable ts %" PRIu64 "\n",
fname, key, stable_fp, stable_val);
fatal = true;
}
/*
* The local table should always have all data.
*/
if ((ret = cur_local->search(cur_local)) != 0) {
if (ret != WT_NOTFOUND)
testutil_die(ret, "search");
if (!inmem)
printf("%s: LOCAL no record with key %"
PRIu64 "\n", fname, key);
absent_local++;
if (l_rep[i].first_miss == INVALID_KEY)
l_rep[i].first_miss = key;
l_rep[i].absent_key = key;
} else if (l_rep[i].absent_key != INVALID_KEY &&
l_rep[i].exist_key == INVALID_KEY) {
/*
* We should never find an existing key after
* we have detected one missing.
*/
l_rep[i].exist_key = key;
fatal = true;
}
/*
* The oplog table should always have all data.
*/
if ((ret = cur_oplog->search(cur_oplog)) != 0) {
if (ret != WT_NOTFOUND)
testutil_die(ret, "search");
if (!inmem)
printf("%s: OPLOG no record with key %"
PRIu64 "\n", fname, key);
absent_oplog++;
if (o_rep[i].first_miss == INVALID_KEY)
o_rep[i].first_miss = key;
o_rep[i].absent_key = key;
} else if (o_rep[i].absent_key != INVALID_KEY &&
o_rep[i].exist_key == INVALID_KEY) {
/*
* We should never find an existing key after
* we have detected one missing.
*/
o_rep[i].exist_key = key;
fatal = true;
}
}
c_rep[i].last_key = last_key;
l_rep[i].last_key = last_key;
o_rep[i].last_key = last_key;
testutil_checksys(fclose(fp) != 0);
print_missing(&c_rep[i], fname, "COLLECTION");
print_missing(&l_rep[i], fname, "LOCAL");
print_missing(&o_rep[i], fname, "OPLOG");
}
testutil_check(conn->close(conn, NULL));
if (!inmem && absent_coll) {
printf("COLLECTION: %" PRIu64
" record(s) absent from %" PRIu64 "\n",
absent_coll, count);
fatal = true;
}
if (!inmem && absent_local) {
printf("LOCAL: %" PRIu64 " record(s) absent from %" PRIu64 "\n",
absent_local, count);
fatal = true;
}
if (!inmem && absent_oplog) {
printf("OPLOG: %" PRIu64 " record(s) absent from %" PRIu64 "\n",
absent_oplog, count);
fatal = true;
}
testutil_check(pthread_rwlock_destroy(&ts_lock));
if (fatal)
return (EXIT_FAILURE);
printf("%" PRIu64 " records verified\n", count);
return (EXIT_SUCCESS);
}
/*
* slow_apply_api --
* Apply a set of modification changes using a different algorithm.
*/
static void
slow_apply_api(WT_ITEM *orig)
{
static WT_ITEM _tb;
WT_ITEM *ta, *tb, *tmp, _tmp;
size_t len, size;
int i;
ta = orig;
tb = &_tb;
/* Mess up anything not initialized in the buffers. */
memset((uint8_t *)ta->mem + ta->size, 0xff, ta->memsize - ta->size);
memset((uint8_t *)tb->mem, 0xff, tb->memsize);
/*
* Process the entries to figure out how large a buffer we need. This is
* a bit pessimistic because we're ignoring replacement bytes, but it's
* a simpler calculation.
*/
for (size = ta->size, i = 0; i < nentries; ++i) {
if (entries[i].offset >= size)
size = entries[i].offset;
size += entries[i].data.size;
}
testutil_check(__wt_buf_grow(NULL, ta, size));
testutil_check(__wt_buf_grow(NULL, tb, size));
#if DEBUG
show(ta, "slow-apply start");
#endif
/*
* From the starting buffer, create a new buffer b based on changes
* in the entries array. We're doing a brute force solution here to
* test the faster solution implemented in the library.
*/
for (i = 0; i < nentries; ++i) {
/* Take leading bytes from the original, plus any gap bytes. */
if (entries[i].offset >= ta->size) {
memcpy(tb->mem, ta->mem, ta->size);
if (entries[i].offset > ta->size)
memset((uint8_t *)tb->mem + ta->size,
'\0', entries[i].offset - ta->size);
} else
if (entries[i].offset > 0)
memcpy(tb->mem, ta->mem, entries[i].offset);
tb->size = entries[i].offset;
/* Take replacement bytes. */
if (entries[i].data.size > 0) {
memcpy((uint8_t *)tb->mem + tb->size,
entries[i].data.data, entries[i].data.size);
tb->size += entries[i].data.size;
}
/* Take trailing bytes from the original. */
len = entries[i].offset + entries[i].size;
if (ta->size > len) {
memcpy((uint8_t *)tb->mem + tb->size,
(uint8_t *)ta->mem + len, ta->size - len);
tb->size += ta->size - len;
}
testutil_assert(tb->size <= size);
/* Swap the buffers and do it again. */
tmp = ta;
ta = tb;
tb = tmp;
}
ta->data = ta->mem;
tb->data = tb->mem;
/*
* The final results may not be in the original buffer, in which case
* we swap them back around.
*/
if (ta != orig) {
_tmp = *ta;
*ta = *tb;
*tb = _tmp;
}
#if DEBUG
show(ta, "slow-apply finish");
#endif
}
int
main(int argc, char *argv[])
{
TEST_OPTS *opts, _opts;
WT_CURSOR *cursor;
WT_SESSION *session;
pthread_t thr[NR_THREADS];
size_t t;
uint64_t f[NR_FIELDS], r, ts;
int i, ret;
char table_format[256];
opts = &_opts;
memset(opts, 0, sizeof(*opts));
testutil_check(testutil_parse_opts(argc, argv, opts));
testutil_make_work_dir(opts->home);
testutil_check(wiredtiger_open(opts->home, NULL,
"create,cache_size=1G,checkpoint=(wait=30),"
"eviction_trigger=80,eviction_target=64,eviction_dirty_target=65,"
"log=(enabled,file_max=10M),"
"transaction_sync=(enabled=true,method=none)", &opts->conn));
testutil_check(opts->conn->open_session(
opts->conn, NULL, NULL, &session));
sprintf(table_format, "key_format=r,value_format=");
for (i = 0; i < NR_FIELDS; i++)
strcat(table_format, "Q");
/* recno -> timestamp + NR_FIELDS * Q */
testutil_check(session->create(
session, opts->uri, table_format));
/* timestamp -> recno */
testutil_check(session->create(session,
"table:index", "key_format=Q,value_format=Q"));
testutil_check(session->close(session, NULL));
for (t = 0; t < NR_THREADS; ++t)
testutil_check(pthread_create(
&thr[t], NULL, thread_func, (void *)opts));
for (t = 0; t < NR_THREADS; ++t)
(void)pthread_join(thr[t], NULL);
testutil_check(opts->conn->open_session(
opts->conn, NULL, NULL, &session));
/* recno -> timestamp + NR_FIELDS * Q */
testutil_check(session->create(session, opts->uri, table_format));
testutil_check(session->open_cursor(
session, opts->uri, NULL, NULL, &cursor));
while ((ret = cursor->next(cursor)) == 0) {
testutil_check(cursor->get_key(cursor, &r));
testutil_check(cursor->get_value(cursor, &ts,
&f[0], &f[1], &f[2], &f[3], &f[4], &f[5], &f[6], &f[7]));
if (!opts->verbose)
continue;
printf("(%" PRIu64 ",%llu)\t\t%" PRIu64,
(r >> 40), r & ((1ULL << 40) - 1), ts);
for (i = 0; i < NR_FIELDS; i++)
printf("\t%" PRIu64, f[i]);
printf("\n");
}
testutil_assert(ret == WT_NOTFOUND);
testutil_cleanup(opts);
return (EXIT_SUCCESS);
}
/*
* check_results --
* Check all the tables and verify the results.
*/
static int
check_results(TEST_OPTS *opts, uint64_t *foundp)
{
WT_CURSOR *maincur, *maincur2, *v0cur, *v1cur, *v2cur;
WT_SESSION *session;
uint64_t count, idxcount, nrecords;
uint32_t rndint;
int key, key_got, ret, v0, v1, v2;
char *big, *bigref;
testutil_check(create_big_string(&bigref));
nrecords = opts->nrecords;
testutil_check(wiredtiger_open(opts->home, NULL,
"create,log=(enabled)", &opts->conn));
testutil_check(
opts->conn->open_session(opts->conn, NULL, NULL, &session));
testutil_check(session->open_cursor(session, "table:subtest", NULL,
NULL, &maincur));
testutil_check(session->open_cursor(session, "table:subtest2", NULL,
NULL, &maincur2));
testutil_check(session->open_cursor(session, "index:subtest:v0", NULL,
NULL, &v0cur));
testutil_check(session->open_cursor(session, "index:subtest:v1", NULL,
NULL, &v1cur));
testutil_check(session->open_cursor(session, "index:subtest:v2", NULL,
NULL, &v2cur));
count = 0;
while ((ret = maincur->next(maincur)) == 0) {
testutil_check(maincur2->next(maincur2));
testutil_check(maincur2->get_key(maincur2, &key_got));
testutil_check(maincur2->get_value(maincur2, &rndint));
generate_key(count, &key);
generate_value(rndint, count, bigref, &v0, &v1, &v2, &big);
testutil_assert(key == key_got);
/* Check the key/values in main table. */
testutil_check(maincur->get_key(maincur, &key_got));
testutil_assert(key == key_got);
check_values(maincur, v0, v1, v2, big);
/* Check the values in the indices. */
v0cur->set_key(v0cur, v0);
testutil_check(v0cur->search(v0cur));
check_values(v0cur, v0, v1, v2, big);
v1cur->set_key(v1cur, v1);
testutil_check(v1cur->search(v1cur));
check_values(v1cur, v0, v1, v2, big);
v2cur->set_key(v2cur, v2);
testutil_check(v2cur->search(v2cur));
check_values(v2cur, v0, v1, v2, big);
count++;
if (count % VERBOSE_PRINT == 0 && opts->verbose)
printf("checked %" PRIu64 "/%" PRIu64 "\n", count,
nrecords);
}
if (count % VERBOSE_PRINT != 0 && opts->verbose)
printf("checked %" PRIu64 "/%" PRIu64 "\n", count, nrecords);
/*
* Always expect at least one entry, as populate does a
* checkpoint after the first insert.
*/
testutil_assert(count > 0);
testutil_assert(ret == WT_NOTFOUND);
testutil_assert(maincur2->next(maincur2) == WT_NOTFOUND);
cursor_count_items(v0cur, &idxcount);
testutil_assert(count == idxcount);
cursor_count_items(v1cur, &idxcount);
testutil_assert(count == idxcount);
cursor_count_items(v2cur, &idxcount);
testutil_assert(count == idxcount);
testutil_check(opts->conn->close(opts->conn, NULL));
opts->conn = NULL;
free(bigref);
*foundp = count;
return (0);
}
int
main(int argc, char *argv[])
{
TEST_OPTS *opts, _opts;
WT_CURSOR *rcursor, *wcursor;
WT_ITEM key, value;
WT_SESSION *session, *session2;
pthread_t thread;
uint64_t i;
char str[] = "0000000000000000";
/*
* Create a clean test directory for this run of the test program if the
* environment variable isn't already set (as is done by make check).
*/
opts = &_opts;
memset(opts, 0, sizeof(*opts));
testutil_check(testutil_parse_opts(argc, argv, opts));
testutil_make_work_dir(opts->home);
testutil_check(wiredtiger_open(opts->home,
NULL, "create,cache_size=200M", &opts->conn));
testutil_check(
opts->conn->open_session(opts->conn, NULL, NULL, &session));
testutil_check(
opts->conn->open_session(opts->conn, NULL, NULL, &session2));
testutil_check(session->create(session, name,
"key_format=Q,value_format=S"));
/* Populate the table with some data. */
testutil_check(session->open_cursor(
session, name, NULL, "overwrite", &wcursor));
for (i = 0; i < NUM_DOCS; i++) {
wcursor->set_key(wcursor, i);
rand_str(i, str);
wcursor->set_value(wcursor, str);
testutil_check(wcursor->insert(wcursor));
}
testutil_check(wcursor->close(wcursor));
printf("%d documents inserted\n", NUM_DOCS);
/* Perform some random reads */
testutil_check(session->open_cursor(
session, name, NULL, "next_random=true", &rcursor));
query_docs(rcursor, false);
testutil_check(rcursor->close(rcursor));
/* Setup Transaction to pin the current values */
testutil_check(
session2->begin_transaction(session2, "isolation=snapshot"));
testutil_check(session2->open_cursor(
session2, name, NULL, "next_random=true", &rcursor));
/* Perform updates in a txn to confirm that we see only the original. */
testutil_check(session->open_cursor(
session, name, NULL, "overwrite", &wcursor));
for (i = 0; i < NUM_DOCS; i++) {
rand_str(i, str);
str[0] = 'A';
wcursor->set_key(wcursor, i);
wcursor->set_value(wcursor, str);
testutil_check(wcursor->update(wcursor));
}
testutil_check(wcursor->close(wcursor));
printf("%d documents set to update\n", NUM_DOCS);
/* Random reads, which should see the original values */
query_docs(rcursor, false);
testutil_check(rcursor->close(rcursor));
/* Finish the txn */
testutil_check(session2->rollback_transaction(session2, NULL));
/* Random reads, which should see the updated values */
testutil_check(session2->open_cursor(
session2, name, NULL, "next_random=true", &rcursor));
query_docs(rcursor, true);
testutil_check(rcursor->close(rcursor));
/* Setup a pre-delete txn */
testutil_check(
session2->begin_transaction(session2, "isolation=snapshot"));
testutil_check(session2->open_cursor(
session2, name, NULL, "next_random=true", &rcursor));
/* Delete all but one document */
testutil_check(session->open_cursor(
session, name, NULL, "overwrite", &wcursor));
for (i = 0; i < NUM_DOCS - 1; i++) {
wcursor->set_key(wcursor, i);
testutil_check(wcursor->remove(wcursor));
}
testutil_check(wcursor->close(wcursor));
printf("%d documents deleted\n", NUM_DOCS - 1);
/* Random reads, which should not see the deletes */
query_docs(rcursor, true);
testutil_check(rcursor->close(rcursor));
/* Rollback the txn so we can see the deletes */
testutil_check(session2->rollback_transaction(session2, NULL));
/* Find the one remaining document 3 times */
testutil_check(session2->open_cursor(
session2, name, NULL, "next_random=true", &rcursor));
for (i = 0; i < 3; i++) {
testutil_check(rcursor->next(rcursor));
testutil_check(rcursor->get_key(rcursor, &key));
testutil_check(rcursor->get_value(rcursor, &value));
/* There should only be one value available to us */
testutil_assertfmt((uint64_t)key.data == NUM_DOCS - 1,
"expected %d and got %" PRIu64,
NUM_DOCS - 1, (uint64_t)key.data);
check_str((uint64_t)key.data, (char *)value.data, true);
}
printf("Found the deleted doc 3 times\n");
testutil_check(rcursor->close(rcursor));
/* Repopulate the table for compact. */
testutil_check(session->open_cursor(
session, name, NULL, "overwrite", &wcursor));
for (i = 0; i < NUM_DOCS - 1; i++) {
wcursor->set_key(wcursor, i);
rand_str(i, str);
str[0] = 'A';
wcursor->set_value(wcursor, str);
testutil_check(wcursor->insert(wcursor));
}
testutil_check(wcursor->close(wcursor));
/* Run random cursor queries while compact is running */
testutil_check(session2->open_cursor(
session2, name, NULL, "next_random=true", &rcursor));
testutil_check(pthread_create(&thread, NULL, compact_thread, session));
query_docs(rcursor, true);
testutil_check(rcursor->close(rcursor));
testutil_check(pthread_join(thread, NULL));
/* Delete everything. Check for infinite loops */
testutil_check(session->open_cursor(
session, name, NULL, "overwrite", &wcursor));
for (i = 0; i < NUM_DOCS; i++) {
wcursor->set_key(wcursor, i);
testutil_check(wcursor->remove(wcursor));
}
testutil_check(wcursor->close(wcursor));
testutil_check(session2->open_cursor(
session2, name, NULL, "next_random=true", &rcursor));
for (i = 0; i < 3; i++)
testutil_assert(rcursor->next(rcursor) == WT_NOTFOUND);
printf("Successfully got WT_NOTFOUND\n");
testutil_cleanup(opts);
return (EXIT_SUCCESS);
}
void
test_value(int64_t val)
{
const uint8_t *cp;
uint8_t buf[WT_INTPACK64_MAXSIZE + 8]; /* -Werror=array-bounds */
uint8_t *p;
int64_t sinput, soutput;
uint64_t uinput, uoutput;
size_t used_len;
memset(buf, 0xff, sizeof(buf)); /* -Werror=maybe-uninitialized */
sinput = val;
soutput = 0; /* -Werror=maybe-uninitialized */
/*
* Required on some systems to pull in parts of the library
* for which we have data references.
*/
testutil_check(__wt_library_init());
p = buf;
testutil_check(__wt_vpack_int(&p, sizeof(buf), sinput));
used_len = (size_t)(p - buf);
testutil_assert(used_len <= WT_INTPACK64_MAXSIZE);
cp = buf;
testutil_check(__wt_vunpack_int(&cp, used_len, &soutput));
/* Ensure we got the correct value back */
if (sinput != soutput) {
fprintf(stderr,
"mismatch %" PRId64 ", %" PRId64 "\n", sinput, soutput);
abort();
}
/* Ensure that decoding used the correct amount of buffer */
if (cp != p) {
fprintf(stderr,
"Unpack consumed wrong size for %" PRId64
", expected %" WT_SIZET_FMT ", got %" WT_SIZET_FMT "\n",
sinput, used_len, cp > p ?
used_len + (size_t)(cp - p) : /* More than buf used */
used_len - (size_t)(p - cp)); /* Less than buf used */
abort();
}
/* Test unsigned, convert negative into bigger positive values */
uinput = (uint64_t)val;
p = buf;
testutil_check(__wt_vpack_uint(&p, sizeof(buf), uinput));
used_len = (size_t)(p - buf);
testutil_assert(used_len <= WT_INTPACK64_MAXSIZE);
cp = buf;
testutil_check(__wt_vunpack_uint(&cp, sizeof(buf), &uoutput));
/* Ensure we got the correct value back */
if (sinput != soutput) {
fprintf(stderr,
"mismatch %" PRId64 ", %" PRId64 "\n", sinput, soutput);
abort();
}
/* Ensure that decoding used the correct amount of buffer */
if (cp != p) {
fprintf(stderr,
"Unpack consumed wrong size for %" PRId64
", expected %" WT_SIZET_FMT ", got %" WT_SIZET_FMT "\n",
sinput, used_len, cp > p ?
used_len + (size_t)(cp - p) :
used_len - (size_t)(p - cp));
abort();
}
}
static int32_t
item_to_int(WT_ITEM *item)
{
testutil_assert(item->size == sizeof(int32_t));
return (*(int32_t *)item->data);
}
int
main(int argc, char *argv[])
{
TEST_OPTS *opts, _opts;
WT_CURSOR *cursor, *cursor1;
WT_ITEM got, k, v;
WT_SESSION *session;
int32_t ki, vi;
opts = &_opts;
memset(opts, 0, sizeof(*opts));
testutil_check(testutil_parse_opts(argc, argv, opts));
testutil_make_work_dir(opts->home);
testutil_check(wiredtiger_open(opts->home, NULL, "create",
&opts->conn));
testutil_check(
opts->conn->open_session(opts->conn, NULL, NULL, &session));
testutil_check(opts->conn->add_collator(opts->conn, "index_coll",
&index_coll, NULL));
testutil_check(session->create(session,
"table:main", "key_format=u,value_format=u,columns=(k,v)"));
testutil_check(session->create(session,
"index:main:index", "columns=(v),collator=index_coll"));
printf("adding new record\n");
testutil_check(session->open_cursor(session, "table:main", NULL, NULL,
&cursor));
ki = 13;
vi = 17;
k.data = &ki; k.size = sizeof(ki);
v.data = &vi; v.size = sizeof(vi);
cursor->set_key(cursor, &k);
cursor->set_value(cursor, &v);
testutil_check(cursor->insert(cursor));
testutil_check(cursor->close(cursor));
printf("positioning index cursor\n");
testutil_check(session->open_cursor(session, "index:main:index", NULL,
NULL, &cursor));
cursor->set_key(cursor, &v);
testutil_check(cursor->search(cursor));
printf("duplicating cursor\n");
testutil_check(session->open_cursor(session, NULL, cursor, NULL,
&cursor1));
testutil_check(cursor->get_value(cursor, &got));
testutil_assert(item_to_int(&got) == 17);
testutil_check(cursor1->get_value(cursor1, &got));
testutil_assert(item_to_int(&got) == 17);
testutil_check(session->close(session, NULL));
testutil_cleanup(opts);
return (EXIT_SUCCESS);
}
int
main(int argc, char *argv[])
{
POP_RECORD *p;
TEST_OPTS *opts, _opts;
WT_CURSOR *country_cursor, *country_cursor2, *cursor, *join_cursor,
*subjoin_cursor, *year_cursor;
WT_SESSION *session;
const char *country, *tablename;
char countryuri[256], joinuri[256], yearuri[256];
uint64_t recno, population;
uint16_t year;
int count, ret;
opts = &_opts;
memset(opts, 0, sizeof(*opts));
testutil_check(testutil_parse_opts(argc, argv, opts));
testutil_make_work_dir(opts->home);
tablename = strchr(opts->uri, ':');
testutil_assert(tablename != NULL);
tablename++;
snprintf(countryuri, sizeof(countryuri), "index:%s:country", tablename);
snprintf(yearuri, sizeof(yearuri), "index:%s:year", tablename);
snprintf(joinuri, sizeof(joinuri), "join:%s", opts->uri);
testutil_check(wiredtiger_open(opts->home, NULL,
"create,cache_size=200M", &opts->conn));
testutil_check(
opts->conn->open_session(opts->conn, NULL, NULL, &session));
testutil_check(session->create(session, opts->uri,
"key_format=r,"
"value_format=5sHQ,"
"columns=(id,country,year,population)"));
/* Create an index with a simple key. */
testutil_check(session->create(session,
countryuri, "columns=(country)"));
/* Create an immutable index. */
testutil_check(session->create(session,
yearuri, "columns=(year),immutable"));
/* Insert the records into the table. */
testutil_check(session->open_cursor(
session, opts->uri, NULL, "append", &cursor));
count = 1;
for (p = pop_data; p->year != 0; p++) {
cursor->set_key(cursor, count);
cursor->set_value(cursor, p->country, p->year, p->population);
testutil_check(cursor->insert(cursor));
count++;
}
testutil_check(cursor->close(cursor));
/* Open cursors needed by the join. */
testutil_check(session->open_cursor(session,
joinuri, NULL, NULL, &join_cursor));
testutil_check(session->open_cursor(session,
countryuri, NULL, NULL, &country_cursor));
testutil_check(session->open_cursor(session,
yearuri, NULL, NULL, &year_cursor));
/* select values WHERE country == "AU" AND year > 1900 */
country_cursor->set_key(country_cursor, "AU\0\0\0");
testutil_check(country_cursor->search(country_cursor));
testutil_check(session->join(session, join_cursor, country_cursor,
"compare=eq,count=10"));
year_cursor->set_key(year_cursor, (uint16_t)1900);
testutil_check(year_cursor->search(year_cursor));
testutil_check(session->join(session, join_cursor, year_cursor,
"compare=gt,count=10,strategy=bloom"));
count = 0;
/* List the values that are joined */
while ((ret = join_cursor->next(join_cursor)) == 0) {
testutil_check(join_cursor->get_key(join_cursor, &recno));
testutil_check(join_cursor->get_value(join_cursor, &country,
&year, &population));
printf("ID %" PRIu64, recno);
printf(
": country %s, year %" PRIu16 ", population %" PRIu64 "\n",
country, year, population);
count++;
}
testutil_assert(ret == WT_NOTFOUND);
testutil_assert(count == 2);
testutil_check(join_cursor->close(join_cursor));
testutil_check(year_cursor->close(year_cursor));
testutil_check(country_cursor->close(country_cursor));
/* Open cursors needed by the join. */
testutil_check(session->open_cursor(session,
joinuri, NULL, NULL, &join_cursor));
testutil_check(session->open_cursor(session,
joinuri, NULL, NULL, &subjoin_cursor));
testutil_check(session->open_cursor(session,
countryuri, NULL, NULL, &country_cursor));
testutil_check(session->open_cursor(session,
countryuri, NULL, NULL, &country_cursor2));
testutil_check(session->open_cursor(session,
yearuri, NULL, NULL, &year_cursor));
/*
* select values WHERE (country == "AU" OR country == "UK")
* AND year > 1900
*
* First, set up the join representing the country clause.
*/
country_cursor->set_key(country_cursor, "AU\0\0\0");
testutil_check(country_cursor->search(country_cursor));
testutil_check(session->join(session, subjoin_cursor, country_cursor,
"operation=or,compare=eq,count=10"));
country_cursor2->set_key(country_cursor2, "UK\0\0\0");
testutil_check(country_cursor2->search(country_cursor2));
testutil_check(session->join(session, subjoin_cursor, country_cursor2,
"operation=or,compare=eq,count=10"));
/* Join that to the top join, and add the year clause */
testutil_check(session->join(session, join_cursor, subjoin_cursor,
NULL));
year_cursor->set_key(year_cursor, (uint16_t)1900);
testutil_check(year_cursor->search(year_cursor));
testutil_check(session->join(session, join_cursor, year_cursor,
"compare=gt,count=10,strategy=bloom"));
count = 0;
/* List the values that are joined */
while ((ret = join_cursor->next(join_cursor)) == 0) {
testutil_check(join_cursor->get_key(join_cursor, &recno));
testutil_check(join_cursor->get_value(join_cursor, &country,
&year, &population));
printf("ID %" PRIu64, recno);
printf(
": country %s, year %" PRIu16 ", population %" PRIu64 "\n",
country, year, population);
count++;
}
testutil_assert(ret == WT_NOTFOUND);
testutil_assert(count == 4);
testutil_check(join_cursor->close(join_cursor));
testutil_check(subjoin_cursor->close(subjoin_cursor));
testutil_check(country_cursor->close(country_cursor));
testutil_check(country_cursor2->close(country_cursor2));
testutil_check(year_cursor->close(year_cursor));
testutil_check(session->close(session, NULL));
testutil_cleanup(opts);
return (EXIT_SUCCESS);
}
int
main(int argc, char *argv[])
{
SHARED_OPTS *sharedopts, _sharedopts;
TEST_OPTS *opts, _opts;
THREAD_ARGS get_args[N_GET_THREAD], insert_args[N_INSERT_THREAD];
WT_CURSOR *maincur;
WT_SESSION *session;
pthread_t get_tid[N_GET_THREAD], insert_tid[N_INSERT_THREAD];
int i, nfail;
const char *tablename;
opts = &_opts;
sharedopts = &_sharedopts;
if (testutil_disable_long_tests())
return (0);
memset(opts, 0, sizeof(*opts));
memset(sharedopts, 0, sizeof(*sharedopts));
memset(insert_args, 0, sizeof(insert_args));
memset(get_args, 0, sizeof(get_args));
nfail = 0;
sharedopts->bloom = BLOOM;
testutil_check(testutil_parse_opts(argc, argv, opts));
testutil_make_work_dir(opts->home);
testutil_check(wiredtiger_open(opts->home, NULL,
"create,cache_size=1G", &opts->conn));
testutil_check(
opts->conn->open_session(opts->conn, NULL, NULL, &session));
/*
* Note: id is repeated as id2. This makes it easier to
* identify the primary key in dumps of the index files.
*/
testutil_check(session->create(session, opts->uri,
"key_format=i,value_format=iiSii,"
"columns=(id,post,bal,extra,flag,id2)"));
tablename = strchr(opts->uri, ':');
testutil_assert(tablename != NULL);
tablename++;
snprintf(sharedopts->posturi, sizeof(sharedopts->posturi),
"index:%s:post", tablename);
snprintf(sharedopts->baluri, sizeof(sharedopts->baluri),
"index:%s:bal", tablename);
snprintf(sharedopts->flaguri, sizeof(sharedopts->flaguri),
"index:%s:flag", tablename);
testutil_check(session->create(session, sharedopts->posturi,
"columns=(post)"));
testutil_check(session->create(session, sharedopts->baluri,
"columns=(bal)"));
testutil_check(session->create(session, sharedopts->flaguri,
"columns=(flag)"));
/*
* Insert a single record with all items we need to
* call search() on, this makes our join logic easier.
*/
testutil_check(session->open_cursor(session, opts->uri, NULL, NULL,
&maincur));
maincur->set_key(maincur, N_RECORDS);
maincur->set_value(maincur, 54321, 0, "", 0, N_RECORDS);
testutil_check(maincur->insert(maincur));
testutil_check(maincur->close(maincur));
testutil_check(session->close(session, NULL));
for (i = 0; i < N_INSERT_THREAD; ++i) {
insert_args[i].threadnum = i;
insert_args[i].nthread = N_INSERT_THREAD;
insert_args[i].testopts = opts;
insert_args[i].sharedopts = sharedopts;
testutil_check(pthread_create(&insert_tid[i], NULL,
thread_insert, (void *)&insert_args[i]));
}
for (i = 0; i < N_GET_THREAD; ++i) {
get_args[i].threadnum = i;
get_args[i].nthread = N_GET_THREAD;
get_args[i].testopts = opts;
get_args[i].sharedopts = sharedopts;
testutil_check(pthread_create(&get_tid[i], NULL,
thread_get, (void *)&get_args[i]));
}
/*
* Wait for insert threads to finish. When they
* are done, signal get threads to complete.
*/
for (i = 0; i < N_INSERT_THREAD; ++i)
testutil_check(pthread_join(insert_tid[i], NULL));
for (i = 0; i < N_GET_THREAD; ++i)
get_args[i].done = 1;
for (i = 0; i < N_GET_THREAD; ++i)
testutil_check(pthread_join(get_tid[i], NULL));
fprintf(stderr, "\n");
for (i = 0; i < N_GET_THREAD; ++i) {
fprintf(stderr, " thread %d did %d joins (%d fails)\n", i,
get_args[i].njoins, get_args[i].nfail);
nfail += get_args[i].nfail;
}
testutil_assert(nfail == 0);
testutil_cleanup(opts);
return (0);
}
int
main(int argc, char *argv[])
{
TEST_OPTS *opts, _opts;
WT_CURSOR *balancecur, *flagcur, *joincur, *postcur;
WT_CURSOR *maincur;
WT_SESSION *session;
int balance, count, flag, key, key2, post, ret;
char balanceuri[256];
char cfg[128];
char flaguri[256];
char joinuri[256];
char posturi[256];
const char *tablename;
opts = &_opts;
memset(opts, 0, sizeof(*opts));
testutil_check(testutil_parse_opts(argc, argv, opts));
testutil_make_work_dir(opts->home);
testutil_progress(opts, "start");
testutil_check(wiredtiger_open(opts->home, NULL,
"create,cache_size=250M", &opts->conn));
testutil_progress(opts, "wiredtiger_open");
testutil_check(
opts->conn->open_session(opts->conn, NULL, NULL, &session));
testutil_progress(opts, "sessions opened");
/*
* Note: repeated primary key 'id' as 'id2'. This makes
* it easier to dump an index and know which record we're
* looking at.
*/
testutil_check(session->create(session, opts->uri,
"key_format=i,value_format=iiii,"
"columns=(id,post,balance,flag,id2)"));
tablename = strchr(opts->uri, ':');
testutil_assert(tablename != NULL);
tablename++;
testutil_check(__wt_snprintf(
posturi, sizeof(posturi), "index:%s:post", tablename));
testutil_check(__wt_snprintf(
balanceuri, sizeof(balanceuri), "index:%s:balance", tablename));
testutil_check(__wt_snprintf(
flaguri, sizeof(flaguri), "index:%s:flag", tablename));
testutil_check(__wt_snprintf(
joinuri, sizeof(joinuri), "join:%s", opts->uri));
testutil_check(session->create(session, posturi, "columns=(post)"));
testutil_check(session->create(session, balanceuri,
"columns=(balance)"));
testutil_check(session->create(session, flaguri, "columns=(flag)"));
testutil_progress(opts, "setup complete");
/*
* Insert a single record with all items we are search for,
* this makes our logic easier.
*/
testutil_check(session->open_cursor(session, opts->uri, NULL, NULL,
&maincur));
maincur->set_key(maincur, N_RECORDS);
maincur->set_value(maincur, 54321, 0, "", 0, N_RECORDS);
testutil_check(maincur->insert(maincur));
testutil_check(maincur->close(maincur));
testutil_check(session->close(session, NULL));
testutil_progress(opts, "populate start");
populate(opts);
testutil_progress(opts, "populate end");
testutil_check(opts->conn->open_session(
opts->conn, NULL, NULL, &session));
testutil_check(session->open_cursor(session,
posturi, NULL, NULL, &postcur));
testutil_check(session->open_cursor(session,
balanceuri, NULL, NULL, &balancecur));
testutil_check(session->open_cursor(session,
flaguri, NULL, NULL, &flagcur));
testutil_check(session->open_cursor(session,
joinuri, NULL, NULL, &joincur));
postcur->set_key(postcur, 54321);
testutil_check(postcur->search(postcur));
testutil_check(session->join(session, joincur, postcur,
"compare=eq"));
balancecur->set_key(balancecur, 0);
testutil_check(balancecur->search(balancecur));
testutil_check(__wt_snprintf(cfg, sizeof(cfg),
"compare=lt,strategy=bloom,count=%d", N_RECORDS / 100));
testutil_check(session->join(session, joincur, balancecur, cfg));
flagcur->set_key(flagcur, 0);
testutil_check(flagcur->search(flagcur));
testutil_check(__wt_snprintf(cfg, sizeof(cfg),
"compare=eq,strategy=bloom,count=%d", N_RECORDS / 100));
testutil_check(session->join(session, joincur, flagcur, cfg));
/* Expect no values returned */
count = 0;
while ((ret = joincur->next(joincur)) == 0) {
/*
* The values may already have been changed, but
* print them for informational purposes.
*/
testutil_check(joincur->get_key(joincur, &key));
testutil_check(joincur->get_value(joincur, &post,
&balance, &flag, &key2));
fprintf(stderr, "FAIL: "
"key=%d/%d, postal_code=%d, balance=%d, flag=%d\n",
key, key2, post, balance, flag);
count++;
}
testutil_assert(ret == WT_NOTFOUND);
testutil_assert(count == 0);
testutil_progress(opts, "cleanup starting");
testutil_cleanup(opts);
return (EXIT_SUCCESS);
}
static void *
thread_get(void *arg)
{
SHARED_OPTS *sharedopts;
TEST_OPTS *opts;
THREAD_ARGS *threadargs;
WT_CURSOR *maincur, *postcur;
WT_SESSION *session;
double elapsed;
time_t prevtime, curtime; /* 1 second resolution is okay */
int bal, flag, key, key2, post, bal2, flag2, post2;
char *extra;
threadargs = (THREAD_ARGS *)arg;
opts = threadargs->testopts;
sharedopts = threadargs->sharedopts;
(void)time(&prevtime);
testutil_check(opts->conn->open_session(
opts->conn, NULL, NULL, &session));
testutil_check(session->open_cursor(session, opts->uri, NULL, NULL,
&maincur));
testutil_check(session->open_cursor(
session, sharedopts->posturi, NULL, NULL, &postcur));
for (threadargs->njoins = 0; threadargs->done == 0;
threadargs->njoins++) {
testutil_check(session->begin_transaction(session, NULL));
postcur->set_key(postcur, 54321);
testutil_check(postcur->search(postcur));
while (postcur->next(postcur) == 0) {
testutil_check(postcur->get_key(postcur, &post));
testutil_check(postcur->get_value(postcur, &post2,
&bal, &extra, &flag, &key));
testutil_assert(post == post2);
if (post != 54321)
break;
maincur->set_key(maincur, key);
testutil_check(maincur->search(maincur));
testutil_check(maincur->get_value(maincur, &post2,
&bal2, &extra, &flag2, &key2));
testutil_check(maincur->reset(maincur));
testutil_assert(key == key2);
testutil_assert(post == post2);
testutil_assert(bal == bal2);
testutil_assert(flag == flag2);
testutil_assert((flag2 > 0 && bal2 < 0) ||
(flag2 == 0 && bal2 >= 0));
}
/*
* Reset the cursors, potentially allowing the insert
* threads to proceed.
*/
testutil_check(postcur->reset(postcur));
if (threadargs->njoins % 100 == 0)
fprintf(stderr, "G");
testutil_check(session->rollback_transaction(session, NULL));
(void)time(&curtime);
if ((elapsed = difftime(curtime, prevtime)) > 5.0) {
fprintf(stderr, "\n"
"GAP: %.0f secs after %d gets\n",
elapsed, threadargs->njoins);
threadargs->nfail++;
}
prevtime = curtime;
}
testutil_check(postcur->close(postcur));
testutil_check(maincur->close(maincur));
testutil_check(session->close(session, NULL));
return (NULL);
}
