/*
* row_insert --
* Insert a row in a row-store file.
*/
static int
row_insert(TINFO *tinfo,
WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno)
{
WT_SESSION *session;
int ret;
session = cursor->session;
key_gen_insert(&tinfo->rnd, (uint8_t *)key->data, &key->size, keyno);
val_gen(&tinfo->rnd, (uint8_t *)value->data, &value->size, keyno);
/* Log the operation */
if (g.logging == LOG_OPS)
(void)g.wt_api->msg_printf(g.wt_api, session,
"%-10s{%.*s}\n%-10s{%.*s}",
"insertK", (int)key->size, (char *)key->data,
"insertV", (int)value->size, (char *)value->data);
cursor->set_key(cursor, key);
cursor->set_value(cursor, value);
ret = cursor->insert(cursor);
if (ret == WT_ROLLBACK)
return (WT_ROLLBACK);
if (ret != 0 && ret != WT_NOTFOUND)
testutil_die(ret,
"row_insert: insert row %" PRIu64 " by key", keyno);
#ifdef HAVE_BERKELEY_DB
if (!SINGLETHREADED)
return (0);
{
int notfound;
bdb_update(key->data, key->size, value->data, value->size, ¬found);
(void)notfound_chk("row_insert", ret, notfound, keyno);
}
#endif
return (0);
}
/*
* run_process --
* Run a program with arguments, wait until it completes.
*/
static int
run_process(TEST_OPTS *opts, const char *prog, char *argv[], int *status)
{
int pid;
char **arg;
if (opts->verbose) {
printf("running: ");
for (arg = argv; *arg != NULL; arg++)
printf("%s ", *arg);
printf("\n");
}
if ((pid = fork()) == 0) {
(void)execv(prog, argv);
testutil_die(errno, "%s", prog);
} else if (pid < 0)
return (errno);
(void)waitpid(pid, status, 0);
return (0);
}
/*
* compaction --
* Periodically do a compaction operation.
*/
void *
compact(void *arg)
{
WT_CONNECTION *conn;
WT_SESSION *session;
u_int period;
int ret;
(void)(arg);
/* Compaction isn't supported for all data sources. */
if (DATASOURCE("helium") || DATASOURCE("kvsbdb"))
return (NULL);
/* Open a session. */
conn = g.wts_conn;
testutil_check(conn->open_session(conn, NULL, NULL, &session));
/*
* Perform compaction at somewhere under 15 seconds (so we get at
* least one done), and then at 23 second intervals.
*/
for (period = mmrand(NULL, 1, 15);; period = 23) {
/* Sleep for short periods so we don't make the run wait. */
while (period > 0 && !g.workers_finished) {
--period;
sleep(1);
}
if (g.workers_finished)
break;
if ((ret = session->compact(
session, g.uri, NULL)) != 0 && ret != WT_ROLLBACK)
testutil_die(ret, "session.compact");
}
testutil_check(session->close(session, NULL));
return (NULL);
}
/*
* Create and drop a unique guaranteed table.
*/
void
op_create_unique(void *arg)
{
TEST_OPTS *opts;
TEST_PER_THREAD_OPTS *args;
WT_RAND_STATE rnd;
WT_SESSION *session;
int ret;
char new_uri[64];
args = (TEST_PER_THREAD_OPTS *)arg;
opts = args->testopts;
__wt_random_init_seed(NULL, &rnd);
testutil_check(
opts->conn->open_session(opts->conn, NULL, NULL, &session));
/* Generate a unique object name. */
testutil_check(__wt_snprintf(
new_uri, sizeof(new_uri), "%s.%" PRIu64,
opts->uri, __wt_atomic_add64(&opts->unique_id, 1)));
testutil_check(session->create(session, new_uri, DEFAULT_TABLE_SCHEMA));
__wt_yield();
while ((ret = session->drop(session, new_uri, __wt_random(&rnd) & 1 ?
"force,checkpoint_wait=false" : "checkpoint_wait=false")) != 0)
if (ret != EBUSY)
testutil_die(ret, "session.drop: %s", new_uri);
else
/*
* The EBUSY is expected when we run with
* checkpoint_wait set to false, so we increment the
* counter while in this loop to avoid false positives.
*/
args->thread_counter++;
testutil_check(session->close(session, NULL));
args->thread_counter++;
}
/*
* Create a table.
*/
void
op_create(void *arg)
{
TEST_OPTS *opts;
TEST_PER_THREAD_OPTS *args;
WT_SESSION *session;
int ret;
args = (TEST_PER_THREAD_OPTS *)arg;
opts = args->testopts;
testutil_check(
opts->conn->open_session(opts->conn, NULL, NULL, &session));
if ((ret = session->create(session,
opts->uri, DEFAULT_TABLE_SCHEMA)) != 0)
if (ret != EEXIST && ret != EBUSY)
testutil_die(ret, "session.create");
testutil_check(session->close(session, NULL));
args->thread_counter++;
}
/*
* wt_startup --
* Configure the WiredTiger connection.
*/
static void
wt_startup(char *config_open)
{
static WT_EVENT_HANDLER event_handler = {
handle_error,
handle_message,
NULL,
NULL /* Close handler. */
};
int ret;
char config_buf[128];
testutil_make_work_dir(home);
snprintf(config_buf, sizeof(config_buf),
"create,error_prefix=\"%s\",cache_size=5MB%s%s",
progname,
config_open == NULL ? "" : ",",
config_open == NULL ? "" : config_open);
if ((ret = wiredtiger_open(
home, &event_handler, config_buf, &conn)) != 0)
testutil_die(ret, "wiredtiger_open");
}
/*
* corrupt_metadata --
* Corrupt the metadata by scribbling on the "corrupt" URI string.
*/
static void
corrupt_metadata(void)
{
FILE *fp;
struct stat sb;
long off;
size_t meta_size;
bool corrupted;
uint8_t *buf, *corrupt;
char path[256];
/*
* Open the file, read its contents. Find the string "corrupt" and
* modify one byte at that offset. That will cause a checksum error
* when WiredTiger next reads it.
*/
testutil_check(__wt_snprintf(
path, sizeof(path), "%s/%s", home, WT_METAFILE));
if ((fp = fopen(path, "r+")) == NULL)
testutil_die(errno, "fopen: %s", path);
testutil_check(fstat(fileno(fp), &sb));
meta_size = (size_t)sb.st_size;
buf = dcalloc(meta_size, 1);
if (fread(buf, 1, meta_size, fp) != meta_size)
testutil_die(errno, "fread: %" WT_SIZET_FMT, meta_size);
corrupted = false;
/*
* Corrupt all occurrences of the string in the file.
*/
while ((corrupt = byte_str(buf, meta_size, CORRUPT)) != NULL) {
corrupted = true;
testutil_assert(*(char *)corrupt != 'X');
*(char *)corrupt = 'X';
off = (long)(corrupt - buf);
if (fseek(fp, off, SEEK_SET) != 0)
testutil_die(errno, "fseek: %ld", off);
if (fwrite("X", 1, 1, fp) != 1)
testutil_die(errno, "fwrite");
}
if (!corrupted)
testutil_die(errno, "corrupt string did not occur");
if (fclose(fp) != 0)
testutil_die(errno, "fclose");
free(buf);
}
static void
file_create(const char *name)
{
WT_SESSION *session;
int ret;
char config[128];
testutil_check(conn->open_session(conn, NULL, NULL, &session));
testutil_check(__wt_snprintf(config, sizeof(config),
"key_format=%s,"
"internal_page_max=%d,"
"leaf_page_max=%d,"
"%s",
ftype == ROW ? "u" : "r", 16 * 1024, 128 * 1024,
ftype == FIX ? ",value_format=3t" : ""));
if ((ret = session->create(session, name, config)) != 0)
if (ret != EEXIST)
testutil_die(ret, "session.create");
testutil_check(session->close(session, NULL));
}
/*
* Drop a table.
*/
void
op_drop(void *arg)
{
TEST_OPTS *opts;
TEST_PER_THREAD_OPTS *args;
WT_RAND_STATE rnd;
WT_SESSION *session;
int ret;
args = (TEST_PER_THREAD_OPTS *)arg;
opts = args->testopts;
__wt_random_init_seed(NULL, &rnd);
testutil_check(
opts->conn->open_session(opts->conn, NULL, NULL, &session));
if ((ret = session->drop(session, opts->uri, __wt_random(&rnd) & 1 ?
"force,checkpoint_wait=false" : "checkpoint_wait=false")) != 0)
if (ret != ENOENT && ret != EBUSY)
testutil_die(ret, "session.drop");
testutil_check(session->close(session, NULL));
args->thread_counter++;
}
/*
* Child process creates the database and table, and then writes data into
* the table until it is killed by the parent.
*/
static void
fill_db(void)
{
FILE *fp;
WT_CONNECTION *conn;
WT_CURSOR *cursor, *logc;
WT_LSN lsn, save_lsn;
WT_SESSION *session;
uint32_t i, max_key, min_key, units, unused;
int ret;
bool first;
char k[K_SIZE], v[V_SIZE];
/*
* Run in the home directory so that the records file is in there too.
*/
if (chdir(home) != 0)
testutil_die(errno, "chdir: %s", home);
if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG, &conn)) != 0)
testutil_die(ret, "wiredtiger_open");
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
testutil_die(ret, "WT_CONNECTION:open_session");
if ((ret = session->create(session,
uri, "key_format=S,value_format=S")) != 0)
testutil_die(ret, "WT_SESSION.create: %s", uri);
if ((ret =
session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", uri);
/*
* Keep a separate file with the records we wrote for checking.
*/
(void)unlink(RECORDS_FILE);
if ((fp = fopen(RECORDS_FILE, "w")) == NULL)
testutil_die(errno, "fopen");
/*
* Set to no buffering.
*/
(void)setvbuf(fp, NULL, _IONBF, 0);
save_lsn.l.file = 0;
/*
* Write data into the table until we move to log file 2.
* We do the calculation below so that we don't have to walk the
* log for every record.
*
* Calculate about how many records should fit in the log file.
* Subtract a bunch for metadata and file creation records.
* Then subtract out a few more records to be conservative.
*/
units = (K_SIZE + V_SIZE) / 128 + 1;
min_key = 90000 / (units * 128) - 15;
max_key = min_key * 2;
first = true;
for (i = 0; i < max_key; ++i) {
snprintf(k, sizeof(k), "key%03d", (int)i);
snprintf(v, sizeof(v), "value%0*d",
(int)(V_SIZE - strlen("value")), (int)i);
cursor->set_key(cursor, k);
cursor->set_value(cursor, v);
if ((ret = cursor->insert(cursor)) != 0)
testutil_die(ret, "WT_CURSOR.insert");
if (i > min_key) {
if ((ret = session->open_cursor(
session, "log:", NULL, NULL, &logc)) != 0)
testutil_die(ret, "open_cursor: log");
if (save_lsn.l.file != 0) {
logc->set_key(logc,
save_lsn.l.file, save_lsn.l.offset, 0);
if ((ret = logc->search(logc)) != 0)
testutil_die(errno, "search");
}
while ((ret = logc->next(logc)) == 0) {
if ((ret = logc->get_key(logc,
&lsn.l.file, &lsn.l.offset, &unused)) != 0)
testutil_die(errno, "get_key");
if (lsn.l.file < 2)
save_lsn = lsn;
else {
if (first)
testutil_die(EINVAL,
"min_key too high");
if (fprintf(fp,
"%" PRIu32 " %" PRIu32 "\n",
save_lsn.l.offset, i - 1) == -1)
testutil_die(errno, "fprintf");
break;
}
}
first = false;
}
}
if (fclose(fp) != 0)
testutil_die(errno, "fclose");
abort();
/* NOTREACHED */
}
int
main(int argc, char *argv[])
{
FILE *fp;
WT_CONNECTION *conn;
WT_CURSOR *cursor;
WT_SESSION *session;
uint64_t new_offset, offset;
uint32_t count, max_key;
int ch, status, ret;
pid_t pid;
const char *working_dir;
if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL)
progname = argv[0];
else
++progname;
working_dir = "WT_TEST.truncated-log";
while ((ch = __wt_getopt(progname, argc, argv, "h:")) != EOF)
switch (ch) {
case 'h':
working_dir = __wt_optarg;
break;
default:
usage();
}
argc -= __wt_optind;
argv += __wt_optind;
if (argc != 0)
usage();
testutil_work_dir_from_path(home, 512, working_dir);
testutil_make_work_dir(home);
/*
* Fork a child to insert as many items. We will then randomly
* kill the child, run recovery and make sure all items we wrote
* exist after recovery runs.
*/
if ((pid = fork()) < 0)
testutil_die(errno, "fork");
if (pid == 0) { /* child */
fill_db();
return (EXIT_SUCCESS);
}
/* parent */
/* Wait for child to kill itself. */
if (waitpid(pid, &status, 0) == -1)
testutil_die(errno, "waitpid");
/*
* !!! If we wanted to take a copy of the directory before recovery,
* this is the place to do it.
*/
if (chdir(home) != 0)
testutil_die(errno, "chdir: %s", home);
printf("Open database, run recovery and verify content\n");
if ((fp = fopen(RECORDS_FILE, "r")) == NULL)
testutil_die(errno, "fopen");
ret = fscanf(fp, "%" SCNu64 " %" SCNu32 "\n", &offset, &max_key);
if (ret != 2)
testutil_die(errno, "fscanf");
if (fclose(fp) != 0)
testutil_die(errno, "fclose");
/*
* The offset is the beginning of the last record. Truncate to
* the middle of that last record (i.e. ahead of that offset).
*/
if (offset > UINT64_MAX - V_SIZE)
testutil_die(ERANGE, "offset");
new_offset = offset + V_SIZE;
printf("Parent: Truncate to %" PRIu64 "\n", new_offset);
if ((ret = truncate(LOG_FILE_1, (wt_off_t)new_offset)) != 0)
testutil_die(errno, "truncate");
if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn)) != 0)
testutil_die(ret, "wiredtiger_open");
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
testutil_die(ret, "WT_CONNECTION:open_session");
if ((ret =
session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", uri);
/*
* For every key in the saved file, verify that the key exists
* in the table after recovery. Since we did write-no-sync, we
* expect every key to have been recovered.
*/
count = 0;
while ((ret = cursor->next(cursor)) == 0)
++count;
if ((ret = conn->close(conn, NULL)) != 0)
testutil_die(ret, "WT_CONNECTION:close");
if (count > max_key) {
printf("expected %" PRIu32 " records found %" PRIu32 "\n",
max_key, count);
return (EXIT_FAILURE);
}
printf("%" PRIu32 " records verified\n", count);
return (EXIT_SUCCESS);
}
/*
* corrupt --
* Corrupt the file in a random way.
*/
static int
corrupt(void)
{
FILE *fp;
struct stat sb;
size_t len, nw;
wt_off_t offset;
int fd, ret;
char buf[8 * 1024], copycmd[2 * 1024];
/*
* If it's a single Btree file (not LSM), open the file, and corrupt
* roughly 2% of the file at a random spot, including the beginning
* of the file and overlapping the end.
*
* It's a little tricky: if the data source is a file, we're looking
* for "wt", if the data source is a table, we're looking for "wt.wt".
*/
(void)snprintf(buf, sizeof(buf), "%s/%s", g.home, WT_NAME);
if ((fd = open(buf, O_RDWR)) != -1) {
#ifdef _WIN32
(void)snprintf(copycmd, sizeof(copycmd),
"copy %s\\%s %s\\slvg.copy\\%s.corrupted",
g.home, WT_NAME, g.home, WT_NAME);
#else
(void)snprintf(copycmd, sizeof(copycmd),
"cp %s/%s %s/slvg.copy/%s.corrupted",
g.home, WT_NAME, g.home, WT_NAME);
#endif
goto found;
}
(void)snprintf(buf, sizeof(buf), "%s/%s.wt", g.home, WT_NAME);
if ((fd = open(buf, O_RDWR)) != -1) {
#ifdef _WIN32
(void)snprintf(copycmd, sizeof(copycmd),
"copy %s\\%s.wt %s\\slvg.copy\\%s.wt.corrupted",
g.home, WT_NAME, g.home, WT_NAME);
#else
(void)snprintf(copycmd, sizeof(copycmd),
"cp %s/%s.wt %s/slvg.copy/%s.wt.corrupted",
g.home, WT_NAME, g.home, WT_NAME);
#endif
goto found;
}
return (0);
found: if (fstat(fd, &sb) == -1)
testutil_die(errno, "salvage-corrupt: fstat");
offset = mmrand(NULL, 0, (u_int)sb.st_size);
len = (size_t)(20 + (sb.st_size / 100) * 2);
(void)snprintf(buf, sizeof(buf), "%s/slvg.corrupt", g.home);
if ((fp = fopen(buf, "w")) == NULL)
testutil_die(errno, "salvage-corrupt: open: %s", buf);
(void)fprintf(fp,
"salvage-corrupt: offset %" PRIuMAX ", length " SIZET_FMT "\n",
(uintmax_t)offset, len);
fclose_and_clear(&fp);
if (lseek(fd, offset, SEEK_SET) == -1)
testutil_die(errno, "salvage-corrupt: lseek");
memset(buf, 'z', sizeof(buf));
for (; len > 0; len -= nw) {
nw = (size_t)(len > sizeof(buf) ? sizeof(buf) : len);
if (write(fd, buf, nw) == -1)
testutil_die(errno, "salvage-corrupt: write");
}
if (close(fd) == -1)
testutil_die(errno, "salvage-corrupt: close");
/*
* Save a copy of the corrupted file so we can replay the salvage step
* as necessary.
*/
if ((ret = system(copycmd)) != 0)
testutil_die(ret, "salvage corrupt copy step failed");
return (1);
}
int
main(int argc, char *argv[])
{
time_t start;
int ch, onerun, reps;
const char *config, *home;
config = NULL;
#ifdef _WIN32
g.progname = "t_format.exe";
#else
if ((g.progname = strrchr(argv[0], DIR_DELIM)) == NULL)
g.progname = argv[0];
else
++g.progname;
#endif
#if 0
/* Configure the GNU malloc for debugging. */
(void)setenv("MALLOC_CHECK_", "2", 1);
#endif
#if 0
/* Configure the FreeBSD malloc for debugging. */
(void)setenv("MALLOC_OPTIONS", "AJ", 1);
#endif
/* Track progress unless we're re-directing output to a file. */
g.c_quiet = isatty(1) ? 0 : 1;
/* Set values from the command line. */
home = NULL;
onerun = 0;
while ((ch = __wt_getopt(
g.progname, argc, argv, "1C:c:H:h:Llqrt:")) != EOF)
switch (ch) {
case '1': /* One run */
onerun = 1;
break;
case 'C': /* wiredtiger_open config */
g.config_open = __wt_optarg;
break;
case 'c': /* Configuration from a file */
config = __wt_optarg;
break;
case 'H':
g.helium_mount = __wt_optarg;
break;
case 'h':
home = __wt_optarg;
break;
case 'L': /* Re-direct output to a log */
/*
* The -l option is a superset of -L, ignore -L if we
* have already configured logging for operations.
*/
if (g.logging == 0)
g.logging = LOG_FILE;
break;
case 'l': /* Turn on operation logging */
g.logging = LOG_OPS;
break;
case 'q': /* Quiet */
g.c_quiet = 1;
break;
case 'r': /* Replay a run */
g.replay = 1;
break;
default:
usage();
}
argc -= __wt_optind;
argv += __wt_optind;
/* Initialize the global RNG. */
testutil_check(__wt_random_init_seed(NULL, &g.rnd));
/* Set up paths. */
path_setup(home);
/* If it's a replay, use the home directory's CONFIG file. */
if (g.replay) {
if (config != NULL)
testutil_die(EINVAL, "-c incompatible with -r");
if (access(g.home_config, R_OK) != 0)
testutil_die(ENOENT, "%s", g.home_config);
config = g.home_config;
}
/*
* If we weren't given a configuration file, set values from "CONFIG",
* if it exists.
*
* Small hack to ignore any CONFIG file named ".", that just makes it
* possible to ignore any local CONFIG file, used when running checks.
*/
if (config == NULL && access("CONFIG", R_OK) == 0)
config = "CONFIG";
if (config != NULL && strcmp(config, ".") != 0)
config_file(config);
/*
* The rest of the arguments are individual configurations that modify
* the base configuration.
*/
for (; *argv != NULL; ++argv)
config_single(*argv, 1);
/*
* Multithreaded runs can be replayed: it's useful and we'll get the
* configuration correct. Obviously the order of operations changes,
* warn the user.
*/
if (g.replay && !SINGLETHREADED)
printf("Warning: replaying a threaded run\n");
/*
* Single-threaded runs historically exited after a single replay, which
* makes sense when you're debugging, leave that semantic in place.
*/
if (g.replay && SINGLETHREADED)
g.c_runs = 1;
/*
* Let the command line -1 flag override runs configured from other
* sources.
*/
if (onerun)
g.c_runs = 1;
/*
* Initialize locks to single-thread named checkpoints and backups, last
* last-record updates, and failures.
*/
testutil_check(pthread_rwlock_init(&g.append_lock, NULL));
testutil_check(pthread_rwlock_init(&g.backup_lock, NULL));
testutil_check(pthread_rwlock_init(&g.checkpoint_lock, NULL));
testutil_check(pthread_rwlock_init(&g.death_lock, NULL));
printf("%s: process %" PRIdMAX "\n", g.progname, (intmax_t)getpid());
while (++g.run_cnt <= g.c_runs || g.c_runs == 0 ) {
startup(); /* Start a run */
config_setup(); /* Run configuration */
config_print(0); /* Dump run configuration */
key_len_setup(); /* Setup keys */
start = time(NULL);
track("starting up", 0ULL, NULL);
#ifdef HAVE_BERKELEY_DB
if (SINGLETHREADED)
bdb_open(); /* Initial file config */
#endif
wts_open(g.home, true, &g.wts_conn);
wts_init();
wts_load(); /* Load initial records */
wts_verify("post-bulk verify"); /* Verify */
/*
* If we're not doing any operations, scan the bulk-load, copy
* the statistics and we're done. Otherwise, loop reading and
* operations, with a verify after each set.
*/
if (g.c_timer == 0 && g.c_ops == 0) {
wts_read_scan(); /* Read scan */
wts_stats(); /* Statistics */
} else
for (reps = 1; reps <= FORMAT_OPERATION_REPS; ++reps) {
wts_read_scan(); /* Read scan */
/* Operations */
wts_ops(reps == FORMAT_OPERATION_REPS);
/*
* Copy out the run's statistics after the last
* set of operations.
*
* XXX
* Verify closes the underlying handle and
* discards the statistics, read them first.
*/
if (reps == FORMAT_OPERATION_REPS)
wts_stats();
/* Verify */
wts_verify("post-ops verify");
}
track("shutting down", 0ULL, NULL);
#ifdef HAVE_BERKELEY_DB
if (SINGLETHREADED)
bdb_close();
#endif
wts_close();
/*
* Rebalance testing.
*/
wts_rebalance();
/*
* If single-threaded, we can dump and compare the WiredTiger
* and Berkeley DB data sets.
*/
if (SINGLETHREADED)
wts_dump("standard", 1);
/*
* Salvage testing.
*/
wts_salvage();
/* Overwrite the progress line with a completion line. */
if (!g.c_quiet)
printf("\r%78s\r", " ");
printf("%4d: %s, %s (%.0f seconds)\n",
g.run_cnt, g.c_data_source,
g.c_file_type, difftime(time(NULL), start));
fflush(stdout);
}
/* Flush/close any logging information. */
fclose_and_clear(&g.logfp);
fclose_and_clear(&g.randfp);
config_print(0);
testutil_check(pthread_rwlock_destroy(&g.append_lock));
testutil_check(pthread_rwlock_destroy(&g.backup_lock));
testutil_check(pthread_rwlock_destroy(&g.checkpoint_lock));
testutil_check(pthread_rwlock_destroy(&g.death_lock));
config_clear();
return (EXIT_SUCCESS);
}
/*
* Child process creates the database and table, and then writes data into
* the table until it is killed by the parent.
*/
static void
fill_db(void)
{
FILE *fp;
WT_CONNECTION *conn;
WT_CURSOR *cursor;
WT_ITEM data;
WT_RAND_STATE rnd;
WT_SESSION *session;
uint64_t i;
int ret;
uint8_t buf[MAX_VAL];
__wt_random_init(&rnd);
memset(buf, 0, sizeof(buf));
/*
* Initialize the first 25% to random values. Leave a bunch of data
* space at the end to emphasize zero data.
*/
for (i = 0; i < MAX_VAL/4; i++)
buf[i] = (uint8_t)__wt_random(&rnd);
/*
* Run in the home directory so that the records file is in there too.
*/
chdir(home);
if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG, &conn)) != 0)
testutil_die(ret, "wiredtiger_open");
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
testutil_die(ret, "WT_CONNECTION:open_session");
if ((ret = session->create(session,
uri, "key_format=Q,value_format=u")) != 0)
testutil_die(ret, "WT_SESSION.create: %s", uri);
if ((ret =
session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", uri);
/*
* Keep a separate file with the records we wrote for checking.
*/
(void)unlink(RECORDS_FILE);
if ((fp = fopen(RECORDS_FILE, "w")) == NULL)
testutil_die(errno, "fopen");
/*
* Set to no buffering.
*/
setvbuf(fp, NULL, _IONBF, 0);
/*
* Write data into the table until we are killed by the parent.
* The data in the buffer is already set to random content.
*/
data.data = buf;
for (i = 0;; ++i) {
data.size = __wt_random(&rnd) % MAX_VAL;
cursor->set_key(cursor, i);
cursor->set_value(cursor, &data);
if ((ret = cursor->insert(cursor)) != 0)
testutil_die(ret, "WT_CURSOR.insert");
/*
* Save the key separately for checking later.
*/
if (fprintf(fp, "%" PRIu64 "\n", i) == -1)
testutil_die(errno, "fprintf");
if (i % 5000)
__wt_yield();
}
}
static void
run_workload(uint32_t nth)
{
WT_CONNECTION *conn;
WT_SESSION *session;
WT_THREAD_DATA *td;
wt_thread_t *thr;
uint32_t i;
int ret;
char envconf[512];
thr = dcalloc(nth+1, sizeof(*thr));
td = dcalloc(nth+1, sizeof(WT_THREAD_DATA));
if (chdir(home) != 0)
testutil_die(errno, "Child chdir: %s", home);
if (inmem)
strcpy(envconf, ENV_CONFIG_DEF);
else
strcpy(envconf, ENV_CONFIG_TXNSYNC);
if (compat)
strcat(envconf, ENV_CONFIG_COMPAT);
if ((ret = wiredtiger_open(NULL, NULL, envconf, &conn)) != 0)
testutil_die(ret, "wiredtiger_open");
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
testutil_die(ret, "WT_CONNECTION:open_session");
/*
* Create all the tables.
*/
if ((ret = session->create(session, uri_collection,
"key_format=S,value_format=u,log=(enabled=false)")) != 0)
testutil_die(ret, "WT_SESSION.create: %s", uri_collection);
if ((ret = session->create(session,
uri_local, "key_format=S,value_format=u")) != 0)
testutil_die(ret, "WT_SESSION.create: %s", uri_local);
if ((ret = session->create(session,
uri_oplog, "key_format=S,value_format=u")) != 0)
testutil_die(ret, "WT_SESSION.create: %s", uri_oplog);
/*
* Don't log the stable timestamp table so that we know what timestamp
* was stored at the checkpoint.
*/
if ((ret = session->create(session, stable_store,
"key_format=Q,value_format=Q,log=(enabled=false)")) != 0)
testutil_die(ret, "WT_SESSION.create: %s", stable_store);
if ((ret = session->close(session, NULL)) != 0)
testutil_die(ret, "WT_SESSION:close");
/*
* Thread 0 is the checkpoint thread.
*/
td[0].conn = conn;
td[0].id = 0;
printf("Create checkpoint thread\n");
testutil_check(__wt_thread_create(
NULL, &thr[0], thread_ckpt_run, &td[0]));
for (i = 1; i <= nth; ++i) {
td[i].conn = conn;
td[i].start = (UINT64_MAX / nth) * (i - 1);
td[i].id = i;
testutil_check(__wt_thread_create(
NULL, &thr[i], thread_run, &td[i]));
}
/*
* The threads never exit, so the child will just wait here until
* it is killed.
*/
printf("Create %" PRIu32 " writer threads\n", nth);
fflush(stdout);
for (i = 0; i <= nth; ++i)
testutil_check(__wt_thread_join(NULL, thr[i]));
/*
* NOTREACHED
*/
free(thr);
free(td);
exit(EXIT_SUCCESS);
}
/*
* nextprev --
* Read and verify the next/prev element in a row- or column-store file.
*/
static int
nextprev(WT_CURSOR *cursor, int next, int *notfoundp)
{
WT_ITEM key, value;
uint64_t keyno;
int ret;
uint8_t bitfield;
const char *which;
which = next ? "next" : "prev";
keyno = 0;
ret = next ? cursor->next(cursor) : cursor->prev(cursor);
if (ret == WT_ROLLBACK)
return (WT_ROLLBACK);
if (ret == 0)
switch (g.type) {
case FIX:
if ((ret = cursor->get_key(cursor, &keyno)) == 0 &&
(ret = cursor->get_value(cursor, &bitfield)) == 0) {
value.data = &bitfield;
value.size = 1;
}
break;
case ROW:
if ((ret = cursor->get_key(cursor, &key)) == 0)
ret = cursor->get_value(cursor, &value);
break;
case VAR:
if ((ret = cursor->get_key(cursor, &keyno)) == 0)
ret = cursor->get_value(cursor, &value);
break;
}
if (ret != 0 && ret != WT_NOTFOUND)
testutil_die(ret, "%s", which);
*notfoundp = (ret == WT_NOTFOUND);
#ifdef HAVE_BERKELEY_DB
if (!SINGLETHREADED)
return (0);
{
WT_ITEM bdb_key, bdb_value;
WT_SESSION *session;
int notfound;
char *p;
session = cursor->session;
/* Retrieve the BDB value. */
bdb_np(next, &bdb_key.data, &bdb_key.size,
&bdb_value.data, &bdb_value.size, ¬found);
if (notfound_chk(
next ? "nextprev(next)" : "nextprev(prev)", ret, notfound, keyno))
return (0);
/* Compare the two. */
if (g.type == ROW) {
if (key.size != bdb_key.size ||
memcmp(key.data, bdb_key.data, key.size) != 0) {
fprintf(stderr, "nextprev: %s key mismatch:\n", which);
print_item("bdb-key", &bdb_key);
print_item(" wt-key", &key);
testutil_die(0, NULL);
}
} else {
if (keyno != (uint64_t)atoll(bdb_key.data)) {
if ((p = strchr((char *)bdb_key.data, '.')) != NULL)
*p = '\0';
fprintf(stderr,
"nextprev: %s key mismatch: %.*s != %" PRIu64 "\n",
which,
(int)bdb_key.size, (char *)bdb_key.data, keyno);
testutil_die(0, NULL);
}
}
if (value.size != bdb_value.size ||
memcmp(value.data, bdb_value.data, value.size) != 0) {
fprintf(stderr, "nextprev: %s value mismatch:\n", which);
print_item("bdb-value", &bdb_value);
print_item(" wt-value", &value);
testutil_die(0, NULL);
}
if (g.logging == LOG_OPS)
switch (g.type) {
case FIX:
(void)g.wt_api->msg_printf(g.wt_api,
session, "%-10s%" PRIu64 " {0x%02x}", which,
keyno, ((char *)value.data)[0]);
break;
case ROW:
(void)g.wt_api->msg_printf(
g.wt_api, session, "%-10s{%.*s/%.*s}", which,
(int)key.size, (char *)key.data,
(int)value.size, (char *)value.data);
break;
case VAR:
(void)g.wt_api->msg_printf(g.wt_api, session,
"%-10s%" PRIu64 " {%.*s}", which,
keyno, (int)value.size, (char *)value.data);
break;
}
}
#endif
return (0);
}
/*
* backup --
* Periodically do a backup and verify it.
*/
void *
backup(void *arg)
{
WT_CONNECTION *conn;
WT_CURSOR *backup_cursor;
WT_SESSION *session;
u_int period;
int ret;
const char *key;
(void)(arg);
conn = g.wts_conn;
/* Backups aren't supported for non-standard data sources. */
if (DATASOURCE("helium") || DATASOURCE("kvsbdb"))
return (NULL);
/* Open a session. */
testutil_check(conn->open_session(conn, NULL, NULL, &session));
/*
* Perform a backup at somewhere under 10 seconds (so we get at
* least one done), and then at 45 second intervals.
*/
for (period = mmrand(NULL, 1, 10);; period = 45) {
/* Sleep for short periods so we don't make the run wait. */
while (period > 0 && !g.workers_finished) {
--period;
sleep(1);
}
if (g.workers_finished)
break;
/* Lock out named checkpoints */
testutil_check(pthread_rwlock_wrlock(&g.backup_lock));
/* Re-create the backup directory. */
testutil_checkfmt(
system(g.home_backup_init),
"%s", "backup directory creation failed");
/*
* open_cursor can return EBUSY if a metadata operation is
* currently happening - retry in that case.
*/
while ((ret = session->open_cursor(session,
"backup:", NULL, NULL, &backup_cursor)) == EBUSY)
sleep(1);
if (ret != 0)
testutil_die(ret, "session.open_cursor: backup");
while ((ret = backup_cursor->next(backup_cursor)) == 0) {
testutil_check(
backup_cursor->get_key(backup_cursor, &key));
copy_file(key);
}
testutil_check(backup_cursor->close(backup_cursor));
testutil_check(pthread_rwlock_unlock(&g.backup_lock));
check_copy();
}
testutil_check(session->close(session, NULL));
return (NULL);
}
int
main(int argc, char *argv[])
{
table_type ttype;
int ch, cnt, ret, runs;
char *working_dir;
const char *config_open;
if ((g.progname = strrchr(argv[0], DIR_DELIM)) == NULL)
g.progname = argv[0];
else
++g.progname;
config_open = NULL;
ret = 0;
working_dir = NULL;
ttype = MIX;
g.checkpoint_name = "WiredTigerCheckpoint";
if ((g.home = malloc(512)) == NULL)
testutil_die(ENOMEM, "Unable to allocate memory");
g.nkeys = 10000;
g.nops = 100000;
g.ntables = 3;
g.nworkers = 1;
runs = 1;
while ((ch = __wt_getopt(
g.progname, argc, argv, "c:C:h:k:l:n:r:t:T:W:")) != EOF)
switch (ch) {
case 'c':
g.checkpoint_name = __wt_optarg;
break;
case 'C': /* wiredtiger_open config */
config_open = __wt_optarg;
break;
case 'h': /* wiredtiger_open config */
working_dir = __wt_optarg;
break;
case 'k': /* rows */
g.nkeys = (u_int)atoi(__wt_optarg);
break;
case 'l': /* log */
if ((g.logfp = fopen(__wt_optarg, "w")) == NULL) {
fprintf(stderr,
"%s: %s\n", __wt_optarg, strerror(errno));
return (EXIT_FAILURE);
}
break;
case 'n': /* operations */
g.nops = (u_int)atoi(__wt_optarg);
break;
case 'r': /* runs */
runs = atoi(__wt_optarg);
break;
case 't':
switch (__wt_optarg[0]) {
case 'c':
ttype = COL;
break;
case 'l':
ttype = LSM;
break;
case 'm':
ttype = MIX;
break;
case 'r':
ttype = ROW;
break;
default:
return (usage());
}
break;
case 'T':
g.ntables = atoi(__wt_optarg);
break;
case 'W':
g.nworkers = atoi(__wt_optarg);
break;
default:
return (usage());
}
argc -= __wt_optind;
if (argc != 0)
return (usage());
/* Clean up on signal. */
(void)signal(SIGINT, onint);
testutil_work_dir_from_path(g.home, 512, working_dir);
printf("%s: process %" PRIu64 "\n", g.progname, (uint64_t)getpid());
for (cnt = 1; (runs == 0 || cnt <= runs) && g.status == 0; ++cnt) {
printf(" %d: %u workers, %u tables\n",
cnt, g.nworkers, g.ntables);
(void)cleanup(); /* Clean up previous runs */
/* Setup a fresh set of cookies in the global array. */
if ((g.cookies = calloc(
(size_t)(g.ntables), sizeof(COOKIE))) == NULL) {
(void)log_print_err("No memory", ENOMEM, 1);
break;
}
g.running = 1;
if ((ret = wt_connect(config_open)) != 0) {
(void)log_print_err("Connection failed", ret, 1);
break;
}
if ((ret = start_checkpoints()) != 0) {
(void)log_print_err("Start checkpoints failed", ret, 1);
break;
}
if ((ret = start_workers(ttype)) != 0) {
(void)log_print_err("Start workers failed", ret, 1);
break;
}
g.running = 0;
if ((ret = end_checkpoints()) != 0) {
(void)log_print_err("Start workers failed", ret, 1);
break;
}
free(g.cookies);
g.cookies = NULL;
if ((ret = wt_shutdown()) != 0) {
(void)log_print_err("Start workers failed", ret, 1);
break;
}
}
if (g.logfp != NULL)
(void)fclose(g.logfp);
/* Ensure that cleanup is done on error. */
(void)wt_shutdown();
free(g.cookies);
return (g.status);
}
int
main(int argc, char *argv[])
{
FILE *fp;
WT_CONNECTION *conn;
WT_CURSOR *cursor;
WT_SESSION *session;
pid_t pid;
uint64_t new_offset, offset;
uint32_t count, max_key;
int ch, ret, status;
const char *working_dir;
(void)testutil_set_progname(argv);
working_dir = "WT_TEST.truncated-log";
while ((ch = __wt_getopt(progname, argc, argv, "h:")) != EOF)
switch (ch) {
case 'h':
working_dir = __wt_optarg;
break;
default:
usage();
}
argc -= __wt_optind;
if (argc != 0)
usage();
testutil_work_dir_from_path(home, sizeof(home), working_dir);
testutil_make_work_dir(home);
/*
* Fork a child to do its work. Wait for it to exit.
*/
if ((pid = fork()) < 0)
testutil_die(errno, "fork");
if (pid == 0) { /* child */
fill_db();
return (EXIT_SUCCESS);
}
/* parent */
/* Wait for child to kill itself. */
if (waitpid(pid, &status, 0) == -1)
testutil_die(errno, "waitpid");
/*
* !!! If we wanted to take a copy of the directory before recovery,
* this is the place to do it.
*/
if (chdir(home) != 0)
testutil_die(errno, "chdir: %s", home);
printf("Open database, run recovery and verify content\n");
if ((fp = fopen(RECORDS_FILE, "r")) == NULL)
testutil_die(errno, "fopen");
ret = fscanf(fp, "%" SCNu64 " %" SCNu32 "\n", &offset, &max_key);
if (ret != 2)
testutil_die(errno, "fscanf");
if (fclose(fp) != 0)
testutil_die(errno, "fclose");
/*
* The offset is the beginning of the last record. Truncate to
* the middle of that last record (i.e. ahead of that offset).
*/
if (offset > UINT64_MAX - V_SIZE)
testutil_die(ERANGE, "offset");
new_offset = offset + V_SIZE;
printf("Parent: Log file 1: Key %" PRIu32 " at %" PRIu64 "\n",
max_key, offset);
printf("Parent: Truncate mid-record to %" PRIu64 "\n", new_offset);
if (truncate(LOG_FILE_1, (wt_off_t)new_offset) != 0)
testutil_die(errno, "truncate");
testutil_check(wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn));
testutil_check(conn->open_session(conn, NULL, NULL, &session));
testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor));
/*
* For every key in the saved file, verify that the key exists
* in the table after recovery. Since we did write-no-sync, we
* expect every key to have been recovered.
*/
count = 0;
while (cursor->next(cursor) == 0)
++count;
/*
* The max key in the saved file is the key we truncated, but the
* key space starts at 0 and we're counting the records here, so we
* expect the max key number of records. Add one for the system
* record for the previous LSN that the cursor will see too.
*/
if (count > (max_key + 1)) {
printf("expected %" PRIu32 " records found %" PRIu32 "\n",
max_key, count);
return (EXIT_FAILURE);
}
printf("%" PRIu32 " records verified\n", count);
/*
* Write a log record and then walk the log to make sure we can
* read that log record that is beyond the truncated record.
*/
write_and_read_new(session);
testutil_check(conn->close(conn, NULL));
return (EXIT_SUCCESS);
}
static void
fill_db(void)
{
FILE *fp;
WT_CONNECTION *conn;
WT_CURSOR *cursor, *logc;
WT_LSN lsn, save_lsn;
WT_SESSION *session;
uint32_t i, max_key, min_key, units, unused;
char k[K_SIZE], v[V_SIZE];
bool first;
/*
* Run in the home directory so that the records file is in there too.
*/
if (chdir(home) != 0)
testutil_die(errno, "chdir: %s", home);
testutil_check(wiredtiger_open(NULL, NULL, ENV_CONFIG, &conn));
testutil_check(conn->open_session(conn, NULL, NULL, &session));
testutil_check(
session->create(session, uri, "key_format=S,value_format=S"));
testutil_check(session->open_cursor(session, uri, NULL, NULL, &cursor));
/*
* Keep a separate file with the records we wrote for checking.
*/
(void)unlink(RECORDS_FILE);
if ((fp = fopen(RECORDS_FILE, "w")) == NULL)
testutil_die(errno, "fopen");
/*
* Set to no buffering.
*/
__wt_stream_set_no_buffer(fp);
save_lsn.l.file = 0;
/*
* Write data into the table until we move to log file 2.
* We do the calculation below so that we don't have to walk the
* log for every record.
*
* Calculate about how many records should fit in the log file.
* Subtract a bunch for metadata and file creation records.
* Then subtract out a few more records to be conservative.
*/
units = (K_SIZE + V_SIZE) / 128 + 1;
min_key = 90000 / (units * 128) - 15;
max_key = min_key * 2;
first = true;
for (i = 0; i < max_key; ++i) {
testutil_check(__wt_snprintf(k, sizeof(k), "key%03d", (int)i));
testutil_check(__wt_snprintf(v, sizeof(v), "value%0*d",
(int)(V_SIZE - (strlen("value") + 1)), (int)i));
cursor->set_key(cursor, k);
cursor->set_value(cursor, v);
testutil_check(cursor->insert(cursor));
/*
* Walking the ever growing log can be slow, so only start
* looking for the cross into log file 2 after a minimum.
*/
if (i > min_key) {
testutil_check(session->open_cursor(
session, "log:", NULL, NULL, &logc));
if (save_lsn.l.file != 0) {
logc->set_key(logc,
save_lsn.l.file, save_lsn.l.offset, 0);
testutil_check(logc->search(logc));
}
while (logc->next(logc) == 0) {
testutil_check(logc->get_key(
logc, &lsn.l.file, &lsn.l.offset, &unused));
/*
* Save the LSN so that we know the offset
* of the last LSN in log file 1 later.
*/
if (lsn.l.file < 2)
save_lsn = lsn;
else {
/*
* If this is the first time through
* that the key is larger than the
* minimum key and we're already in
* log file 2 then we did not calculate
* correctly and the test should fail.
*/
if (first)
testutil_die(EINVAL,
"min_key too high");
if (fprintf(fp,
"%" PRIu32 " %" PRIu32 "\n",
save_lsn.l.offset, i - 1) == -1)
testutil_die(errno, "fprintf");
break;
}
}
first = false;
testutil_check(logc->close(logc));
}
}
if (fclose(fp) != 0)
testutil_die(errno, "fclose");
exit(0);
/* NOTREACHED */
}
int
main(int argc, char *argv[])
{
WT_CONNECTION *conn, *conn2, *conn3, *conn4;
WT_CURSOR *cursor;
WT_ITEM data;
WT_SESSION *session;
uint64_t i;
int ch, status, op, ret;
bool child;
const char *working_dir;
char cmd[512];
uint8_t buf[MAX_VAL];
if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL)
progname = argv[0];
else
++progname;
/*
* Needed unaltered for system command later.
*/
saved_argv0 = argv[0];
working_dir = "WT_RD";
child = false;
op = OP_READ;
while ((ch = __wt_getopt(progname, argc, argv, "Rh:W")) != EOF)
switch (ch) {
case 'R':
child = true;
op = OP_READ;
break;
case 'W':
child = true;
op = OP_WRITE;
break;
case 'h':
working_dir = __wt_optarg;
break;
default:
usage();
}
argc -= __wt_optind;
argv += __wt_optind;
if (argc != 0)
usage();
/*
* Set up all the directory names.
*/
testutil_work_dir_from_path(home, sizeof(home), working_dir);
(void)snprintf(home_wr, sizeof(home_wr), "%s%s", home, HOME_WR_SUFFIX);
(void)snprintf(home_rd, sizeof(home_rd), "%s%s", home, HOME_RD_SUFFIX);
(void)snprintf(
home_rd2, sizeof(home_rd2), "%s%s", home, HOME_RD2_SUFFIX);
if (!child) {
testutil_make_work_dir(home);
testutil_make_work_dir(home_wr);
testutil_make_work_dir(home_rd);
testutil_make_work_dir(home_rd2);
} else
/*
* We are a child process, we just want to call
* the open_dbs with the directories we have.
* The child function will exit.
*/
open_dbs(op, home, home_wr, home_rd, home_rd2);
/*
* Parent creates a database and table. Then cleanly shuts down.
* Then copy database to read-only directory and chmod.
* Also copy database to read-only directory and remove the lock
* file. One read-only database will have a lock file in the
* file system and the other will not.
* Parent opens all databases with read-only configuration flag.
* Parent forks off child who tries to also open all databases
* with the read-only flag. It should error on the writeable
* directory, but allow it on the read-only directories.
* The child then confirms it can read all the data.
*/
/*
* Run in the home directory and create the table.
*/
if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG, &conn)) != 0)
testutil_die(ret, "wiredtiger_open");
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
testutil_die(ret, "WT_CONNECTION:open_session");
if ((ret = session->create(session,
uri, "key_format=Q,value_format=u")) != 0)
testutil_die(ret, "WT_SESSION.create: %s", uri);
if ((ret =
session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", uri);
/*
* Write data into the table and then cleanly shut down connection.
*/
memset(buf, 0, sizeof(buf));
data.data = buf;
data.size = MAX_VAL;
for (i = 0; i < MAX_KV; ++i) {
cursor->set_key(cursor, i);
cursor->set_value(cursor, &data);
if ((ret = cursor->insert(cursor)) != 0)
testutil_die(ret, "WT_CURSOR.insert");
}
if ((ret = conn->close(conn, NULL)) != 0)
testutil_die(ret, "WT_CONNECTION:close");
/*
* Copy the database. Remove any lock file from one copy
* and chmod the copies to be read-only permissions.
*/
(void)snprintf(cmd, sizeof(cmd),
"cp -rp %s/* %s; rm -f %s/WiredTiger.lock",
home, home_wr, home_wr);
if ((status = system(cmd)) < 0)
testutil_die(status, "system: %s", cmd);
(void)snprintf(cmd, sizeof(cmd),
"cp -rp %s/* %s; chmod 0555 %s; chmod -R 0444 %s/*",
home, home_rd, home_rd, home_rd);
if ((status = system(cmd)) < 0)
testutil_die(status, "system: %s", cmd);
(void)snprintf(cmd, sizeof(cmd),
"cp -rp %s/* %s; rm -f %s/WiredTiger.lock; "
"chmod 0555 %s; chmod -R 0444 %s/*",
home, home_rd2, home_rd2, home_rd2, home_rd2);
if ((status = system(cmd)) < 0)
testutil_die(status, "system: %s", cmd);
/*
* Run four scenarios. Sometimes expect errors, sometimes success.
* The writable database directories should always fail to allow the
* child to open due to the lock file. The read-only ones will only
* succeed when the child attempts read-only.
*
* 1. Parent has read-only handle to all databases. Child opens
* read-only also.
* 2. Parent has read-only handle to all databases. Child opens
* read-write.
* 3. Parent has read-write handle to writable databases and
* read-only to read-only databases. Child opens read-only.
* 4. Parent has read-write handle to writable databases and
* read-only to read-only databases. Child opens read-write.
*/
/*
* Open a connection handle to all databases.
*/
fprintf(stderr, " *** Expect several error messages from WT ***\n");
/*
* Scenario 1.
*/
if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG_RD, &conn)) != 0)
testutil_die(ret, "wiredtiger_open original home");
if ((ret = wiredtiger_open(home_wr, NULL, ENV_CONFIG_RD, &conn2)) != 0)
testutil_die(ret, "wiredtiger_open write nolock");
if ((ret = wiredtiger_open(home_rd, NULL, ENV_CONFIG_RD, &conn3)) != 0)
testutil_die(ret, "wiredtiger_open readonly");
if ((ret = wiredtiger_open(home_rd2, NULL, ENV_CONFIG_RD, &conn4)) != 0)
testutil_die(ret, "wiredtiger_open readonly nolock");
/*
* Create a child to also open a connection handle to the databases.
* We cannot use fork here because using fork the child inherits the
* same memory image. Therefore the WT process structure is set in
* the child even though it should not be. So use 'system' to spawn
* an entirely new process.
*
* The child will exit with success if its test passes.
*/
(void)snprintf(
cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir);
if ((status = system(cmd)) < 0)
testutil_die(status, "system: %s", cmd);
if (WEXITSTATUS(status) != 0)
testutil_die(WEXITSTATUS(status), "system: %s", cmd);
/*
* Scenario 2. Run child with writable config.
*/
(void)snprintf(
cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir);
if ((status = system(cmd)) < 0)
testutil_die(status, "system: %s", cmd);
if (WEXITSTATUS(status) != 0)
testutil_die(WEXITSTATUS(status), "system: %s", cmd);
/*
* Reopen the two writable directories and rerun the child.
*/
if ((ret = conn->close(conn, NULL)) != 0)
testutil_die(ret, "WT_CONNECTION:close");
if ((ret = conn2->close(conn2, NULL)) != 0)
testutil_die(ret, "WT_CONNECTION:close");
if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG_RD, &conn)) != 0)
testutil_die(ret, "wiredtiger_open original home");
if ((ret = wiredtiger_open(home_wr, NULL, ENV_CONFIG_RD, &conn2)) != 0)
testutil_die(ret, "wiredtiger_open write nolock");
/*
* Scenario 3. Child read-only.
*/
(void)snprintf(
cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir);
if ((status = system(cmd)) < 0)
testutil_die(status, "system: %s", cmd);
if (WEXITSTATUS(status) != 0)
testutil_die(WEXITSTATUS(status), "system: %s", cmd);
/*
* Scenario 4. Run child with writable config.
*/
(void)snprintf(
cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir);
if ((status = system(cmd)) < 0)
testutil_die(status, "system: %s", cmd);
if (WEXITSTATUS(status) != 0)
testutil_die(WEXITSTATUS(status), "system: %s", cmd);
/*
* Clean-up.
*/
if ((ret = conn->close(conn, NULL)) != 0)
testutil_die(ret, "WT_CONNECTION:close");
if ((ret = conn2->close(conn2, NULL)) != 0)
testutil_die(ret, "WT_CONNECTION:close");
if ((ret = conn3->close(conn3, NULL)) != 0)
testutil_die(ret, "WT_CONNECTION:close");
if ((ret = conn4->close(conn4, NULL)) != 0)
testutil_die(ret, "WT_CONNECTION:close");
/*
* We need to chmod the read-only databases back so that they can
* be removed by scripts.
*/
(void)snprintf(cmd, sizeof(cmd), "chmod 0777 %s %s", home_rd, home_rd2);
if ((status = system(cmd)) < 0)
testutil_die(status, "system: %s", cmd);
(void)snprintf(cmd, sizeof(cmd), "chmod -R 0666 %s/* %s/*",
home_rd, home_rd2);
if ((status = system(cmd)) < 0)
testutil_die(status, "system: %s", cmd);
printf(" *** Readonly test successful ***\n");
return (EXIT_SUCCESS);
}
int
main(int argc, char *argv[])
{
/*
* Add a bunch of tables so that some of the metadata ends up on
* other pages and a good number of tables are available after
* salvage completes.
*/
TABLE_INFO table_data[] = {
{ "file:aaa-file.SS", "key_format=S,value_format=S", false },
{ "file:bbb-file.rS", "key_format=r,value_format=S", false },
{ "lsm:ccc-lsm.SS", "key_format=S,value_format=S", false },
{ "table:ddd-table.SS", "key_format=S,value_format=S", false },
{ "table:eee-table.rS", "key_format=r,value_format=S", false },
{ "file:fff-file.SS", "key_format=S,value_format=S", false },
{ "file:ggg-file.rS", "key_format=r,value_format=S", false },
{ "lsm:hhh-lsm.SS", "key_format=S,value_format=S", false },
{ "table:iii-table.SS", "key_format=S,value_format=S", false },
{ "table:jjj-table.rS", "key_format=r,value_format=S", false },
{ CORRUPT, "key_format=S,value_format=S", false },
{ NULL, NULL, false }
};
TABLE_INFO *t;
TEST_OPTS *opts, _opts;
int ret;
char buf[1024];
opts = &_opts;
memset(opts, 0, sizeof(*opts));
testutil_check(testutil_parse_opts(argc, argv, opts));
/*
* Set a global. We use this everywhere.
*/
home = opts->home;
testutil_make_work_dir(home);
testutil_check(
wiredtiger_open(home, &event_handler, "create", &opts->conn));
testutil_check(opts->conn->open_session(
opts->conn, NULL, NULL, &wt_session));
/*
* Create a bunch of different tables.
*/
for (t = table_data; t->name != NULL; t++)
create_data(t);
/*
* Take some checkpoints and add more data for out of sync testing.
*/
make_database_copies(table_data);
testutil_check(opts->conn->close(opts->conn, NULL));
opts->conn = NULL;
/*
* Make copy of original directory.
*/
copy_database(SAVE);
/*
* Damage/corrupt WiredTiger.wt.
*/
printf("corrupt metadata\n");
corrupt_metadata();
testutil_check(__wt_snprintf(buf, sizeof(buf),
"cp -p %s/WiredTiger.wt ./%s.SAVE/WiredTiger.wt.CORRUPT",
home, home));
printf("copy: %s\n", buf);
if ((ret = system(buf)) < 0)
testutil_die(ret, "system: %s", buf);
run_all_verification(NULL, &table_data[0]);
out_of_sync(&table_data[0]);
/*
* We need to set up the string before we clean up
* the structure. Then after the clean up we will
* run this command.
*/
testutil_check(__wt_snprintf(buf, sizeof(buf),
"rm -rf core* %s*", home));
testutil_cleanup(opts);
/*
* We've created a lot of extra directories and possibly some core
* files from child process aborts. Manually clean them up.
*/
printf("cleanup and remove: %s\n", buf);
if ((ret = system(buf)) < 0)
testutil_die(ret, "system: %s", buf);
return (EXIT_SUCCESS);
}
int
main(int argc, char *argv[])
{
FILE *fp;
WT_CONNECTION *conn;
WT_CURSOR *cursor;
WT_SESSION *session;
WT_RAND_STATE rnd;
uint64_t key;
uint32_t absent, count, timeout;
int ch, status, ret;
pid_t pid;
char *working_dir;
if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL)
progname = argv[0];
else
++progname;
working_dir = NULL;
timeout = 10;
while ((ch = __wt_getopt(progname, argc, argv, "h:t:")) != EOF)
switch (ch) {
case 'h':
working_dir = __wt_optarg;
break;
case 't':
timeout = (uint32_t)atoi(__wt_optarg);
break;
default:
usage();
}
argc -= __wt_optind;
argv += __wt_optind;
if (argc != 0)
usage();
testutil_work_dir_from_path(home, 512, working_dir);
testutil_make_work_dir(home);
/*
* Fork a child to insert as many items. We will then randomly
* kill the child, run recovery and make sure all items we wrote
* exist after recovery runs.
*/
if ((pid = fork()) < 0)
testutil_die(errno, "fork");
if (pid == 0) { /* child */
fill_db();
return (EXIT_SUCCESS);
}
/* parent */
__wt_random_init(&rnd);
/* Sleep for the configured amount of time before killing the child. */
printf("Parent: sleep %" PRIu32 " seconds, then kill child\n", timeout);
sleep(timeout);
/*
* !!! It should be plenty long enough to make sure more than one
* log file exists. If wanted, that check would be added here.
*/
printf("Kill child\n");
if (kill(pid, SIGKILL) != 0)
testutil_die(errno, "kill");
waitpid(pid, &status, 0);
/*
* !!! If we wanted to take a copy of the directory before recovery,
* this is the place to do it.
*/
chdir(home);
printf("Open database, run recovery and verify content\n");
if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn)) != 0)
testutil_die(ret, "wiredtiger_open");
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
testutil_die(ret, "WT_CONNECTION:open_session");
if ((ret =
session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", uri);
if ((fp = fopen(RECORDS_FILE, "r")) == NULL)
testutil_die(errno, "fopen");
/*
* For every key in the saved file, verify that the key exists
* in the table after recovery. Since we did write-no-sync, we
* expect every key to have been recovered.
*/
for (absent = count = 0;; ++count) {
ret = fscanf(fp, "%" SCNu64 "\n", &key);
if (ret != EOF && ret != 1)
testutil_die(errno, "fscanf");
if (ret == EOF)
break;
cursor->set_key(cursor, key);
if ((ret = cursor->search(cursor)) != 0) {
if (ret != WT_NOTFOUND)
testutil_die(ret, "search");
printf("no record with key %" PRIu64 "\n", key);
++absent;
}
}
fclose(fp);
if ((ret = conn->close(conn, NULL)) != 0)
testutil_die(ret, "WT_CONNECTION:close");
if (absent) {
printf("%u record(s) absent from %u\n", absent, count);
return (EXIT_FAILURE);
}
printf("%u records verified\n", count);
return (EXIT_SUCCESS);
}
/*
* thread_run --
* Runner function for the worker threads.
*/
static WT_THREAD_RET
thread_run(void *arg)
{
FILE *fp;
WT_CURSOR *cur_coll, *cur_local, *cur_oplog;
WT_ITEM data;
WT_RAND_STATE rnd;
WT_SESSION *prepared_session, *session;
THREAD_DATA *td;
uint64_t i, active_ts;
char cbuf[MAX_VAL], lbuf[MAX_VAL], obuf[MAX_VAL];
char kname[64], tscfg[64], uri[128];
bool use_prep;
__wt_random_init(&rnd);
memset(cbuf, 0, sizeof(cbuf));
memset(lbuf, 0, sizeof(lbuf));
memset(obuf, 0, sizeof(obuf));
memset(kname, 0, sizeof(kname));
prepared_session = NULL;
td = (THREAD_DATA *)arg;
/*
* Set up the separate file for checking.
*/
testutil_check(__wt_snprintf(
cbuf, sizeof(cbuf), RECORDS_FILE, td->info));
(void)unlink(cbuf);
testutil_checksys((fp = fopen(cbuf, "w")) == NULL);
/*
* Set to line buffering. But that is advisory only. We've seen
* cases where the result files end up with partial lines.
*/
__wt_stream_set_line_buffer(fp);
/*
* Have 10% of the threads use prepared transactions if timestamps
* are in use. Thread numbers start at 0 so we're always guaranteed
* that at least one thread is using prepared transactions.
*/
use_prep = (use_ts && td->info % PREPARE_PCT == 0) ? true : false;
/*
* For the prepared case we have two sessions so that the oplog session
* can have its own transaction in parallel with the collection session
* We need this because prepared transactions cannot have any operations
* that modify a table that is logged. But we also want to test mixed
* logged and not-logged transactions.
*/
testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session));
if (use_prep)
testutil_check(td->conn->open_session(
td->conn, NULL, NULL, &prepared_session));
/*
* Open a cursor to each table.
*/
testutil_check(__wt_snprintf(
uri, sizeof(uri), "%s:%s", table_pfx, uri_collection));
if (use_prep)
testutil_check(prepared_session->open_cursor(prepared_session,
uri, NULL, NULL, &cur_coll));
else
testutil_check(session->open_cursor(session,
uri, NULL, NULL, &cur_coll));
testutil_check(__wt_snprintf(
uri, sizeof(uri), "%s:%s", table_pfx, uri_local));
if (use_prep)
testutil_check(prepared_session->open_cursor(prepared_session,
uri, NULL, NULL, &cur_local));
else
testutil_check(session->open_cursor(session,
uri, NULL, NULL, &cur_local));
testutil_check(__wt_snprintf(
uri, sizeof(uri), "%s:%s", table_pfx, uri_oplog));
testutil_check(session->open_cursor(session,
uri, NULL, NULL, &cur_oplog));
/*
* Write our portion of the key space until we're killed.
*/
printf("Thread %" PRIu32 " starts at %" PRIu64 "\n",
td->info, td->start);
active_ts = 0;
for (i = td->start;; ++i) {
testutil_check(__wt_snprintf(
kname, sizeof(kname), "%" PRIu64, i));
testutil_check(session->begin_transaction(session, NULL));
if (use_prep)
testutil_check(prepared_session->begin_transaction(
prepared_session, NULL));
if (use_ts) {
testutil_check(pthread_rwlock_rdlock(&ts_lock));
active_ts = __wt_atomic_addv64(&global_ts, 1);
testutil_check(__wt_snprintf(tscfg,
sizeof(tscfg), "commit_timestamp=%" PRIx64,
active_ts));
/*
* Set the transaction's timestamp now before performing
* the operation. If we are using prepared transactions,
* set the timestamp for the session used for oplog. The
* collection session in that case would continue to use
* this timestamp.
*/
testutil_check(session->timestamp_transaction(
session, tscfg));
testutil_check(pthread_rwlock_unlock(&ts_lock));
}
cur_coll->set_key(cur_coll, kname);
cur_local->set_key(cur_local, kname);
cur_oplog->set_key(cur_oplog, kname);
/*
* Put an informative string into the value so that it
* can be viewed well in a binary dump.
*/
testutil_check(__wt_snprintf(cbuf, sizeof(cbuf),
"COLL: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
td->info, active_ts, i));
testutil_check(__wt_snprintf(lbuf, sizeof(lbuf),
"LOCAL: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
td->info, active_ts, i));
testutil_check(__wt_snprintf(obuf, sizeof(obuf),
"OPLOG: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
td->info, active_ts, i));
data.size = __wt_random(&rnd) % MAX_VAL;
data.data = cbuf;
cur_coll->set_value(cur_coll, &data);
testutil_check(cur_coll->insert(cur_coll));
data.size = __wt_random(&rnd) % MAX_VAL;
data.data = obuf;
cur_oplog->set_value(cur_oplog, &data);
testutil_check(cur_oplog->insert(cur_oplog));
if (use_prep) {
/*
* Run with prepare every once in a while. And also
* yield after prepare sometimes too. This is only done
* on the collection session.
*/
if (i % PREPARE_FREQ == 0) {
testutil_check(__wt_snprintf(tscfg,
sizeof(tscfg), "prepare_timestamp=%"
PRIx64, active_ts));
testutil_check(
prepared_session->prepare_transaction(
prepared_session, tscfg));
if (i % PREPARE_YIELD == 0)
__wt_yield();
testutil_check(
__wt_snprintf(tscfg, sizeof(tscfg),
"commit_timestamp=%" PRIx64
",durable_timestamp=%" PRIx64,
active_ts, active_ts));
} else
testutil_check(
__wt_snprintf(tscfg, sizeof(tscfg),
"commit_timestamp=%" PRIx64, active_ts));
testutil_check(
prepared_session->commit_transaction(
prepared_session, tscfg));
}
testutil_check(
session->commit_transaction(session, NULL));
/*
* Insert into the local table outside the timestamp txn.
*/
data.size = __wt_random(&rnd) % MAX_VAL;
data.data = lbuf;
cur_local->set_value(cur_local, &data);
testutil_check(cur_local->insert(cur_local));
/*
* Save the timestamp and key separately for checking later.
*/
if (fprintf(fp,
"%" PRIu64 " %" PRIu64 "\n", active_ts, i) < 0)
testutil_die(EIO, "fprintf");
}
/* NOTREACHED */
}
/*
* thread_run --
* Runner function for the worker threads.
*/
static WT_THREAD_RET
thread_run(void *arg)
{
FILE *fp;
WT_CURSOR *cur_coll, *cur_local, *cur_oplog, *cur_stable;
WT_ITEM data;
WT_RAND_STATE rnd;
WT_SESSION *session;
WT_THREAD_DATA *td;
uint64_t i, stable_ts;
int ret;
char cbuf[MAX_VAL], lbuf[MAX_VAL], obuf[MAX_VAL];
char kname[64], tscfg[64];
__wt_random_init(&rnd);
memset(cbuf, 0, sizeof(cbuf));
memset(lbuf, 0, sizeof(lbuf));
memset(obuf, 0, sizeof(obuf));
memset(kname, 0, sizeof(kname));
td = (WT_THREAD_DATA *)arg;
/*
* Set up the separate file for checking.
*/
testutil_check(__wt_snprintf(cbuf, sizeof(cbuf), RECORDS_FILE, td->id));
(void)unlink(cbuf);
testutil_checksys((fp = fopen(cbuf, "w")) == NULL);
/*
* Set to line buffering. But that is advisory only. We've seen
* cases where the result files end up with partial lines.
*/
__wt_stream_set_line_buffer(fp);
if ((ret = td->conn->open_session(td->conn, NULL, NULL, &session)) != 0)
testutil_die(ret, "WT_CONNECTION:open_session");
/*
* Open a cursor to each table.
*/
if ((ret = session->open_cursor(session,
uri_collection, NULL, NULL, &cur_coll)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_collection);
if ((ret = session->open_cursor(session,
uri_local, NULL, NULL, &cur_local)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_local);
if ((ret = session->open_cursor(session,
uri_oplog, NULL, NULL, &cur_oplog)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_oplog);
if ((ret = session->open_cursor(
session, stable_store, NULL, NULL, &cur_stable)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", stable_store);
/*
* Write our portion of the key space until we're killed.
*/
printf("Thread %" PRIu32 " starts at %" PRIu64 "\n", td->id, td->start);
for (i = td->start; ; ++i) {
if (use_ts)
stable_ts = global_ts++;
else
stable_ts = 0;
testutil_check(__wt_snprintf(
kname, sizeof(kname), "%" PRIu64, i));
testutil_check(session->begin_transaction(session, NULL));
cur_coll->set_key(cur_coll, kname);
cur_local->set_key(cur_local, kname);
cur_oplog->set_key(cur_oplog, kname);
/*
* Put an informative string into the value so that it
* can be viewed well in a binary dump.
*/
testutil_check(__wt_snprintf(cbuf, sizeof(cbuf),
"COLL: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
td->id, stable_ts, i));
testutil_check(__wt_snprintf(lbuf, sizeof(lbuf),
"LOCAL: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
td->id, stable_ts, i));
testutil_check(__wt_snprintf(obuf, sizeof(obuf),
"OPLOG: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
td->id, stable_ts, i));
data.size = __wt_random(&rnd) % MAX_VAL;
data.data = cbuf;
cur_coll->set_value(cur_coll, &data);
if ((ret = cur_coll->insert(cur_coll)) != 0)
testutil_die(ret, "WT_CURSOR.insert");
data.size = __wt_random(&rnd) % MAX_VAL;
data.data = obuf;
cur_oplog->set_value(cur_oplog, &data);
if ((ret = cur_oplog->insert(cur_oplog)) != 0)
testutil_die(ret, "WT_CURSOR.insert");
if (use_ts) {
testutil_check(__wt_snprintf(tscfg, sizeof(tscfg),
"commit_timestamp=%" PRIx64, stable_ts));
testutil_check(
session->commit_transaction(session, tscfg));
} else
testutil_check(
session->commit_transaction(session, NULL));
/*
* Insert into the local table outside the timestamp txn.
*/
data.size = __wt_random(&rnd) % MAX_VAL;
data.data = lbuf;
cur_local->set_value(cur_local, &data);
if ((ret = cur_local->insert(cur_local)) != 0)
testutil_die(ret, "WT_CURSOR.insert");
/*
* Every N records we will record our stable timestamp into the
* stable table. That will define our threshold where we
* expect to find records after recovery.
*/
if (i % STABLE_PERIOD == 0) {
if (use_ts) {
/*
* Set both the oldest and stable timestamp
* so that we don't need to maintain read
* availability at older timestamps.
*/
testutil_check(__wt_snprintf(
tscfg, sizeof(tscfg),
"oldest_timestamp=%" PRIx64
",stable_timestamp=%" PRIx64,
stable_ts, stable_ts));
testutil_check(
td->conn->set_timestamp(td->conn, tscfg));
}
cur_stable->set_key(cur_stable, td->id);
cur_stable->set_value(cur_stable, stable_ts);
testutil_check(cur_stable->insert(cur_stable));
}
/*
* Save the timestamp and key separately for checking later.
*/
if (fprintf(fp,
"%" PRIu64 " %" PRIu64 "\n", stable_ts, i) < 0)
testutil_die(EIO, "fprintf");
}
/* NOTREACHED */
}
static void
run_workload(uint32_t nth)
{
WT_CONNECTION *conn;
WT_SESSION *session;
THREAD_DATA *td;
wt_thread_t *thr;
uint32_t ckpt_id, i, ts_id;
char envconf[512], uri[128];
thr = dcalloc(nth+2, sizeof(*thr));
td = dcalloc(nth+2, sizeof(THREAD_DATA));
if (chdir(home) != 0)
testutil_die(errno, "Child chdir: %s", home);
if (inmem)
(void)__wt_snprintf(envconf, sizeof(envconf),
ENV_CONFIG_DEF, SESSION_MAX);
else
(void)__wt_snprintf(envconf, sizeof(envconf),
ENV_CONFIG_TXNSYNC, SESSION_MAX);
if (compat)
strcat(envconf, ENV_CONFIG_COMPAT);
testutil_check(wiredtiger_open(NULL, NULL, envconf, &conn));
testutil_check(conn->open_session(conn, NULL, NULL, &session));
/*
* Create all the tables.
*/
testutil_check(__wt_snprintf(
uri, sizeof(uri), "%s:%s", table_pfx, uri_collection));
testutil_check(session->create(session, uri,
"key_format=S,value_format=u,log=(enabled=false)"));
testutil_check(__wt_snprintf(
uri, sizeof(uri), "%s:%s", table_pfx, uri_local));
testutil_check(session->create(session,
uri, "key_format=S,value_format=u"));
testutil_check(__wt_snprintf(
uri, sizeof(uri), "%s:%s", table_pfx, uri_oplog));
testutil_check(session->create(session,
uri, "key_format=S,value_format=u"));
/*
* Don't log the stable timestamp table so that we know what timestamp
* was stored at the checkpoint.
*/
testutil_check(session->close(session, NULL));
/*
* The checkpoint thread and the timestamp threads are added at the end.
*/
ckpt_id = nth;
td[ckpt_id].conn = conn;
td[ckpt_id].info = nth;
printf("Create checkpoint thread\n");
testutil_check(__wt_thread_create(
NULL, &thr[ckpt_id], thread_ckpt_run, &td[ckpt_id]));
ts_id = nth + 1;
if (use_ts) {
td[ts_id].conn = conn;
td[ts_id].info = nth;
printf("Create timestamp thread\n");
testutil_check(__wt_thread_create(
NULL, &thr[ts_id], thread_ts_run, &td[ts_id]));
}
printf("Create %" PRIu32 " writer threads\n", nth);
for (i = 0; i < nth; ++i) {
td[i].conn = conn;
td[i].start = WT_BILLION * (uint64_t)i;
td[i].info = i;
testutil_check(__wt_thread_create(
NULL, &thr[i], thread_run, &td[i]));
}
/*
* The threads never exit, so the child will just wait here until
* it is killed.
*/
fflush(stdout);
for (i = 0; i <= ts_id; ++i)
testutil_check(__wt_thread_join(NULL, &thr[i]));
/*
* NOTREACHED
*/
free(thr);
free(td);
exit(EXIT_SUCCESS);
}
int
main(int argc, char *argv[])
{
struct stat sb;
FILE *fp;
WT_CONNECTION *conn;
WT_CURSOR *cur_coll, *cur_local, *cur_oplog, *cur_stable;
WT_RAND_STATE rnd;
WT_SESSION *session;
pid_t pid;
uint64_t absent_coll, absent_local, absent_oplog, count, key, last_key;
uint64_t first_miss, middle_coll, middle_local, middle_oplog;
uint64_t stable_fp, stable_val, val[MAX_TH+1];
uint32_t i, nth, timeout;
int ch, status, ret;
const char *working_dir;
char buf[512], fname[64], kname[64], statname[1024];
bool fatal, rand_th, rand_time, verify_only;
(void)testutil_set_progname(argv);
compat = inmem = false;
use_ts = true;
nth = MIN_TH;
rand_th = rand_time = true;
timeout = MIN_TIME;
verify_only = false;
working_dir = "WT_TEST.timestamp-abort";
while ((ch = __wt_getopt(progname, argc, argv, "Ch:mT:t:vz")) != EOF)
switch (ch) {
case 'C':
compat = true;
break;
case 'h':
working_dir = __wt_optarg;
break;
case 'm':
inmem = true;
break;
case 'T':
rand_th = false;
nth = (uint32_t)atoi(__wt_optarg);
break;
case 't':
rand_time = false;
timeout = (uint32_t)atoi(__wt_optarg);
break;
case 'v':
verify_only = true;
break;
case 'z':
use_ts = false;
break;
default:
usage();
}
argc -= __wt_optind;
argv += __wt_optind;
if (argc != 0)
usage();
testutil_work_dir_from_path(home, sizeof(home), working_dir);
/*
* If the user wants to verify they need to tell us how many threads
* there were so we can find the old record files.
*/
if (verify_only && rand_th) {
fprintf(stderr,
"Verify option requires specifying number of threads\n");
exit (EXIT_FAILURE);
}
if (!verify_only) {
testutil_make_work_dir(home);
__wt_random_init_seed(NULL, &rnd);
if (rand_time) {
timeout = __wt_random(&rnd) % MAX_TIME;
if (timeout < MIN_TIME)
timeout = MIN_TIME;
}
if (rand_th) {
nth = __wt_random(&rnd) % MAX_TH;
if (nth < MIN_TH)
nth = MIN_TH;
}
printf("Parent: compatibility: %s, "
"in-mem log sync: %s, timestamp in use: %s\n",
compat ? "true" : "false",
inmem ? "true" : "false",
use_ts ? "true" : "false");
printf("Parent: Create %" PRIu32
" threads; sleep %" PRIu32 " seconds\n", nth, timeout);
/*
* Fork a child to insert as many items. We will then randomly
* kill the child, run recovery and make sure all items we wrote
* exist after recovery runs.
*/
testutil_checksys((pid = fork()) < 0);
if (pid == 0) { /* child */
run_workload(nth);
return (EXIT_SUCCESS);
}
/* parent */
/*
* Sleep for the configured amount of time before killing
* the child. Start the timeout from the time we notice that
* the file has been created. That allows the test to run
* correctly on really slow machines. Verify the process ID
* still exists in case the child aborts for some reason we
* don't stay in this loop forever.
*/
testutil_check(__wt_snprintf(
statname, sizeof(statname), "%s/%s", home, ckpt_file));
while (stat(statname, &sb) != 0 && kill(pid, 0) == 0)
sleep(1);
sleep(timeout);
/*
* !!! It should be plenty long enough to make sure more than
* one log file exists. If wanted, that check would be added
* here.
*/
printf("Kill child\n");
testutil_checksys(kill(pid, SIGKILL) != 0);
testutil_checksys(waitpid(pid, &status, 0) == -1);
}
/*
* !!! If we wanted to take a copy of the directory before recovery,
* this is the place to do it.
*/
if (chdir(home) != 0)
testutil_die(errno, "parent chdir: %s", home);
testutil_check(__wt_snprintf(buf, sizeof(buf),
"rm -rf ../%s.SAVE && mkdir ../%s.SAVE && "
"cp -p WiredTigerLog.* ../%s.SAVE",
home, home, home));
(void)system(buf);
printf("Open database, run recovery and verify content\n");
/*
* Open the connection which forces recovery to be run.
*/
if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn)) != 0)
testutil_die(ret, "wiredtiger_open");
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
testutil_die(ret, "WT_CONNECTION:open_session");
/*
* Open a cursor on all the tables.
*/
if ((ret = session->open_cursor(session,
uri_collection, NULL, NULL, &cur_coll)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_collection);
if ((ret = session->open_cursor(session,
uri_local, NULL, NULL, &cur_local)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_local);
if ((ret = session->open_cursor(session,
uri_oplog, NULL, NULL, &cur_oplog)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_oplog);
if ((ret = session->open_cursor(session,
stable_store, NULL, NULL, &cur_stable)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", stable_store);
/*
* Find the biggest stable timestamp value that was saved.
*/
stable_val = 0;
memset(val, 0, sizeof(val));
while (cur_stable->next(cur_stable) == 0) {
cur_stable->get_key(cur_stable, &key);
cur_stable->get_value(cur_stable, &val[key]);
if (val[key] > stable_val)
stable_val = val[key];
if (use_ts)
printf("Stable: key %" PRIu64 " value %" PRIu64 "\n",
key, val[key]);
}
if (use_ts)
printf("Got stable_val %" PRIu64 "\n", stable_val);
count = 0;
absent_coll = absent_local = absent_oplog = 0;
fatal = false;
for (i = 1; i <= nth; ++i) {
first_miss = middle_coll = middle_local = middle_oplog = 0;
testutil_check(__wt_snprintf(
fname, sizeof(fname), RECORDS_FILE, i));
if ((fp = fopen(fname, "r")) == NULL)
testutil_die(errno, "fopen: %s", fname);
/*
* For every key in the saved file, verify that the key exists
* in the table after recovery. If we're doing in-memory
* log buffering we never expect a record missing in the middle,
* but records may be missing at the end. If we did
* write-no-sync, we expect every key to have been recovered.
*/
for (last_key = UINT64_MAX;; ++count, last_key = key) {
ret = fscanf(fp, "%" SCNu64 "%" SCNu64 "\n",
&stable_fp, &key);
if (ret != EOF && ret != 2) {
/*
* If we find a partial line, consider it
* like an EOF.
*/
if (ret == 1 || ret == 0)
break;
testutil_die(errno, "fscanf");
}
if (ret == EOF)
break;
/*
* If we're unlucky, the last line may be a partially
* written key at the end that can result in a false
* negative error for a missing record. Detect it.
*/
if (last_key != UINT64_MAX && key != last_key + 1) {
printf("%s: Ignore partial record %" PRIu64
" last valid key %" PRIu64 "\n",
fname, key, last_key);
break;
}
testutil_check(__wt_snprintf(
kname, sizeof(kname), "%" PRIu64, key));
cur_coll->set_key(cur_coll, kname);
cur_local->set_key(cur_local, kname);
cur_oplog->set_key(cur_oplog, kname);
/*
* The collection table should always only have the
* data as of the checkpoint.
*/
if ((ret = cur_coll->search(cur_coll)) != 0) {
if (ret != WT_NOTFOUND)
testutil_die(ret, "search");
/*
* If we don't find a record, the stable
* timestamp written to our file better be
* larger than the saved one.
*/
if (!inmem &&
stable_fp != 0 && stable_fp <= val[i]) {
printf("%s: COLLECTION no record with "
"key %" PRIu64 " record ts %" PRIu64
" <= stable ts %" PRIu64 "\n",
fname, key, stable_fp, val[i]);
absent_coll++;
}
if (middle_coll == 0)
first_miss = key;
middle_coll = key;
} else if (middle_coll != 0) {
/*
* We should never find an existing key after
* we have detected one missing.
*/
printf("%s: COLLECTION after absent records %"
PRIu64 "-%" PRIu64 " key %" PRIu64
" exists\n",
fname, first_miss, middle_coll, key);
fatal = true;
}
/*
* The local table should always have all data.
*/
if ((ret = cur_local->search(cur_local)) != 0) {
if (ret != WT_NOTFOUND)
testutil_die(ret, "search");
if (!inmem)
printf("%s: LOCAL no record with key %"
PRIu64 "\n", fname, key);
absent_local++;
middle_local = key;
} else if (middle_local != 0) {
/*
* We should never find an existing key after
* we have detected one missing.
*/
printf("%s: LOCAL after absent record at %"
PRIu64 " key %" PRIu64 " exists\n",
fname, middle_local, key);
fatal = true;
}
/*
* The oplog table should always have all data.
*/
if ((ret = cur_oplog->search(cur_oplog)) != 0) {
if (ret != WT_NOTFOUND)
testutil_die(ret, "search");
if (!inmem)
printf("%s: OPLOG no record with key %"
PRIu64 "\n", fname, key);
absent_oplog++;
middle_oplog = key;
} else if (middle_oplog != 0) {
/*
* We should never find an existing key after
* we have detected one missing.
*/
printf("%s: OPLOG after absent record at %"
PRIu64 " key %" PRIu64 " exists\n",
fname, middle_oplog, key);
fatal = true;
}
}
testutil_checksys(fclose(fp) != 0);
}
if ((ret = conn->close(conn, NULL)) != 0)
testutil_die(ret, "WT_CONNECTION:close");
if (fatal)
return (EXIT_FAILURE);
if (!inmem && absent_coll) {
printf("COLLECTION: %" PRIu64
" record(s) absent from %" PRIu64 "\n",
absent_coll, count);
fatal = true;
}
if (!inmem && absent_local) {
printf("LOCAL: %" PRIu64 " record(s) absent from %" PRIu64 "\n",
absent_local, count);
fatal = true;
}
if (!inmem && absent_oplog) {
printf("OPLOG: %" PRIu64 " record(s) absent from %" PRIu64 "\n",
absent_oplog, count);
fatal = true;
}
if (fatal)
return (EXIT_FAILURE);
printf("%" PRIu64 " records verified\n", count);
return (EXIT_SUCCESS);
}
int
main(int argc, char *argv[])
{
struct sigaction sa;
struct stat sb;
FILE *fp;
REPORT c_rep[MAX_TH], l_rep[MAX_TH], o_rep[MAX_TH];
WT_CONNECTION *conn;
WT_CURSOR *cur_coll, *cur_local, *cur_oplog;
WT_RAND_STATE rnd;
WT_SESSION *session;
pid_t pid;
uint64_t absent_coll, absent_local, absent_oplog, count, key, last_key;
uint64_t stable_fp, stable_val;
uint32_t i, nth, timeout;
int ch, status, ret;
const char *working_dir;
char buf[512], fname[64], kname[64], statname[1024];
char ts_string[WT_TS_HEX_STRING_SIZE];
bool fatal, rand_th, rand_time, verify_only;
(void)testutil_set_progname(argv);
compat = inmem = false;
use_ts = true;
nth = MIN_TH;
rand_th = rand_time = true;
timeout = MIN_TIME;
verify_only = false;
working_dir = "WT_TEST.timestamp-abort";
while ((ch = __wt_getopt(progname, argc, argv, "Ch:LmT:t:vz")) != EOF)
switch (ch) {
case 'C':
compat = true;
break;
case 'h':
working_dir = __wt_optarg;
break;
case 'L':
table_pfx = "lsm";
break;
case 'm':
inmem = true;
break;
case 'T':
rand_th = false;
nth = (uint32_t)atoi(__wt_optarg);
if (nth > MAX_TH) {
fprintf(stderr,
"Number of threads is larger than the"
" maximum %" PRId32 "\n", MAX_TH);
return (EXIT_FAILURE);
}
break;
case 't':
rand_time = false;
timeout = (uint32_t)atoi(__wt_optarg);
break;
case 'v':
verify_only = true;
break;
case 'z':
use_ts = false;
break;
default:
usage();
}
argc -= __wt_optind;
if (argc != 0)
usage();
testutil_work_dir_from_path(home, sizeof(home), working_dir);
testutil_check(pthread_rwlock_init(&ts_lock, NULL));
/*
* If the user wants to verify they need to tell us how many threads
* there were so we can find the old record files.
*/
if (verify_only && rand_th) {
fprintf(stderr,
"Verify option requires specifying number of threads\n");
exit (EXIT_FAILURE);
}
if (!verify_only) {
testutil_make_work_dir(home);
__wt_random_init_seed(NULL, &rnd);
if (rand_time) {
timeout = __wt_random(&rnd) % MAX_TIME;
if (timeout < MIN_TIME)
timeout = MIN_TIME;
}
if (rand_th) {
nth = __wt_random(&rnd) % MAX_TH;
if (nth < MIN_TH)
nth = MIN_TH;
}
printf("Parent: compatibility: %s, "
"in-mem log sync: %s, timestamp in use: %s\n",
compat ? "true" : "false",
inmem ? "true" : "false",
use_ts ? "true" : "false");
printf("Parent: Create %" PRIu32
" threads; sleep %" PRIu32 " seconds\n", nth, timeout);
printf("CONFIG: %s%s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n",
progname,
compat ? " -C" : "",
inmem ? " -m" : "",
!use_ts ? " -z" : "",
working_dir, nth, timeout);
/*
* Fork a child to insert as many items. We will then randomly
* kill the child, run recovery and make sure all items we wrote
* exist after recovery runs.
*/
memset(&sa, 0, sizeof(sa));
sa.sa_handler = handler;
testutil_checksys(sigaction(SIGCHLD, &sa, NULL));
testutil_checksys((pid = fork()) < 0);
if (pid == 0) { /* child */
run_workload(nth);
return (EXIT_SUCCESS);
}
/* parent */
/*
* Sleep for the configured amount of time before killing
* the child. Start the timeout from the time we notice that
* the file has been created. That allows the test to run
* correctly on really slow machines.
*/
testutil_check(__wt_snprintf(
statname, sizeof(statname), "%s/%s", home, ckpt_file));
while (stat(statname, &sb) != 0)
testutil_sleep_wait(1, pid);
sleep(timeout);
sa.sa_handler = SIG_DFL;
testutil_checksys(sigaction(SIGCHLD, &sa, NULL));
/*
* !!! It should be plenty long enough to make sure more than
* one log file exists. If wanted, that check would be added
* here.
*/
printf("Kill child\n");
testutil_checksys(kill(pid, SIGKILL) != 0);
testutil_checksys(waitpid(pid, &status, 0) == -1);
}
/*
* !!! If we wanted to take a copy of the directory before recovery,
* this is the place to do it. Don't do it all the time because
* it can use a lot of disk space, which can cause test machine
* issues.
*/
if (chdir(home) != 0)
testutil_die(errno, "parent chdir: %s", home);
/*
* The tables can get very large, so while we'd ideally like to
* copy the entire database, we only copy the log files for now.
* Otherwise it can take far too long to run the test, particularly
* in automated testing.
*/
testutil_check(__wt_snprintf(buf, sizeof(buf),
"rm -rf ../%s.SAVE && mkdir ../%s.SAVE && "
"cp -p * ../%s.SAVE",
home, home, home));
if ((status = system(buf)) < 0)
testutil_die(status, "system: %s", buf);
printf("Open database, run recovery and verify content\n");
/*
* Open the connection which forces recovery to be run.
*/
testutil_check(wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn));
testutil_check(conn->open_session(conn, NULL, NULL, &session));
/*
* Open a cursor on all the tables.
*/
testutil_check(__wt_snprintf(
buf, sizeof(buf), "%s:%s", table_pfx, uri_collection));
testutil_check(session->open_cursor(session,
buf, NULL, NULL, &cur_coll));
testutil_check(__wt_snprintf(
buf, sizeof(buf), "%s:%s", table_pfx, uri_local));
testutil_check(session->open_cursor(session,
buf, NULL, NULL, &cur_local));
testutil_check(__wt_snprintf(
buf, sizeof(buf), "%s:%s", table_pfx, uri_oplog));
testutil_check(session->open_cursor(session,
buf, NULL, NULL, &cur_oplog));
/*
* Find the biggest stable timestamp value that was saved.
*/
stable_val = 0;
if (use_ts) {
testutil_check(
conn->query_timestamp(conn, ts_string, "get=recovery"));
testutil_assert(
sscanf(ts_string, "%" SCNx64, &stable_val) == 1);
printf("Got stable_val %" PRIu64 "\n", stable_val);
}
count = 0;
absent_coll = absent_local = absent_oplog = 0;
fatal = false;
for (i = 0; i < nth; ++i) {
initialize_rep(&c_rep[i]);
initialize_rep(&l_rep[i]);
initialize_rep(&o_rep[i]);
testutil_check(__wt_snprintf(
fname, sizeof(fname), RECORDS_FILE, i));
if ((fp = fopen(fname, "r")) == NULL)
testutil_die(errno, "fopen: %s", fname);
/*
* For every key in the saved file, verify that the key exists
* in the table after recovery. If we're doing in-memory
* log buffering we never expect a record missing in the middle,
* but records may be missing at the end. If we did
* write-no-sync, we expect every key to have been recovered.
*/
for (last_key = INVALID_KEY;; ++count, last_key = key) {
ret = fscanf(fp, "%" SCNu64 "%" SCNu64 "\n",
&stable_fp, &key);
if (last_key == INVALID_KEY) {
c_rep[i].first_key = key;
l_rep[i].first_key = key;
o_rep[i].first_key = key;
}
if (ret != EOF && ret != 2) {
/*
* If we find a partial line, consider it
* like an EOF.
*/
if (ret == 1 || ret == 0)
break;
testutil_die(errno, "fscanf");
}
if (ret == EOF)
break;
/*
* If we're unlucky, the last line may be a partially
* written key at the end that can result in a false
* negative error for a missing record. Detect it.
*/
if (last_key != INVALID_KEY && key != last_key + 1) {
printf("%s: Ignore partial record %" PRIu64
" last valid key %" PRIu64 "\n",
fname, key, last_key);
break;
}
testutil_check(__wt_snprintf(
kname, sizeof(kname), "%" PRIu64, key));
cur_coll->set_key(cur_coll, kname);
cur_local->set_key(cur_local, kname);
cur_oplog->set_key(cur_oplog, kname);
/*
* The collection table should always only have the
* data as of the checkpoint.
*/
if ((ret = cur_coll->search(cur_coll)) != 0) {
if (ret != WT_NOTFOUND)
testutil_die(ret, "search");
/*
* If we don't find a record, the stable
* timestamp written to our file better be
* larger than the saved one.
*/
if (!inmem &&
stable_fp != 0 && stable_fp <= stable_val) {
printf("%s: COLLECTION no record with "
"key %" PRIu64 " record ts %" PRIu64
" <= stable ts %" PRIu64 "\n",
fname, key, stable_fp, stable_val);
absent_coll++;
}
if (c_rep[i].first_miss == INVALID_KEY)
c_rep[i].first_miss = key;
c_rep[i].absent_key = key;
} else if (c_rep[i].absent_key != INVALID_KEY &&
c_rep[i].exist_key == INVALID_KEY) {
/*
* If we get here we found a record that exists
* after absent records, a hole in our data.
*/
c_rep[i].exist_key = key;
fatal = true;
} else if (!inmem &&
stable_fp != 0 && stable_fp > stable_val) {
/*
* If we found a record, the stable timestamp
* written to our file better be no larger
* than the checkpoint one.
*/
printf("%s: COLLECTION record with "
"key %" PRIu64 " record ts %" PRIu64
" > stable ts %" PRIu64 "\n",
fname, key, stable_fp, stable_val);
fatal = true;
}
/*
* The local table should always have all data.
*/
if ((ret = cur_local->search(cur_local)) != 0) {
if (ret != WT_NOTFOUND)
testutil_die(ret, "search");
if (!inmem)
printf("%s: LOCAL no record with key %"
PRIu64 "\n", fname, key);
absent_local++;
if (l_rep[i].first_miss == INVALID_KEY)
l_rep[i].first_miss = key;
l_rep[i].absent_key = key;
} else if (l_rep[i].absent_key != INVALID_KEY &&
l_rep[i].exist_key == INVALID_KEY) {
/*
* We should never find an existing key after
* we have detected one missing.
*/
l_rep[i].exist_key = key;
fatal = true;
}
/*
* The oplog table should always have all data.
*/
if ((ret = cur_oplog->search(cur_oplog)) != 0) {
if (ret != WT_NOTFOUND)
testutil_die(ret, "search");
if (!inmem)
printf("%s: OPLOG no record with key %"
PRIu64 "\n", fname, key);
absent_oplog++;
if (o_rep[i].first_miss == INVALID_KEY)
o_rep[i].first_miss = key;
o_rep[i].absent_key = key;
} else if (o_rep[i].absent_key != INVALID_KEY &&
o_rep[i].exist_key == INVALID_KEY) {
/*
* We should never find an existing key after
* we have detected one missing.
*/
o_rep[i].exist_key = key;
fatal = true;
}
}
c_rep[i].last_key = last_key;
l_rep[i].last_key = last_key;
o_rep[i].last_key = last_key;
testutil_checksys(fclose(fp) != 0);
print_missing(&c_rep[i], fname, "COLLECTION");
print_missing(&l_rep[i], fname, "LOCAL");
print_missing(&o_rep[i], fname, "OPLOG");
}
testutil_check(conn->close(conn, NULL));
if (!inmem && absent_coll) {
printf("COLLECTION: %" PRIu64
" record(s) absent from %" PRIu64 "\n",
absent_coll, count);
fatal = true;
}
if (!inmem && absent_local) {
printf("LOCAL: %" PRIu64 " record(s) absent from %" PRIu64 "\n",
absent_local, count);
fatal = true;
}
if (!inmem && absent_oplog) {
printf("OPLOG: %" PRIu64 " record(s) absent from %" PRIu64 "\n",
absent_oplog, count);
fatal = true;
}
testutil_check(pthread_rwlock_destroy(&ts_lock));
if (fatal)
return (EXIT_FAILURE);
printf("%" PRIu64 " records verified\n", count);
return (EXIT_SUCCESS);
}
void
run(void)
{
WT_BLOOM *bloomp;
WT_ITEM item;
WT_SESSION_IMPL *sess;
uint32_t fp, i;
int ret;
const char *uri = "file:my_bloom.bf";
/* Use the internal session handle to access private APIs. */
sess = (WT_SESSION_IMPL *)g.wt_session;
testutil_check(__wt_bloom_create(
sess, uri, NULL, g.c_ops, g.c_factor, g.c_k, &bloomp));
item.size = g.c_key_max;
for (i = 0; i < g.c_ops; i++) {
item.data = g.entries[i];
if ((ret = __wt_bloom_insert(bloomp, &item)) != 0)
testutil_die(ret, "__wt_bloom_insert: %" PRIu32, i);
}
testutil_check(__wt_bloom_finalize(bloomp));
for (i = 0; i < g.c_ops; i++) {
item.data = g.entries[i];
if ((ret = __wt_bloom_get(bloomp, &item)) != 0) {
fprintf(stderr,
"get failed at record: %" PRIu32 "\n", i);
testutil_die(ret, "__wt_bloom_get");
}
}
testutil_check(__wt_bloom_close(bloomp));
testutil_check(g.wt_session->checkpoint(g.wt_session, NULL));
testutil_check(__wt_bloom_open(
sess, uri, g.c_factor, g.c_k, NULL, &bloomp));
for (i = 0; i < g.c_ops; i++) {
item.data = g.entries[i];
testutil_check(__wt_bloom_get(bloomp, &item));
}
/*
* Try out some values we didn't insert - choose a different size to
* ensure the value doesn't overlap with existing values.
*/
item.size = g.c_key_max + 10;
item.data = dcalloc(item.size, 1);
memset((void *)item.data, 'a', item.size);
for (i = 0, fp = 0; i < g.c_ops; i++) {
((uint8_t *)item.data)[i % item.size] =
'a' + ((uint8_t)rand() % 26);
if ((ret = __wt_bloom_get(bloomp, &item)) == 0)
++fp;
if (ret != 0 && ret != WT_NOTFOUND)
testutil_die(ret, "__wt_bloom_get");
}
free((void *)item.data);
printf(
"Out of %" PRIu32 " ops, got %" PRIu32 " false positives, %.4f%%\n",
g.c_ops, fp, 100.0 * fp/g.c_ops);
testutil_check(__wt_bloom_drop(bloomp, NULL));
}
void
wts_load(void)
{
WT_CONNECTION *conn;
WT_CURSOR *cursor;
WT_DECL_RET;
WT_ITEM key, value;
WT_SESSION *session;
bool is_bulk;
conn = g.wts_conn;
testutil_check(conn->open_session(conn, NULL, NULL, &session));
if (g.logging != 0)
(void)g.wt_api->msg_printf(g.wt_api, session,
"=============== bulk load start ===============");
/*
* No bulk load with data-sources.
*
* No bulk load with custom collators, the order of insertion will not
* match the collation order.
*/
is_bulk = true;
if (DATASOURCE("kvsbdb"))
is_bulk = false;
if (g.c_reverse)
is_bulk = false;
/*
* open_cursor can return EBUSY if concurrent with a metadata
* operation, retry in that case.
*/
while ((ret = session->open_cursor(session, g.uri, NULL,
is_bulk ? "bulk,append" : NULL, &cursor)) == EBUSY)
__wt_yield();
testutil_check(ret);
/* Set up the key/value buffers. */
key_gen_init(&key);
val_gen_init(&value);
for (;;) {
if (++g.key_cnt > g.c_rows) {
g.key_cnt = g.rows = g.c_rows;
break;
}
/* Report on progress every 100 inserts. */
if (g.key_cnt % 1000 == 0)
track("bulk load", g.key_cnt, NULL);
key_gen(&key, g.key_cnt);
val_gen(NULL, &value, g.key_cnt);
switch (g.type) {
case FIX:
if (!is_bulk)
cursor->set_key(cursor, g.key_cnt);
cursor->set_value(cursor, *(uint8_t *)value.data);
if (g.logging == LOG_OPS)
(void)g.wt_api->msg_printf(g.wt_api, session,
"%-10s %" PRIu64 " {0x%02" PRIx8 "}",
"bulk V",
g.key_cnt, ((uint8_t *)value.data)[0]);
break;
case VAR:
if (!is_bulk)
cursor->set_key(cursor, g.key_cnt);
cursor->set_value(cursor, &value);
if (g.logging == LOG_OPS)
(void)g.wt_api->msg_printf(g.wt_api, session,
"%-10s %" PRIu64 " {%.*s}", "bulk V",
g.key_cnt,
(int)value.size, (char *)value.data);
break;
case ROW:
cursor->set_key(cursor, &key);
if (g.logging == LOG_OPS)
(void)g.wt_api->msg_printf(g.wt_api, session,
"%-10s %" PRIu64 " {%.*s}", "bulk K",
g.key_cnt, (int)key.size, (char *)key.data);
cursor->set_value(cursor, &value);
if (g.logging == LOG_OPS)
(void)g.wt_api->msg_printf(g.wt_api, session,
"%-10s %" PRIu64 " {%.*s}", "bulk V",
g.key_cnt,
(int)value.size, (char *)value.data);
break;
}
/*
* We don't want to size the cache to ensure the initial data
* set can load in the in-memory case, guaranteeing the load
* succeeds probably means future updates are also guaranteed
* to succeed, which isn't what we want. If we run out of space
* in the initial load, reset the row counter and continue.
*
* Decrease inserts, they can't be successful if we're at the
* cache limit, and increase the delete percentage to get some
* extra space once the run starts.
*/
if ((ret = cursor->insert(cursor)) != 0) {
if (ret != WT_CACHE_FULL)
testutil_die(ret, "cursor.insert");
g.rows = --g.key_cnt;
g.c_rows = (uint32_t)g.key_cnt;
if (g.c_insert_pct > 5)
g.c_insert_pct = 5;
if (g.c_delete_pct < 20)
g.c_delete_pct += 20;
break;
}
#ifdef HAVE_BERKELEY_DB
if (SINGLETHREADED)
bdb_insert(key.data, key.size, value.data, value.size);
#endif
}
testutil_check(cursor->close(cursor));
if (g.logging != 0)
(void)g.wt_api->msg_printf(g.wt_api, session,
"=============== bulk load stop ===============");
testutil_check(session->close(session, NULL));
key_gen_teardown(&key);
val_gen_teardown(&value);
}
