void
obj_create(void)
{
WT_SESSION *session;
int ret;
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
testutil_die(ret, "conn.session");
if ((ret = session->create(session, uri, config)) != 0)
if (ret != EEXIST && ret != EBUSY)
testutil_die(ret, "session.create");
if ((ret = session->close(session, NULL)) != 0)
testutil_die(ret, "session.close");
}
/*
* stats
* Dump the database/file statistics.
*/
void
stats(void)
{
FILE *fp;
WT_CURSOR *cursor;
WT_SESSION *session;
uint64_t v;
int ret;
const char *desc, *pval;
char name[64];
testutil_check(conn->open_session(conn, NULL, NULL, &session));
if ((fp = fopen(FNAME_STAT, "w")) == NULL)
testutil_die(errno, "fopen " FNAME_STAT);
/* Connection statistics. */
testutil_check(session->open_cursor(
session, "statistics:", NULL, NULL, &cursor));
while ((ret = cursor->next(cursor)) == 0 &&
(ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0)
(void)fprintf(fp, "%s=%s\n", desc, pval);
if (ret != WT_NOTFOUND)
testutil_die(ret, "cursor.next");
testutil_check(cursor->close(cursor));
/* File statistics. */
if (!multiple_files) {
testutil_check(__wt_snprintf(
name, sizeof(name), "statistics:" FNAME, 0));
testutil_check(session->open_cursor(
session, name, NULL, NULL, &cursor));
while ((ret = cursor->next(cursor)) == 0 &&
(ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0)
(void)fprintf(fp, "%s=%s\n", desc, pval);
if (ret != WT_NOTFOUND)
testutil_die(ret, "cursor.next");
testutil_check(cursor->close(cursor));
testutil_check(session->close(session, NULL));
}
(void)fclose(fp);
}
void *
checkpoint_worker(void *arg)
{
CONFIG *cfg;
WT_CONNECTION *conn;
WT_SESSION *session;
int ret;
struct timeval e, s;
uint32_t i;
uint64_t ms;
session = NULL;
cfg = (CONFIG *)arg;
conn = cfg->conn;
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) {
lprintf(cfg, ret, 0,
"open_session failed in checkpoint thread.");
goto err;
}
while (g_util_running) {
/*
* TODO: do we care how long the checkpoint takes?
*/
/* Break the sleep up, so we notice interrupts faster. */
for (i = 0; i < cfg->checkpoint_interval; i++) {
sleep(cfg->report_interval);
if (!g_util_running)
break;
}
gettimeofday(&s, NULL);
if ((ret = session->checkpoint(session, NULL)) != 0)
/* Report errors and continue. */
lprintf(cfg, ret, 0, "Checkpoint failed.");
gettimeofday(&e, NULL);
ms = (e.tv_sec * 1000) + (e.tv_usec / 1000.0);
ms -= (s.tv_sec * 1000) + (s.tv_usec / 1000.0);
lprintf(cfg, 0, 1,
"Finished checkpoint in %" PRIu64 " ms.", ms);
}
err: if (session != NULL)
session->close(session, NULL);
return (arg);
}
/*
* salvage --
* A single salvage.
*/
static void
salvage(void)
{
WT_CONNECTION *conn;
WT_SESSION *session;
int ret;
conn = g.wts_conn;
track("salvage", 0ULL, NULL);
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
die(ret, "connection.open_session");
if ((ret = session->salvage(session, g.uri, "force=true")) != 0)
die(ret, "session.salvage: %s", g.uri);
if ((ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
}
void
wts_verify(const char *tag)
{
WT_CONNECTION *conn;
WT_SESSION *session;
int ret;
conn = g.wts_conn;
track("verify", 0ULL, NULL);
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
die(ret, "connection.open_session");
if ((ret = session->verify(session, WT_TABLENAME, NULL)) != 0)
die(ret, "session.verify: %s: %s", WT_TABLENAME, tag);
if ((ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
}
/*! [thread main] */
int
main(void)
{
WT_CONNECTION *conn;
WT_SESSION *session;
WT_CURSOR *cursor;
pthread_t threads[NUM_THREADS];
int i, ret;
/*
* Create a clean test directory for this run of the test program if the
* environment variable isn't already set (as is done by make check).
*/
if (getenv("WIREDTIGER_HOME") == NULL) {
home = "WT_HOME";
ret = system("rm -rf WT_HOME && mkdir WT_HOME");
} else
home = NULL;
if ((ret = wiredtiger_open(home, NULL, "create", &conn)) != 0)
fprintf(stderr, "Error connecting to %s: %s\n",
home == NULL ? "." : home, wiredtiger_strerror(ret));
/* Note: further error checking omitted for clarity. */
ret = conn->open_session(conn, NULL, NULL, &session);
ret = session->create(session, "table:access",
"key_format=S,value_format=S");
ret = session->open_cursor(session, "table:access", NULL,
"overwrite", &cursor);
cursor->set_key(cursor, "key1");
cursor->set_value(cursor, "value1");
ret = cursor->insert(cursor);
ret = session->close(session, NULL);
for (i = 0; i < NUM_THREADS; i++)
ret = pthread_create(&threads[i], NULL, scan_thread, conn);
for (i = 0; i < NUM_THREADS; i++)
ret = pthread_join(threads[i], NULL);
ret = conn->close(conn, NULL);
return (ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE);
}
void
load(const char *name)
{
WT_CURSOR *cursor;
WT_ITEM *key, _key, *value, _value;
WT_SESSION *session;
char keybuf[64], valuebuf[64];
u_int keyno;
int ret;
file_create(name);
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
testutil_die(ret, "conn.session");
if ((ret =
session->open_cursor(session, name, NULL, "bulk", &cursor)) != 0)
testutil_die(ret, "cursor.open");
key = &_key;
value = &_value;
for (keyno = 1; keyno <= nkeys; ++keyno) {
if (ftype == ROW) {
key->data = keybuf;
key->size = (uint32_t)
snprintf(keybuf, sizeof(keybuf), "%017u", keyno);
cursor->set_key(cursor, key);
} else
cursor->set_key(cursor, (uint32_t)keyno);
value->data = valuebuf;
if (ftype == FIX)
cursor->set_value(cursor, 0x01);
else {
value->size = (uint32_t)
snprintf(valuebuf, sizeof(valuebuf), "%37u", keyno);
cursor->set_value(cursor, value);
}
if ((ret = cursor->insert(cursor)) != 0)
testutil_die(ret, "cursor.insert");
}
if ((ret = session->close(session, NULL)) != 0)
testutil_die(ret, "session.close");
}
/*
* wts_read_scan --
* Read and verify all elements in a file.
*/
void
wts_read_scan(void)
{
WT_CONNECTION *conn;
WT_CURSOR *cursor;
WT_ITEM key;
WT_SESSION *session;
uint64_t cnt, last_cnt;
uint8_t *keybuf;
int ret;
conn = g.wts_conn;
/* Set up the default key buffer. */
key_gen_setup(&keybuf);
/* Open a session and cursor pair. */
if ((ret = conn->open_session(
conn, NULL, ops_session_config(NULL), &session)) != 0)
die(ret, "connection.open_session");
if ((ret = session->open_cursor(
session, g.uri, NULL, NULL, &cursor)) != 0)
die(ret, "session.open_cursor");
/* Check a random subset of the records using the key. */
for (last_cnt = cnt = 0; cnt < g.key_cnt;) {
cnt += mmrand(NULL, 1, 17);
if (cnt > g.rows)
cnt = g.rows;
if (cnt - last_cnt > 1000) {
track("read row scan", cnt, NULL);
last_cnt = cnt;
}
key.data = keybuf;
if ((ret = read_row(cursor, &key, cnt)) != 0)
die(ret, "read_scan");
}
if ((ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
free(keybuf);
}
/*
* real_checkpointer --
* Do the work of creating checkpoints and then verifying them. Also
* responsible for finishing in a timely fashion.
*/
static int
real_checkpointer(void)
{
WT_SESSION *session;
char *checkpoint_config, _buf[128];
int ret;
if (g.running == 0)
return (log_print_err(
"Checkpoint thread started stopped\n", EINVAL, 1));
while (g.ntables > g.ntables_created)
sched_yield();
if ((ret = g.conn->open_session(g.conn, NULL, NULL, &session)) != 0)
return (log_print_err("conn.open_session", ret, 1));
if (strncmp(g.checkpoint_name,
"WiredTigerCheckpoint", strlen("WiredTigerCheckpoint")) == 0)
checkpoint_config = NULL;
else {
checkpoint_config = _buf;
snprintf(checkpoint_config, 128, "name=%s", g.checkpoint_name);
}
while (g.running) {
/* Execute a checkpoint */
if ((ret = session->checkpoint(
session, checkpoint_config)) != 0)
return (log_print_err("session.checkpoint", ret, 1));
printf("Finished a checkpoint\n");
if (!g.running)
goto done;
/* Verify the content of the checkpoint. */
if ((ret = verify_checkpoint(session)) != 0)
return (log_print_err("verify_checkpoint", ret, 1));
}
done: if ((ret = session->close(session, NULL)) != 0)
return (log_print_err("session.close", ret, 1));
return (0);
}
void
load(const char *name)
{
WT_CURSOR *cursor;
WT_ITEM *key, _key, *value, _value;
WT_SESSION *session;
size_t len;
uint64_t keyno;
char keybuf[64], valuebuf[64];
file_create(name);
testutil_check(conn->open_session(conn, NULL, NULL, &session));
testutil_check(
session->open_cursor(session, name, NULL, "bulk", &cursor));
key = &_key;
value = &_value;
for (keyno = 1; keyno <= nkeys; ++keyno) {
if (ftype == ROW) {
testutil_check(__wt_snprintf_len_set(
keybuf, sizeof(keybuf),
&len, "%017" PRIu64, keyno));
key->data = keybuf;
key->size = (uint32_t)len;
cursor->set_key(cursor, key);
} else
cursor->set_key(cursor, keyno);
if (ftype == FIX)
cursor->set_value(cursor, 0x01);
else {
testutil_check(__wt_snprintf_len_set(
valuebuf, sizeof(valuebuf),
&len, "%37" PRIu64, keyno));
value->data = valuebuf;
value->size = (uint32_t)len;
cursor->set_value(cursor, value);
}
testutil_check(cursor->insert(cursor));
}
testutil_check(session->close(session, NULL));
}
static void
wts_sync(void)
{
WT_CONNECTION *conn;
WT_SESSION *session;
int ret;
conn = g.wts_conn;
track("sync", 0ULL, NULL);
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
die(ret, "connection.open_session");
if ((ret = session->sync(
session, WT_TABLENAME, NULL)) != 0 && ret != EBUSY)
die(ret, "session.sync: %s", WT_TABLENAME);
if ((ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
}
void
obj_bulk_unique(int force)
{
WT_CURSOR *c;
WT_SESSION *session;
int ret;
char new_uri[64];
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
testutil_die(ret, "conn.session");
/* Generate a unique object name. */
if ((ret = pthread_rwlock_wrlock(&single)) != 0)
testutil_die(ret, "pthread_rwlock_wrlock single");
testutil_check(__wt_snprintf(
new_uri, sizeof(new_uri), "%s.%u", uri, ++uid));
if ((ret = pthread_rwlock_unlock(&single)) != 0)
testutil_die(ret, "pthread_rwlock_unlock single");
if ((ret = session->create(session, new_uri, config)) != 0)
testutil_die(ret, "session.create: %s", new_uri);
__wt_yield();
/*
* Opening a bulk cursor may have raced with a forced checkpoint
* which created a checkpoint of the empty file, and triggers an EINVAL
*/
if ((ret = session->open_cursor(
session, new_uri, NULL, "bulk", &c)) == 0) {
if ((ret = c->close(c)) != 0)
testutil_die(ret, "cursor.close");
} else if (ret != EINVAL)
testutil_die(ret,
"session.open_cursor bulk unique: %s, new_uri");
while ((ret = session->drop(
session, new_uri, force ? "force" : NULL)) != 0)
if (ret != EBUSY)
testutil_die(ret, "session.drop: %s", new_uri);
if ((ret = session->close(session, NULL)) != 0)
testutil_die(ret, "session.close");
}
int WiredTigerUtil::verifyTable(OperationContext* opCtx,
const std::string& uri,
std::vector<std::string>* errors) {
ErrorAccumulator eventHandler(errors);
// Try to close as much as possible to avoid EBUSY errors.
WiredTigerRecoveryUnit::get(opCtx)->getSession()->closeAllCursors(uri);
WiredTigerSessionCache* sessionCache = WiredTigerRecoveryUnit::get(opCtx)->getSessionCache();
sessionCache->closeAllCursors(uri);
// Open a new session with custom error handlers.
WT_CONNECTION* conn = WiredTigerRecoveryUnit::get(opCtx)->getSessionCache()->conn();
WT_SESSION* session;
invariantWTOK(conn->open_session(conn, &eventHandler, NULL, &session));
ON_BLOCK_EXIT([&] { session->close(session, ""); });
// Do the verify. Weird parens prevent treating "verify" as a macro.
return (session->verify)(session, uri.c_str(), NULL);
}
void
obj_cursor(void)
{
WT_SESSION *session;
WT_CURSOR *cursor;
int ret;
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
testutil_die(ret, "conn.session");
if ((ret =
session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0) {
if (ret != ENOENT && ret != EBUSY)
testutil_die(ret, "session.open_cursor");
} else {
if ((ret = cursor->close(cursor)) != 0)
testutil_die(ret, "cursor.close");
}
if ((ret = session->close(session, NULL)) != 0)
testutil_die(ret, "session.close");
}
void
obj_checkpoint(void)
{
WT_SESSION *session;
int ret;
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
testutil_die(ret, "conn.session");
/*
* Force the checkpoint so it has to be taken. Forced checkpoints can
* race with other metadata operations and return EBUSY - we'd expect
* applications using forced checkpoints to retry on EBUSY.
*/
if ((ret = session->checkpoint(session, "force")) != 0)
if (ret != EBUSY && ret != ENOENT)
testutil_die(ret, "session.checkpoint");
if ((ret = session->close(session, NULL)) != 0)
testutil_die(ret, "session.close");
}
/*
* compaction --
* Periodically do a compaction operation.
*/
void *
compact(void *arg)
{
WT_CONNECTION *conn;
WT_SESSION *session;
u_int period;
int ret;
(void)(arg);
/* Compaction isn't supported for all data sources. */
if (DATASOURCE("helium") || DATASOURCE("kvsbdb"))
return (NULL);
/* Open a session. */
conn = g.wts_conn;
testutil_check(conn->open_session(conn, NULL, NULL, &session));
/*
* Perform compaction at somewhere under 15 seconds (so we get at
* least one done), and then at 23 second intervals.
*/
for (period = mmrand(NULL, 1, 15);; period = 23) {
/* Sleep for short periods so we don't make the run wait. */
while (period > 0 && !g.workers_finished) {
--period;
sleep(1);
}
if (g.workers_finished)
break;
if ((ret = session->compact(
session, g.uri, NULL)) != 0 && ret != WT_ROLLBACK)
testutil_die(ret, "session.compact");
}
testutil_check(session->close(session, NULL));
return (NULL);
}
static void
uri_init(void)
{
WT_CURSOR *cursor;
WT_SESSION *session;
u_int i, key;
char buf[128];
for (i = 0; i < uris; ++i)
if (uri_list[i] == NULL) {
testutil_check(
__wt_snprintf(buf, sizeof(buf), "table:%u", i));
uri_list[i] = dstrdup(buf);
}
testutil_check(conn->open_session(conn, NULL, NULL, &session));
/* Initialize the file contents. */
for (i = 0; i < uris; ++i) {
testutil_check(__wt_snprintf(buf, sizeof(buf),
"key_format=S,value_format=S,"
"allocation_size=4K,leaf_page_max=32KB,"));
testutil_check(session->create(session, uri_list[i], buf));
testutil_check(session->open_cursor(
session, uri_list[i], NULL, NULL, &cursor));
for (key = 1; key < MAXKEY; ++key) {
testutil_check(__wt_snprintf(
buf, sizeof(buf), "key:%020u", key));
cursor->set_key(cursor, buf);
cursor->set_value(cursor, buf);
testutil_check(cursor->insert(cursor));
}
testutil_check(cursor->close(cursor));
}
/* Create a checkpoint we can use for readonly handles. */
testutil_check(session->checkpoint(session, NULL));
testutil_check(session->close(session, NULL));
}
/*
* Create a table.
*/
void
op_create(void *arg)
{
TEST_OPTS *opts;
TEST_PER_THREAD_OPTS *args;
WT_SESSION *session;
int ret;
args = (TEST_PER_THREAD_OPTS *)arg;
opts = args->testopts;
testutil_check(
opts->conn->open_session(opts->conn, NULL, NULL, &session));
if ((ret = session->create(session,
opts->uri, DEFAULT_TABLE_SCHEMA)) != 0)
if (ret != EEXIST && ret != EBUSY)
testutil_die(ret, "session.create");
testutil_check(session->close(session, NULL));
args->thread_counter++;
}
void
populate(TEST_OPTS *opts)
{
WT_CURSOR *maincur;
WT_RAND_STATE rnd;
WT_SESSION *session;
uint32_t key;
int balance, i, flag, post;
__wt_random_init_seed(NULL, &rnd);
testutil_check(opts->conn->open_session(
opts->conn, NULL, NULL, &session));
testutil_check(session->open_cursor(session, opts->uri, NULL, NULL,
&maincur));
for (i = 0; i < N_INSERT; i++) {
testutil_check(session->begin_transaction(session, NULL));
key = (__wt_random(&rnd) % (N_RECORDS));
maincur->set_key(maincur, key);
if (__wt_random(&rnd) % 11 == 0)
post = 54321;
else
post = i % 100000;
if (__wt_random(&rnd) % 4 == 0) {
balance = -100;
flag = 1;
} else {
balance = 100 * (i + 1);
flag = 0;
}
maincur->set_value(maincur, post, balance, flag, key);
testutil_check(maincur->insert(maincur));
testutil_check(session->commit_transaction(session, NULL));
}
testutil_check(maincur->close(maincur));
testutil_check(session->close(session, NULL));
}
/*
* alter --
* Periodically alter a table's metadata.
*/
void *
alter(void *arg)
{
WT_CONNECTION *conn;
WT_SESSION *session;
u_int period;
bool access_value;
char buf[32];
(void)(arg);
conn = g.wts_conn;
/*
* Only alter the access pattern hint. If we alter the cache resident
* setting we may end up with a setting that fills cache and doesn't
* allow it to be evicted.
*/
access_value = false;
/* Open a session */
testutil_check(conn->open_session(conn, NULL, NULL, &session));
while (!g.workers_finished) {
period = mmrand(NULL, 1, 10);
snprintf(buf, sizeof(buf),
"access_pattern_hint=%s", access_value ? "random" : "none");
access_value = !access_value;
if (session->alter(session, g.uri, buf) != 0)
break;
while (period > 0 && !g.workers_finished) {
--period;
sleep(1);
}
}
testutil_check(session->close(session, NULL));
return (NULL);
}
/*
* Create and drop a unique guaranteed table.
*/
void
op_create_unique(void *arg)
{
TEST_OPTS *opts;
TEST_PER_THREAD_OPTS *args;
WT_RAND_STATE rnd;
WT_SESSION *session;
int ret;
char new_uri[64];
args = (TEST_PER_THREAD_OPTS *)arg;
opts = args->testopts;
__wt_random_init_seed(NULL, &rnd);
testutil_check(
opts->conn->open_session(opts->conn, NULL, NULL, &session));
/* Generate a unique object name. */
testutil_check(__wt_snprintf(
new_uri, sizeof(new_uri), "%s.%" PRIu64,
opts->uri, __wt_atomic_add64(&opts->unique_id, 1)));
testutil_check(session->create(session, new_uri, DEFAULT_TABLE_SCHEMA));
__wt_yield();
while ((ret = session->drop(session, new_uri, __wt_random(&rnd) & 1 ?
"force,checkpoint_wait=false" : "checkpoint_wait=false")) != 0)
if (ret != EBUSY)
testutil_die(ret, "session.drop: %s", new_uri);
else
/*
* The EBUSY is expected when we run with
* checkpoint_wait set to false, so we increment the
* counter while in this loop to avoid false positives.
*/
args->thread_counter++;
testutil_check(session->close(session, NULL));
args->thread_counter++;
}
void *
read_thread(void *arg)
{
CONFIG *cfg;
WT_CONNECTION *conn;
WT_SESSION *session;
WT_CURSOR *cursor;
char *key_buf;
int ret;
cfg = (CONFIG *)arg;
conn = cfg->conn;
key_buf = calloc(cfg->key_sz, 1);
if (key_buf == NULL)
return (arg);
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) {
fprintf(stderr,
"open_session failed in read thread: %d\n", ret);
return (NULL);
}
if ((ret = session->open_cursor(session, cfg->uri,
NULL, NULL, &cursor)) != 0) {
fprintf(stderr, "open_cursor failed in read thread: %d\n", ret);
return (NULL);
}
while (running) {
++nops;
sprintf(key_buf, "%d", rand() % cfg->icount);
cursor->set_key(cursor, key_buf);
cursor->search(cursor);
}
session->close(session, NULL);
free(key_buf);
return (arg);
}
static void
file_create(const char *name)
{
WT_SESSION *session;
int ret;
char config[128];
testutil_check(conn->open_session(conn, NULL, NULL, &session));
testutil_check(__wt_snprintf(config, sizeof(config),
"key_format=%s,"
"internal_page_max=%d,"
"leaf_page_max=%d,"
"%s",
ftype == ROW ? "u" : "r", 16 * 1024, 128 * 1024,
ftype == FIX ? ",value_format=3t" : ""));
if ((ret = session->create(session, name, config)) != 0)
if (ret != EEXIST)
testutil_die(ret, "session.create");
testutil_check(session->close(session, NULL));
}
void
wts_verify(const char *tag)
{
WT_CONNECTION *conn;
WT_SESSION *session;
int ret;
conn = g.wts_conn;
track("verify", 0ULL, NULL);
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
die(ret, "connection.open_session");
if (g.logging != 0)
(void)session->msg_printf(session,
"=============== verify start ===============");
if ((ret = session->verify(session, g.uri, NULL)) != 0)
die(ret, "session.verify: %s: %s", g.uri, tag);
if (g.logging != 0)
(void)session->msg_printf(session,
"=============== verify stop ===============");
if ((ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
}
/*
* Drop a table.
*/
void
op_drop(void *arg)
{
TEST_OPTS *opts;
TEST_PER_THREAD_OPTS *args;
WT_RAND_STATE rnd;
WT_SESSION *session;
int ret;
args = (TEST_PER_THREAD_OPTS *)arg;
opts = args->testopts;
__wt_random_init_seed(NULL, &rnd);
testutil_check(
opts->conn->open_session(opts->conn, NULL, NULL, &session));
if ((ret = session->drop(session, opts->uri, __wt_random(&rnd) & 1 ?
"force,checkpoint_wait=false" : "checkpoint_wait=false")) != 0)
if (ret != ENOENT && ret != EBUSY)
testutil_die(ret, "session.drop");
testutil_check(session->close(session, NULL));
args->thread_counter++;
}
/*
* wts_ops --
* Perform a number of operations in a set of threads.
*/
void
wts_ops(int lastrun)
{
TINFO *tinfo, total;
WT_CONNECTION *conn;
WT_SESSION *session;
pthread_t backup_tid, compact_tid;
int64_t fourths, thread_ops;
uint32_t i;
int ret, running;
conn = g.wts_conn;
session = NULL; /* -Wconditional-uninitialized */
memset(&backup_tid, 0, sizeof(backup_tid));
memset(&compact_tid, 0, sizeof(compact_tid));
/*
* There are two mechanisms to specify the length of the run, a number
* of operations and a timer, when either expire the run terminates.
* Each thread does an equal share of the total operations (and make
* sure that it's not 0).
*
* Calculate how many fourth-of-a-second sleeps until any timer expires.
*/
if (g.c_ops == 0)
thread_ops = -1;
else {
if (g.c_ops < g.c_threads)
g.c_ops = g.c_threads;
thread_ops = g.c_ops / g.c_threads;
}
if (g.c_timer == 0)
fourths = -1;
else
fourths = ((int64_t)g.c_timer * 4 * 60) / FORMAT_OPERATION_REPS;
/* Initialize the table extension code. */
table_append_init();
/*
* We support replay of threaded runs, but don't log random numbers
* after threaded operations start, there's no point.
*/
if (!SINGLETHREADED)
g.rand_log_stop = 1;
/* Open a session. */
if (g.logging != 0) {
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
die(ret, "connection.open_session");
(void)g.wt_api->msg_printf(g.wt_api, session,
"=============== thread ops start ===============");
}
/* Create thread structure; start the worker threads. */
if ((tinfo = calloc((size_t)g.c_threads, sizeof(*tinfo))) == NULL)
die(errno, "calloc");
for (i = 0; i < g.c_threads; ++i) {
tinfo[i].id = (int)i + 1;
tinfo[i].state = TINFO_RUNNING;
if ((ret =
pthread_create(&tinfo[i].tid, NULL, ops, &tinfo[i])) != 0)
die(ret, "pthread_create");
}
/* If a multi-threaded run, start backup and compaction threads. */
if (g.c_backups &&
(ret = pthread_create(&backup_tid, NULL, backup, NULL)) != 0)
die(ret, "pthread_create: backup");
if (g.c_compact &&
(ret = pthread_create(&compact_tid, NULL, compact, NULL)) != 0)
die(ret, "pthread_create: compaction");
/* Spin on the threads, calculating the totals. */
for (;;) {
/* Clear out the totals each pass. */
memset(&total, 0, sizeof(total));
for (i = 0, running = 0; i < g.c_threads; ++i) {
total.commit += tinfo[i].commit;
total.deadlock += tinfo[i].deadlock;
total.insert += tinfo[i].insert;
total.remove += tinfo[i].remove;
total.rollback += tinfo[i].rollback;
total.search += tinfo[i].search;
total.update += tinfo[i].update;
switch (tinfo[i].state) {
case TINFO_RUNNING:
running = 1;
break;
case TINFO_COMPLETE:
tinfo[i].state = TINFO_JOINED;
(void)pthread_join(tinfo[i].tid, NULL);
break;
case TINFO_JOINED:
break;
}
/*
* If the timer has expired or this thread has completed
* its operations, notify the thread it should quit.
*/
if (fourths == 0 ||
(thread_ops != -1 &&
tinfo[i].ops >= (uint64_t)thread_ops)) {
/*
* On the last execution, optionally drop core
* for recovery testing.
*/
if (lastrun && g.c_abort) {
static char *core = NULL;
*core = 0;
}
tinfo[i].quit = 1;
}
}
track("ops", 0ULL, &total);
if (!running)
break;
(void)usleep(250000); /* 1/4th of a second */
if (fourths != -1)
--fourths;
}
free(tinfo);
/* Wait for the backup, compaction thread. */
g.workers_finished = 1;
if (g.c_backups)
(void)pthread_join(backup_tid, NULL);
if (g.c_compact)
(void)pthread_join(compact_tid, NULL);
if (g.logging != 0) {
(void)g.wt_api->msg_printf(g.wt_api, session,
"=============== thread ops stop ===============");
if ((ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
}
}
static void *
ops(void *arg)
{
TINFO *tinfo;
WT_CONNECTION *conn;
WT_CURSOR *cursor, *cursor_insert;
WT_SESSION *session;
WT_ITEM key, value;
uint64_t keyno, ckpt_op, session_op;
uint32_t op;
uint8_t *keybuf, *valbuf;
u_int np;
int ckpt_available, dir, insert, intxn, notfound, readonly, ret;
char *ckpt_config, ckpt_name[64];
tinfo = arg;
/* Initialize the per-thread random number generator. */
__wt_random_init(&tinfo->rnd);
conn = g.wts_conn;
keybuf = valbuf = NULL;
readonly = 0; /* -Wconditional-uninitialized */
/* Set up the default key and value buffers. */
key_gen_setup(&keybuf);
val_gen_setup(&tinfo->rnd, &valbuf);
/* Set the first operation where we'll create sessions and cursors. */
session_op = 0;
session = NULL;
cursor = cursor_insert = NULL;
/* Set the first operation where we'll perform checkpoint operations. */
ckpt_op = g.c_checkpoints ? mmrand(&tinfo->rnd, 100, 10000) : 0;
ckpt_available = 0;
for (intxn = 0; !tinfo->quit; ++tinfo->ops) {
/*
* We can't checkpoint or swap sessions/cursors while in a
* transaction, resolve any running transaction.
*/
if (intxn &&
(tinfo->ops == ckpt_op || tinfo->ops == session_op)) {
if ((ret = session->commit_transaction(
session, NULL)) != 0)
die(ret, "session.commit_transaction");
++tinfo->commit;
intxn = 0;
}
/* Open up a new session and cursors. */
if (tinfo->ops == session_op ||
session == NULL || cursor == NULL) {
if (session != NULL &&
(ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
if ((ret = conn->open_session(conn, NULL,
ops_session_config(&tinfo->rnd), &session)) != 0)
die(ret, "connection.open_session");
/*
* 10% of the time, perform some read-only operations
* from a checkpoint.
*
* Skip that if we single-threaded and doing checks
* against a Berkeley DB database, because that won't
* work because the Berkeley DB database records won't
* match the checkpoint. Also skip if we are using
* LSM, because it doesn't support reads from
* checkpoints.
*/
if (!SINGLETHREADED && !DATASOURCE("lsm") &&
ckpt_available && mmrand(&tinfo->rnd, 1, 10) == 1) {
if ((ret = session->open_cursor(session,
g.uri, NULL, ckpt_name, &cursor)) != 0)
die(ret, "session.open_cursor");
/* Pick the next session/cursor close/open. */
session_op += 250;
/* Checkpoints are read-only. */
readonly = 1;
} else {
/*
* Open two cursors: one for overwriting and one
* for append (if it's a column-store).
*
* The reason is when testing with existing
* records, we don't track if a record was
* deleted or not, which means we must use
* cursor->insert with overwriting configured.
* But, in column-store files where we're
* testing with new, appended records, we don't
* want to have to specify the record number,
* which requires an append configuration.
*/
if ((ret = session->open_cursor(session, g.uri,
NULL, "overwrite", &cursor)) != 0)
die(ret, "session.open_cursor");
if ((g.type == FIX || g.type == VAR) &&
(ret = session->open_cursor(session, g.uri,
NULL, "append", &cursor_insert)) != 0)
die(ret, "session.open_cursor");
/* Pick the next session/cursor close/open. */
session_op += mmrand(&tinfo->rnd, 100, 5000);
/* Updates supported. */
readonly = 0;
}
}
/* Checkpoint the database. */
if (tinfo->ops == ckpt_op && g.c_checkpoints) {
/*
* LSM and data-sources don't support named checkpoints,
* and we can't drop a named checkpoint while there's a
* cursor open on it, otherwise 20% of the time name the
* checkpoint.
*/
if (DATASOURCE("helium") || DATASOURCE("kvsbdb") ||
DATASOURCE("lsm") ||
readonly || mmrand(&tinfo->rnd, 1, 5) == 1)
ckpt_config = NULL;
else {
(void)snprintf(ckpt_name, sizeof(ckpt_name),
"name=thread-%d", tinfo->id);
ckpt_config = ckpt_name;
}
/* Named checkpoints lock out backups */
if (ckpt_config != NULL &&
(ret = pthread_rwlock_wrlock(&g.backup_lock)) != 0)
die(ret,
"pthread_rwlock_wrlock: backup lock");
if ((ret =
session->checkpoint(session, ckpt_config)) != 0)
die(ret, "session.checkpoint%s%s",
ckpt_config == NULL ? "" : ": ",
ckpt_config == NULL ? "" : ckpt_config);
if (ckpt_config != NULL &&
(ret = pthread_rwlock_unlock(&g.backup_lock)) != 0)
die(ret,
"pthread_rwlock_wrlock: backup lock");
/* Rephrase the checkpoint name for cursor open. */
if (ckpt_config == NULL)
strcpy(ckpt_name,
"checkpoint=WiredTigerCheckpoint");
else
(void)snprintf(ckpt_name, sizeof(ckpt_name),
"checkpoint=thread-%d", tinfo->id);
ckpt_available = 1;
/* Pick the next checkpoint operation. */
ckpt_op += mmrand(&tinfo->rnd, 5000, 20000);
}
/*
* If we're not single-threaded and we're not in a transaction,
* start a transaction 20% of the time.
*/
if (!SINGLETHREADED &&
!intxn && mmrand(&tinfo->rnd, 1, 10) >= 8) {
if ((ret =
session->begin_transaction(session, NULL)) != 0)
die(ret, "session.begin_transaction");
intxn = 1;
}
insert = notfound = 0;
keyno = mmrand(&tinfo->rnd, 1, (u_int)g.rows);
key.data = keybuf;
value.data = valbuf;
/*
* Perform some number of operations: the percentage of deletes,
* inserts and writes are specified, reads are the rest. The
* percentages don't have to add up to 100, a high percentage
* of deletes will mean fewer inserts and writes. Modifications
* are always followed by a read to confirm it worked.
*/
op = readonly ? UINT32_MAX : mmrand(&tinfo->rnd, 1, 100);
if (op < g.c_delete_pct) {
++tinfo->remove;
switch (g.type) {
case ROW:
/*
* If deleting a non-existent record, the cursor
* won't be positioned, and so can't do a next.
*/
if (row_remove(cursor, &key, keyno, ¬found))
goto deadlock;
break;
case FIX:
case VAR:
if (col_remove(cursor, &key, keyno, ¬found))
goto deadlock;
break;
}
} else if (op < g.c_delete_pct + g.c_insert_pct) {
++tinfo->insert;
switch (g.type) {
case ROW:
if (row_insert(
tinfo, cursor, &key, &value, keyno))
goto deadlock;
insert = 1;
break;
case FIX:
case VAR:
/*
* We can only append so many new records, if
* we've reached that limit, update a record
* instead of doing an insert.
*/
if (g.append_cnt >= g.append_max)
goto skip_insert;
/* Insert, then reset the insert cursor. */
if (col_insert(tinfo,
cursor_insert, &key, &value, &keyno))
goto deadlock;
if ((ret =
cursor_insert->reset(cursor_insert)) != 0)
die(ret, "cursor.reset");
insert = 1;
break;
}
} else if (
op < g.c_delete_pct + g.c_insert_pct + g.c_write_pct) {
++tinfo->update;
switch (g.type) {
case ROW:
if (row_update(
tinfo, cursor, &key, &value, keyno))
goto deadlock;
break;
case FIX:
case VAR:
skip_insert: if (col_update(tinfo,
cursor, &key, &value, keyno))
goto deadlock;
break;
}
} else {
++tinfo->search;
if (read_row(cursor, &key, keyno))
if (intxn)
goto deadlock;
continue;
}
/*
* The cursor is positioned if we did any operation other than
* insert, do a small number of next/prev cursor operations in
* a random direction.
*/
if (!insert) {
dir = (int)mmrand(&tinfo->rnd, 0, 1);
for (np = 0; np < mmrand(&tinfo->rnd, 1, 8); ++np) {
if (notfound)
break;
if (nextprev(cursor, dir, ¬found))
goto deadlock;
}
}
/* Read to confirm the operation. */
++tinfo->search;
if (read_row(cursor, &key, keyno))
goto deadlock;
/* Reset the cursor: there is no reason to keep pages pinned. */
if ((ret = cursor->reset(cursor)) != 0)
die(ret, "cursor.reset");
/*
* If we're in the transaction, commit 40% of the time and
* rollback 10% of the time.
*/
if (intxn)
switch (mmrand(&tinfo->rnd, 1, 10)) {
case 1: case 2: case 3: case 4: /* 40% */
if ((ret = session->commit_transaction(
session, NULL)) != 0)
die(ret, "session.commit_transaction");
++tinfo->commit;
intxn = 0;
break;
case 5: /* 10% */
if (0) {
deadlock: ++tinfo->deadlock;
}
if ((ret = session->rollback_transaction(
session, NULL)) != 0)
die(ret,
"session.rollback_transaction");
++tinfo->rollback;
intxn = 0;
break;
default:
break;
}
}
if (session != NULL && (ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
free(keybuf);
free(valbuf);
tinfo->state = TINFO_COMPLETE;
return (NULL);
}
void
wts_open(void)
{
WT_CONNECTION *conn;
WT_SESSION *session;
uint32_t maxintlpage, maxintlitem, maxleafpage, maxleafitem;
int ret;
const char *ext1, *ext2;
char config[512], *end, *p;
/* If the bzip2 compression module has been built, use it. */
#define EXTPATH "../../ext"
ext1 = EXTPATH "compressors/bzip2_compress/.libs/bzip2_compress.so";
if (access(ext1, R_OK) != 0) {
ext1 = "";
g.c_bzip = 0;
}
ext2 = EXTPATH "/collators/reverse/.libs/reverse_collator.so";
/*
* Open configuration -- put command line configuration options at the
* end so they can override "standard" configuration.
*/
snprintf(config, sizeof(config),
"create,error_prefix=\"%s\",cache_size=%" PRIu32 "MB,sync=false,"
"extensions=[\"%s\",\"%s\"],%s",
g.progname, g.c_cache, ext1, ext2,
g.config_open == NULL ? "" : g.config_open);
if ((ret =
wiredtiger_open("RUNDIR", &event_handler, config, &conn)) != 0)
die(ret, "wiredtiger_open");
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
die(ret, "connection.open_session");
maxintlpage = 1U << g.c_intl_page_max;
/* Make sure at least 2 internal page per thread can fit in cache. */
while (2 * g.c_threads * maxintlpage > g.c_cache << 20)
maxintlpage >>= 1;
maxintlitem = MMRAND(maxintlpage / 50, maxintlpage / 40);
if (maxintlitem < 40)
maxintlitem = 40;
maxleafpage = 1U << g.c_leaf_page_max;
/* Make sure at least one leaf page per thread can fit in cache. */
while (g.c_threads * (maxintlpage + maxleafpage) > g.c_cache << 20)
maxleafpage >>= 1;
maxleafitem = MMRAND(maxleafpage / 50, maxleafpage / 40);
if (maxleafitem < 40)
maxleafitem = 40;
p = config;
end = config + sizeof(config);
p += snprintf(p, (size_t)(end - p),
"key_format=%s,"
"internal_page_max=%d,internal_item_max=%d,"
"leaf_page_max=%d,leaf_item_max=%d",
(g.type == ROW) ? "u" : "r",
maxintlpage, maxintlitem, maxleafpage, maxleafitem);
if (g.c_bzip)
p += snprintf(p, (size_t)(end - p),
",block_compressor=\"bzip2_compress\"");
switch (g.type) {
case FIX:
p += snprintf(p, (size_t)(end - p),
",value_format=%dt", g.c_bitcnt);
break;
case ROW:
if (g.c_huffman_key)
p += snprintf(p, (size_t)(end - p),
",huffman_key=english");
if (g.c_reverse)
p += snprintf(p, (size_t)(end - p),
",collator=reverse");
/* FALLTHROUGH */
case VAR:
if (g.c_huffman_value)
p += snprintf(p, (size_t)(end - p),
",huffman_value=english");
if (g.c_dictionary)
p += snprintf(p, (size_t)(end - p),
",dictionary=%d", MMRAND(123, 517));
break;
}
if ((ret = session->create(session, g.uri, config)) != 0)
die(ret, "session.create: %s", g.uri);
if ((ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
g.wts_conn = conn;
}
/*
* wts_stats --
* Dump the run's statistics.
*/
void
wts_stats(void)
{
WT_CONNECTION *conn;
WT_CURSOR *cursor;
WT_SESSION *session;
FILE *fp;
char *stat_name;
const char *pval, *desc;
uint64_t v;
int ret;
track("stat", 0ULL, NULL);
conn = g.wts_conn;
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
die(ret, "connection.open_session");
if ((fp = fopen("RUNDIR/stats", "w")) == NULL)
die(errno, "fopen: RUNDIR/stats");
/* Connection statistics. */
fprintf(fp, "====== Connection statistics:\n");
if ((ret = session->open_cursor(session,
"statistics:", NULL, NULL, &cursor)) != 0)
die(ret, "session.open_cursor");
while ((ret = cursor->next(cursor)) == 0 &&
(ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0)
if (fprintf(fp, "%s=%s\n", desc, pval) < 0)
die(errno, "fprintf");
if (ret != WT_NOTFOUND)
die(ret, "cursor.next");
if ((ret = cursor->close(cursor)) != 0)
die(ret, "cursor.close");
/*
* XXX
* WiredTiger only supports file object statistics.
*/
if (strcmp(g.c_data_source, "file") != 0)
goto skip;
/* File statistics. */
fprintf(fp, "\n\n====== File statistics:\n");
if ((stat_name =
malloc(strlen("statistics:") + strlen(g.uri) + 1)) == NULL)
syserr("malloc");
sprintf(stat_name, "statistics:%s", g.uri);
if ((ret = session->open_cursor(
session, stat_name, NULL, NULL, &cursor)) != 0)
die(ret, "session.open_cursor");
free(stat_name);
while ((ret = cursor->next(cursor)) == 0 &&
(ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0)
if (fprintf(fp, "%s=%s\n", desc, pval) < 0)
die(errno, "fprintf");
if (ret != WT_NOTFOUND)
die(ret, "cursor.next");
if ((ret = cursor->close(cursor)) != 0)
die(ret, "cursor.close");
skip: if ((ret = fclose(fp)) != 0)
die(ret, "fclose");
if ((ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
}
int
main(int argc, char *argv[])
{
TEST_OPTS *opts, _opts;
WT_CURSOR *balancecur, *flagcur, *joincur, *postcur;
WT_CURSOR *maincur;
WT_SESSION *session;
int balance, count, flag, key, key2, post, ret;
char balanceuri[256];
char cfg[128];
char flaguri[256];
char joinuri[256];
char posturi[256];
const char *tablename;
opts = &_opts;
memset(opts, 0, sizeof(*opts));
testutil_check(testutil_parse_opts(argc, argv, opts));
testutil_make_work_dir(opts->home);
testutil_progress(opts, "start");
testutil_check(wiredtiger_open(opts->home, NULL,
"create,cache_size=250M", &opts->conn));
testutil_progress(opts, "wiredtiger_open");
testutil_check(
opts->conn->open_session(opts->conn, NULL, NULL, &session));
testutil_progress(opts, "sessions opened");
/*
* Note: repeated primary key 'id' as 'id2'. This makes
* it easier to dump an index and know which record we're
* looking at.
*/
testutil_check(session->create(session, opts->uri,
"key_format=i,value_format=iiii,"
"columns=(id,post,balance,flag,id2)"));
tablename = strchr(opts->uri, ':');
testutil_assert(tablename != NULL);
tablename++;
testutil_check(__wt_snprintf(
posturi, sizeof(posturi), "index:%s:post", tablename));
testutil_check(__wt_snprintf(
balanceuri, sizeof(balanceuri), "index:%s:balance", tablename));
testutil_check(__wt_snprintf(
flaguri, sizeof(flaguri), "index:%s:flag", tablename));
testutil_check(__wt_snprintf(
joinuri, sizeof(joinuri), "join:%s", opts->uri));
testutil_check(session->create(session, posturi, "columns=(post)"));
testutil_check(session->create(session, balanceuri,
"columns=(balance)"));
testutil_check(session->create(session, flaguri, "columns=(flag)"));
testutil_progress(opts, "setup complete");
/*
* Insert a single record with all items we are search for,
* this makes our logic easier.
*/
testutil_check(session->open_cursor(session, opts->uri, NULL, NULL,
&maincur));
maincur->set_key(maincur, N_RECORDS);
maincur->set_value(maincur, 54321, 0, "", 0, N_RECORDS);
testutil_check(maincur->insert(maincur));
testutil_check(maincur->close(maincur));
testutil_check(session->close(session, NULL));
testutil_progress(opts, "populate start");
populate(opts);
testutil_progress(opts, "populate end");
testutil_check(opts->conn->open_session(
opts->conn, NULL, NULL, &session));
testutil_check(session->open_cursor(session,
posturi, NULL, NULL, &postcur));
testutil_check(session->open_cursor(session,
balanceuri, NULL, NULL, &balancecur));
testutil_check(session->open_cursor(session,
flaguri, NULL, NULL, &flagcur));
testutil_check(session->open_cursor(session,
joinuri, NULL, NULL, &joincur));
postcur->set_key(postcur, 54321);
testutil_check(postcur->search(postcur));
testutil_check(session->join(session, joincur, postcur,
"compare=eq"));
balancecur->set_key(balancecur, 0);
testutil_check(balancecur->search(balancecur));
testutil_check(__wt_snprintf(cfg, sizeof(cfg),
"compare=lt,strategy=bloom,count=%d", N_RECORDS / 100));
testutil_check(session->join(session, joincur, balancecur, cfg));
flagcur->set_key(flagcur, 0);
testutil_check(flagcur->search(flagcur));
testutil_check(__wt_snprintf(cfg, sizeof(cfg),
"compare=eq,strategy=bloom,count=%d", N_RECORDS / 100));
testutil_check(session->join(session, joincur, flagcur, cfg));
/* Expect no values returned */
count = 0;
while ((ret = joincur->next(joincur)) == 0) {
/*
* The values may already have been changed, but
* print them for informational purposes.
*/
testutil_check(joincur->get_key(joincur, &key));
testutil_check(joincur->get_value(joincur, &post,
&balance, &flag, &key2));
fprintf(stderr, "FAIL: "
"key=%d/%d, postal_code=%d, balance=%d, flag=%d\n",
key, key2, post, balance, flag);
count++;
}
testutil_assert(ret == WT_NOTFOUND);
testutil_assert(count == 0);
testutil_progress(opts, "cleanup starting");
testutil_cleanup(opts);
return (EXIT_SUCCESS);
}
