void
wts_rebalance(void)
{
WT_CONNECTION *conn;
WT_SESSION *session;
char cmd[1024];
if (g.c_rebalance == 0)
return;
track("rebalance", 0ULL, NULL);
/* Dump the current object. */
testutil_check(__wt_snprintf(cmd, sizeof(cmd),
".." DIR_DELIM_STR ".." DIR_DELIM_STR "wt"
" -h %s dump -f %s/rebalance.orig %s",
g.home, g.home, g.uri));
testutil_checkfmt(system(cmd), "command failed: %s", cmd);
/* Rebalance, then verify the object. */
wts_reopen();
conn = g.wts_conn;
testutil_check(conn->open_session(conn, NULL, NULL, &session));
if (g.logging != 0)
(void)g.wt_api->msg_printf(g.wt_api, session,
"=============== rebalance start ===============");
testutil_checkfmt(
session->rebalance(session, g.uri, NULL), "%s", g.uri);
if (g.logging != 0)
(void)g.wt_api->msg_printf(g.wt_api, session,
"=============== rebalance stop ===============");
testutil_check(session->close(session, NULL));
wts_verify("post-rebalance verify");
wts_close();
testutil_check(__wt_snprintf(cmd, sizeof(cmd),
".." DIR_DELIM_STR ".." DIR_DELIM_STR "wt"
" -h %s dump -f %s/rebalance.new %s",
g.home, g.home, g.uri));
testutil_checkfmt(system(cmd), "command failed: %s", cmd);
/* Compare the old/new versions of the object. */
#ifdef _WIN32
testutil_check(__wt_snprintf(cmd, sizeof(cmd),
"fc /b %s\\rebalance.orig %s\\rebalance.new > NUL",
g.home, g.home));
#else
testutil_check(__wt_snprintf(cmd, sizeof(cmd),
"cmp %s/rebalance.orig %s/rebalance.new > /dev/null",
g.home, g.home));
#endif
testutil_checkfmt(system(cmd), "command failed: %s", cmd);
}
static inline void
check(uint32_t hw, uint32_t sw, size_t len, const char *msg)
{
testutil_checkfmt(hw == sw ? 0 : 1,
"%s checksum mismatch of %" WT_SIZET_FMT " bytes: %#08x != %#08x\n",
msg, len, hw, sw);
}
/*
* check_copy --
* Confirm the backup worked.
*/
static void
check_copy(void)
{
WT_CONNECTION *conn;
WT_SESSION *session;
wts_open(g.home_backup, 0, &conn);
testutil_checkfmt(
conn->open_session(conn, NULL, NULL, &session),
"%s", g.home_backup);
testutil_checkfmt(
session->verify(session, g.uri, NULL),
"%s: %s", g.home_backup, g.uri);
testutil_checkfmt(conn->close(conn, NULL), "%s", g.home_backup);
}
static void
check_str(uint64_t i, char *str, bool mod)
{
char str2[] = "0000000000000000";
rand_str(i, str2);
if (mod)
str2[0] = 'A';
testutil_checkfmt(strcmp(str, str2),
"strcmp failed, got %s, expected %s", str, str2);
}
/*
* copy_file --
* Copy a single file into the backup directory.
*/
static void
copy_file(const char *name)
{
size_t len;
char *cmd;
len = strlen(g.home) + strlen(g.home_backup) + strlen(name) * 2 + 20;
cmd = dmalloc(len);
(void)snprintf(cmd, len,
"cp %s/%s %s/%s", g.home, name, g.home_backup, name);
testutil_checkfmt(system(cmd), "backup copy: %s", cmd);
free(cmd);
len = strlen(g.home) + strlen(g.home_backup2) + strlen(name) * 2 + 20;
cmd = dmalloc(len);
(void)snprintf(cmd, len,
"cp %s/%s %s/%s", g.home, name, g.home_backup2, name);
testutil_checkfmt(system(cmd), "backup copy: %s", cmd);
free(cmd);
}
/*
* wts_read_scan --
* Read and verify all elements in a file.
*/
void
wts_read_scan(void)
{
WT_CONNECTION *conn;
WT_CURSOR *cursor;
WT_ITEM key;
WT_SESSION *session;
uint64_t cnt, last_cnt;
uint8_t *keybuf;
conn = g.wts_conn;
/* Set up the default key buffer. */
key_gen_setup(&keybuf);
/* Open a session and cursor pair. */
testutil_check(conn->open_session(
conn, NULL, ops_session_config(NULL), &session));
testutil_check(session->open_cursor(
session, g.uri, NULL, NULL, &cursor));
/* Check a random subset of the records using the key. */
for (last_cnt = cnt = 0; cnt < g.key_cnt;) {
cnt += mmrand(NULL, 1, 17);
if (cnt > g.rows)
cnt = g.rows;
if (cnt - last_cnt > 1000) {
track("read row scan", cnt, NULL);
last_cnt = cnt;
}
key.data = keybuf;
testutil_checkfmt(
read_row(cursor, &key, cnt, 0), "%s", "read_scan");
}
testutil_check(session->close(session, NULL));
free(keybuf);
}
/*
* backup --
* Periodically do a backup and verify it.
*/
void *
backup(void *arg)
{
WT_CONNECTION *conn;
WT_CURSOR *backup_cursor;
WT_SESSION *session;
u_int period;
int ret;
const char *key;
(void)(arg);
conn = g.wts_conn;
/* Backups aren't supported for non-standard data sources. */
if (DATASOURCE("helium") || DATASOURCE("kvsbdb"))
return (NULL);
/* Open a session. */
testutil_check(conn->open_session(conn, NULL, NULL, &session));
/*
* Perform a backup at somewhere under 10 seconds (so we get at
* least one done), and then at 45 second intervals.
*/
for (period = mmrand(NULL, 1, 10);; period = 45) {
/* Sleep for short periods so we don't make the run wait. */
while (period > 0 && !g.workers_finished) {
--period;
sleep(1);
}
if (g.workers_finished)
break;
/* Lock out named checkpoints */
testutil_check(pthread_rwlock_wrlock(&g.backup_lock));
/* Re-create the backup directory. */
testutil_checkfmt(
system(g.home_backup_init),
"%s", "backup directory creation failed");
/*
* open_cursor can return EBUSY if a metadata operation is
* currently happening - retry in that case.
*/
while ((ret = session->open_cursor(session,
"backup:", NULL, NULL, &backup_cursor)) == EBUSY)
sleep(1);
if (ret != 0)
testutil_die(ret, "session.open_cursor: backup");
while ((ret = backup_cursor->next(backup_cursor)) == 0) {
testutil_check(
backup_cursor->get_key(backup_cursor, &key));
copy_file(key);
}
testutil_check(backup_cursor->close(backup_cursor));
testutil_check(pthread_rwlock_unlock(&g.backup_lock));
check_copy();
}
testutil_check(session->close(session, NULL));
return (NULL);
}
static void *
ops(void *arg)
{
TINFO *tinfo;
WT_CONNECTION *conn;
WT_CURSOR *cursor, *cursor_insert;
WT_SESSION *session;
WT_ITEM key, value;
uint64_t keyno, ckpt_op, reset_op, session_op;
uint32_t op;
uint8_t *keybuf, *valbuf;
u_int np;
int ckpt_available, dir, insert, intxn, notfound, readonly;
char *ckpt_config, ckpt_name[64];
tinfo = arg;
conn = g.wts_conn;
keybuf = valbuf = NULL;
readonly = 0; /* -Wconditional-uninitialized */
/* Initialize the per-thread random number generator. */
__wt_random_init(&tinfo->rnd);
/* Set up the default key and value buffers. */
key_gen_setup(&keybuf);
val_gen_setup(&tinfo->rnd, &valbuf);
/* Set the first operation where we'll create sessions and cursors. */
session_op = 0;
session = NULL;
cursor = cursor_insert = NULL;
/* Set the first operation where we'll perform checkpoint operations. */
ckpt_op = g.c_checkpoints ? mmrand(&tinfo->rnd, 100, 10000) : 0;
ckpt_available = 0;
/* Set the first operation where we'll reset the session. */
reset_op = mmrand(&tinfo->rnd, 100, 10000);
for (intxn = 0; !tinfo->quit; ++tinfo->ops) {
/*
* We can't checkpoint or swap sessions/cursors while in a
* transaction, resolve any running transaction.
*/
if (intxn &&
(tinfo->ops == ckpt_op || tinfo->ops == session_op)) {
testutil_check(
session->commit_transaction(session, NULL));
++tinfo->commit;
intxn = 0;
}
/* Open up a new session and cursors. */
if (tinfo->ops == session_op ||
session == NULL || cursor == NULL) {
if (session != NULL)
testutil_check(session->close(session, NULL));
testutil_check(conn->open_session(conn, NULL,
ops_session_config(&tinfo->rnd), &session));
/*
* 10% of the time, perform some read-only operations
* from a checkpoint.
*
* Skip that if we single-threaded and doing checks
* against a Berkeley DB database, because that won't
* work because the Berkeley DB database records won't
* match the checkpoint. Also skip if we are using
* LSM, because it doesn't support reads from
* checkpoints.
*/
if (!SINGLETHREADED && !DATASOURCE("lsm") &&
ckpt_available && mmrand(&tinfo->rnd, 1, 10) == 1) {
testutil_check(session->open_cursor(session,
g.uri, NULL, ckpt_name, &cursor));
/* Pick the next session/cursor close/open. */
session_op += 250;
/* Checkpoints are read-only. */
readonly = 1;
} else {
/*
* Open two cursors: one for overwriting and one
* for append (if it's a column-store).
*
* The reason is when testing with existing
* records, we don't track if a record was
* deleted or not, which means we must use
* cursor->insert with overwriting configured.
* But, in column-store files where we're
* testing with new, appended records, we don't
* want to have to specify the record number,
* which requires an append configuration.
*/
testutil_check(session->open_cursor(session,
g.uri, NULL, "overwrite", &cursor));
if (g.type == FIX || g.type == VAR)
testutil_check(session->open_cursor(
session, g.uri,
NULL, "append", &cursor_insert));
/* Pick the next session/cursor close/open. */
session_op += mmrand(&tinfo->rnd, 100, 5000);
/* Updates supported. */
readonly = 0;
}
}
/* Checkpoint the database. */
if (tinfo->ops == ckpt_op && g.c_checkpoints) {
/*
* LSM and data-sources don't support named checkpoints,
* and we can't drop a named checkpoint while there's a
* cursor open on it, otherwise 20% of the time name the
* checkpoint.
*/
if (DATASOURCE("helium") || DATASOURCE("kvsbdb") ||
DATASOURCE("lsm") ||
readonly || mmrand(&tinfo->rnd, 1, 5) == 1)
ckpt_config = NULL;
else {
(void)snprintf(ckpt_name, sizeof(ckpt_name),
"name=thread-%d", tinfo->id);
ckpt_config = ckpt_name;
}
/* Named checkpoints lock out backups */
if (ckpt_config != NULL)
testutil_check(
pthread_rwlock_wrlock(&g.backup_lock));
testutil_checkfmt(
session->checkpoint(session, ckpt_config),
"%s", ckpt_config == NULL ? "" : ckpt_config);
if (ckpt_config != NULL)
testutil_check(
pthread_rwlock_unlock(&g.backup_lock));
/* Rephrase the checkpoint name for cursor open. */
if (ckpt_config == NULL)
strcpy(ckpt_name,
"checkpoint=WiredTigerCheckpoint");
else
(void)snprintf(ckpt_name, sizeof(ckpt_name),
"checkpoint=thread-%d", tinfo->id);
ckpt_available = 1;
/* Pick the next checkpoint operation. */
ckpt_op += mmrand(&tinfo->rnd, 5000, 20000);
}
/*
* Reset the session every now and then, just to make sure that
* operation gets tested. Note the test is not for equality, we
* have to do the reset outside of a transaction.
*/
if (tinfo->ops > reset_op && !intxn) {
testutil_check(session->reset(session));
/* Pick the next reset operation. */
reset_op += mmrand(&tinfo->rnd, 20000, 50000);
}
/*
* If we're not single-threaded and we're not in a transaction,
* start a transaction 20% of the time.
*/
if (!SINGLETHREADED &&
!intxn && mmrand(&tinfo->rnd, 1, 10) >= 8) {
testutil_check(
session->begin_transaction(session, NULL));
intxn = 1;
}
insert = notfound = 0;
keyno = mmrand(&tinfo->rnd, 1, (u_int)g.rows);
key.data = keybuf;
value.data = valbuf;
/*
* Perform some number of operations: the percentage of deletes,
* inserts and writes are specified, reads are the rest. The
* percentages don't have to add up to 100, a high percentage
* of deletes will mean fewer inserts and writes. Modifications
* are always followed by a read to confirm it worked.
*/
op = readonly ? UINT32_MAX : mmrand(&tinfo->rnd, 1, 100);
if (op < g.c_delete_pct) {
++tinfo->remove;
switch (g.type) {
case ROW:
/*
* If deleting a non-existent record, the cursor
* won't be positioned, and so can't do a next.
*/
if (row_remove(cursor, &key, keyno, ¬found))
goto deadlock;
break;
case FIX:
case VAR:
if (col_remove(cursor, &key, keyno, ¬found))
goto deadlock;
break;
}
} else if (op < g.c_delete_pct + g.c_insert_pct) {
++tinfo->insert;
switch (g.type) {
case ROW:
if (row_insert(
tinfo, cursor, &key, &value, keyno))
goto deadlock;
insert = 1;
break;
case FIX:
case VAR:
/*
* We can only append so many new records, if
* we've reached that limit, update a record
* instead of doing an insert.
*/
if (g.append_cnt >= g.append_max)
goto skip_insert;
/* Insert, then reset the insert cursor. */
if (col_insert(tinfo,
cursor_insert, &key, &value, &keyno))
goto deadlock;
testutil_check(
cursor_insert->reset(cursor_insert));
insert = 1;
break;
}
} else if (
op < g.c_delete_pct + g.c_insert_pct + g.c_write_pct) {
++tinfo->update;
switch (g.type) {
case ROW:
if (row_update(
tinfo, cursor, &key, &value, keyno))
goto deadlock;
break;
case FIX:
case VAR:
skip_insert:
if (col_update(tinfo,
cursor, &key, &value, keyno))
goto deadlock;
break;
}
} else {
++tinfo->search;
if (read_row(cursor, &key, keyno, 0))
if (intxn)
goto deadlock;
continue;
}
/*
* The cursor is positioned if we did any operation other than
* insert, do a small number of next/prev cursor operations in
* a random direction.
*/
if (!insert) {
dir = (int)mmrand(&tinfo->rnd, 0, 1);
for (np = 0; np < mmrand(&tinfo->rnd, 1, 100); ++np) {
if (notfound)
break;
if (nextprev(cursor, dir, ¬found))
goto deadlock;
}
}
/* Read to confirm the operation. */
++tinfo->search;
if (read_row(cursor, &key, keyno, 0))
goto deadlock;
/* Reset the cursor: there is no reason to keep pages pinned. */
testutil_check(cursor->reset(cursor));
/*
* If we're in the transaction, commit 40% of the time and
* rollback 10% of the time.
*/
if (intxn)
switch (mmrand(&tinfo->rnd, 1, 10)) {
case 1:
case 2:
case 3:
case 4: /* 40% */
testutil_check(session->commit_transaction(
session, NULL));
++tinfo->commit;
intxn = 0;
break;
case 5: /* 10% */
if (0) {
deadlock:
++tinfo->deadlock;
}
testutil_check(session->rollback_transaction(
session, NULL));
++tinfo->rollback;
intxn = 0;
break;
default:
break;
}
}
if (session != NULL)
testutil_check(session->close(session, NULL));
free(keybuf);
free(valbuf);
tinfo->state = TINFO_COMPLETE;
return (NULL);
}
/*
* backup --
* Periodically do a backup and verify it.
*/
void *
backup(void *arg)
{
WT_CONNECTION *conn;
WT_CURSOR *backup_cursor;
WT_DECL_RET;
WT_SESSION *session;
u_int incremental, period;
bool full;
const char *config, *key;
(void)(arg);
conn = g.wts_conn;
/* Backups aren't supported for non-standard data sources. */
if (DATASOURCE("helium") || DATASOURCE("kvsbdb"))
return (NULL);
/* Open a session. */
testutil_check(conn->open_session(conn, NULL, NULL, &session));
/*
* Perform a full backup at somewhere under 10 seconds (that way there's
* at least one), then at larger intervals, optionally do incremental
* backups between full backups.
*/
incremental = 0;
for (period = mmrand(NULL, 1, 10);; period = mmrand(NULL, 20, 45)) {
/* Sleep for short periods so we don't make the run wait. */
while (period > 0 && !g.workers_finished) {
--period;
sleep(1);
}
/*
* We can't drop named checkpoints while there's a backup in
* progress, serialize backups with named checkpoints. Wait
* for the checkpoint to complete, otherwise backups might be
* starved out.
*/
testutil_check(pthread_rwlock_wrlock(&g.backup_lock));
if (g.workers_finished) {
testutil_check(pthread_rwlock_unlock(&g.backup_lock));
break;
}
if (incremental) {
config = "target=(\"log:\")";
full = false;
} else {
/* Re-create the backup directory. */
testutil_checkfmt(
system(g.home_backup_init),
"%s", "backup directory creation failed");
config = NULL;
full = true;
}
/*
* open_cursor can return EBUSY if concurrent with a metadata
* operation, retry in that case.
*/
while ((ret = session->open_cursor(
session, "backup:", NULL, config, &backup_cursor)) == EBUSY)
__wt_yield();
if (ret != 0)
testutil_die(ret, "session.open_cursor: backup");
while ((ret = backup_cursor->next(backup_cursor)) == 0) {
testutil_check(
backup_cursor->get_key(backup_cursor, &key));
copy_file(session, key);
}
if (ret != WT_NOTFOUND)
testutil_die(ret, "backup-cursor");
/* After an incremental backup, truncate the log files. */
if (incremental)
testutil_check(session->truncate(
session, "log:", backup_cursor, NULL, NULL));
testutil_check(backup_cursor->close(backup_cursor));
testutil_check(pthread_rwlock_unlock(&g.backup_lock));
/*
* If automatic log archival isn't configured, optionally do
* incremental backups after each full backup. If we're not
* doing any more incrementals, verify the backup (we can't
* verify intermediate states, once we perform recovery on the
* backup database, we can't do any more incremental backups).
*/
if (full)
incremental =
g.c_logging_archive ? 1 : mmrand(NULL, 1, 5);
if (--incremental == 0)
check_copy();
}
if (incremental != 0)
check_copy();
testutil_check(session->close(session, NULL));
return (NULL);
}
