/*
* config_isolation --
* Isolation configuration.
*/
static void
config_isolation(void)
{
CONFIG *cp;
const char *cstr;
/*
* Isolation: choose something if isolation wasn't specified.
*/
cp = config_find("isolation", strlen("isolation"));
if (!(cp->flags & C_PERM)) {
/* Avoid "maybe uninitialized" warnings. */
switch (MMRAND(1, 4)) {
case 1:
cstr = "isolation=random";
break;
case 2:
cstr = "isolation=read-uncommitted";
break;
case 3:
cstr = "isolation=read-committed";
break;
case 4:
default:
cstr = "isolation=snapshot";
break;
}
config_single(cstr, 0);
}
}
void
value_gen(uint8_t *val, uint32_t *sizep, uint64_t keyno)
{
/*
* Fixed-length records: take the low N bits from the last digit of
* the record number.
*/
if (g.type == FIX) {
switch (g.c_bitcnt) {
case 8: val[0] = MMRAND(1, 0xff); break;
case 7: val[0] = MMRAND(1, 0x7f); break;
case 6: val[0] = MMRAND(1, 0x3f); break;
case 5: val[0] = MMRAND(1, 0x1f); break;
case 4: val[0] = MMRAND(1, 0x0f); break;
case 3: val[0] = MMRAND(1, 0x07); break;
case 2: val[0] = MMRAND(1, 0x03); break;
case 1: val[0] = 1; break;
}
*sizep = 1;
return;
}
/*
* WiredTiger doesn't store zero-length data items in row-store files,
* test that by inserting a zero-length data item every so often.
* LSM doesn't support zero length items.
*/
if (keyno % 63 == 0 && strcmp("lsm", g.c_data_source) != 0) {
val[0] = '\0';
*sizep = 0;
return;
}
/*
* Start the data with a 10-digit number.
*
* For row and non-repeated variable-length column-stores, change the
* leading number to ensure every data item is unique. For repeated
* variable-length column-stores (that is, to test run-length encoding),
* use the same data value all the time.
*/
if ((g.type == ROW || g.type == VAR) &&
g.c_repeat_data_pct != 0 && MMRAND(1, 100) > g.c_repeat_data_pct) {
(void)strcpy((char *)val, "DUPLICATEV");
val[10] = '/';
*sizep = val_dup_data_len;
} else {
(void)sprintf((char *)val, "%010" PRIu64, keyno);
val[10] = '/';
*sizep = MMRAND(g.c_value_min, g.c_value_max);
}
}
/*
* config_compression --
* Compression configuration.
*/
static void
config_compression(void)
{
CONFIG *cp;
const char *cstr;
/*
* Compression: choose something if compression wasn't specified,
* otherwise confirm the appropriate shared library is available.
* We don't include LZO in the test compression choices, we don't
* yet have an LZO module of our own.
*/
cp = config_find("compression", strlen("compression"));
if (!(cp->flags & C_PERM)) {
cstr = "compression=none";
switch (MMRAND(1, 10)) {
case 1: case 2: case 3: /* 30% */
break;
case 4: case 5: /* 20% */
if (access(BZIP_PATH, R_OK) == 0)
cstr = "compression=bzip";
break;
case 6: /* 10% */
if (access(BZIP_PATH, R_OK) == 0)
cstr = "compression=bzip-raw";
break;
case 7: case 8: /* 20% */
if (access(SNAPPY_PATH, R_OK) == 0)
cstr = "compression=snappy";
break;
case 9: case 10: /* 20% */
if (access(ZLIB_PATH, R_OK) == 0)
cstr = "compression=zlib";
break;
}
config_single(cstr, 0);
}
switch (g.c_compression_flag) {
case COMPRESS_BZIP:
case COMPRESS_BZIP_RAW:
if (access(BZIP_PATH, R_OK) != 0)
die(0, "bzip library not found or not readable");
break;
case COMPRESS_LZO:
if (access(LZO_PATH, R_OK) != 0)
die(0, "LZO library not found or not readable");
break;
case COMPRESS_SNAPPY:
if (access(SNAPPY_PATH, R_OK) != 0)
die(0, "snappy library not found or not readable");
break;
case COMPRESS_ZLIB:
if (access(ZLIB_PATH, R_OK) != 0)
die(0, "zlib library not found or not readable");
break;
}
}
void
value_gen(uint8_t *val, uint32_t *sizep, uint64_t keyno)
{
static const char *dup_data = "duplicate data item";
/*
* Fixed-length records: take the low N bits from the last digit of
* the record number.
*/
if (g.c_file_type == FIX) {
switch (g.c_bitcnt) {
case 8: val[0] = MMRAND(1, 0xff); break;
case 7: val[0] = MMRAND(1, 0x7f); break;
case 6: val[0] = MMRAND(1, 0x3f); break;
case 5: val[0] = MMRAND(1, 0x1f); break;
case 4: val[0] = MMRAND(1, 0x0f); break;
case 3: val[0] = MMRAND(1, 0x07); break;
case 2: val[0] = MMRAND(1, 0x03); break;
case 1: val[0] = 1; break;
}
*sizep = 1;
return;
}
/*
* WiredTiger doesn't store zero-length data items in row-store files,
* test that by inserting a zero-length data item every so often.
*/
if (++keyno % 63 == 0) {
val[0] = '\0';
*sizep = 0;
return;
}
/*
* Start the data with a 10-digit number.
*
* For row and non-repeated variable-length column-stores, change the
* leading number to ensure every data item is unique. For repeated
* variable-length column-stores (that is, to test run-length encoding),
* use the same data value all the time.
*/
if (g.c_file_type == VAR &&
g.c_repeat_data_pct != 0 &&
(u_int)wts_rand() % 100 > g.c_repeat_data_pct) {
(void)strcpy((char *)val, dup_data);
*sizep = (uint32_t)strlen(dup_data);
return;
}
sprintf((char *)val, "%010" PRIu64, keyno);
val[10] = '/';
*sizep = MMRAND(g.c_value_min, g.c_value_max);
}
void
key_len_setup()
{
size_t i;
/*
* The key is a variable length item with a leading 10-digit value.
* Since we have to be able re-construct it from the record number
* (when doing row lookups), we pre-load a set of random lengths in
* a lookup table, and then use the record number to choose one of
* the pre-loaded lengths.
*
* Fill in the random key lengths.
*/
for (i = 0; i < sizeof(g.key_rand_len) / sizeof(g.key_rand_len[0]); ++i)
g.key_rand_len[i] = (uint16_t)MMRAND(g.c_key_min, g.c_key_max);
}
/*
* config_checksum --
* Checksum configuration.
*/
static void
config_checksum(void)
{
CONFIG *cp;
/* Choose a checksum mode if nothing was specified. */
cp = config_find("checksum", strlen("checksum"));
if (!(cp->flags & C_PERM))
switch (MMRAND(1, 10)) {
case 1: /* 10% */
config_single("checksum=on", 0);
break;
case 2: /* 10% */
config_single("checksum=off", 0);
break;
default: /* 80% */
config_single("checksum=uncompressed", 0);
break;
}
}
void
val_gen_setup(uint8_t **valp)
{
uint8_t *val;
size_t i, len;
/*
* Set initial buffer contents to recognizable text.
*
* Add a few extra bytes in order to guarantee we can always offset
* into the buffer by a few extra bytes, used to generate different
* data for column-store run-length encoded files.
*/
len = g.c_value_max + 20;
if ((val = malloc(len)) == NULL)
syserr("malloc");
for (i = 0; i < len; ++i)
val[i] = (uint8_t)("ABCDEFGHIJKLMNOPQRSTUVWXYZ"[i % 26]);
*valp = val;
val_dup_data_len = MMRAND(g.c_value_min, g.c_value_max);
}
void
key_gen(uint8_t *key, uint32_t *sizep, uint64_t keyno, int insert)
{
int len, suffix;
/*
* The key always starts with a 10-digit string (the specified cnt)
* followed by two digits, a random number between 1 and 15 if it's
* an insert, otherwise 00.
*/
suffix = insert ? (int)MMRAND(1, 15) : 0;
len = sprintf((char *)key, "%010" PRIu64 ".%02d", keyno, suffix);
/*
* In a column-store, the key is only used for BDB, and so it doesn't
* need a random length.
*/
if (g.c_file_type == ROW) {
key[len] = '/';
len = g.key_rand_len[keyno %
(sizeof(g.key_rand_len) / sizeof(g.key_rand_len[0]))];
}
*sizep = (uint32_t)len;
}
void
wts_open(void)
{
WT_CONNECTION *conn;
WT_SESSION *session;
uint32_t maxintlpage, maxintlitem, maxleafpage, maxleafitem;
int ret;
const char *ext1, *ext2;
char config[512], *end, *p;
/* If the bzip2 compression module has been built, use it. */
#define EXTPATH "../../ext"
ext1 = EXTPATH "compressors/bzip2_compress/.libs/bzip2_compress.so";
if (access(ext1, R_OK) != 0) {
ext1 = "";
g.c_bzip = 0;
}
ext2 = EXTPATH "/collators/reverse/.libs/reverse_collator.so";
/*
* Open configuration -- put command line configuration options at the
* end so they can override "standard" configuration.
*/
snprintf(config, sizeof(config),
"create,error_prefix=\"%s\",cache_size=%" PRIu32 "MB,sync=false,"
"extensions=[\"%s\",\"%s\"],%s",
g.progname, g.c_cache, ext1, ext2,
g.config_open == NULL ? "" : g.config_open);
if ((ret =
wiredtiger_open("RUNDIR", &event_handler, config, &conn)) != 0)
die(ret, "wiredtiger_open");
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
die(ret, "connection.open_session");
maxintlpage = 1U << g.c_intl_page_max;
/* Make sure at least 2 internal page per thread can fit in cache. */
while (2 * g.c_threads * maxintlpage > g.c_cache << 20)
maxintlpage >>= 1;
maxintlitem = MMRAND(maxintlpage / 50, maxintlpage / 40);
if (maxintlitem < 40)
maxintlitem = 40;
maxleafpage = 1U << g.c_leaf_page_max;
/* Make sure at least one leaf page per thread can fit in cache. */
while (g.c_threads * (maxintlpage + maxleafpage) > g.c_cache << 20)
maxleafpage >>= 1;
maxleafitem = MMRAND(maxleafpage / 50, maxleafpage / 40);
if (maxleafitem < 40)
maxleafitem = 40;
p = config;
end = config + sizeof(config);
p += snprintf(p, (size_t)(end - p),
"key_format=%s,"
"internal_page_max=%d,internal_item_max=%d,"
"leaf_page_max=%d,leaf_item_max=%d",
(g.type == ROW) ? "u" : "r",
maxintlpage, maxintlitem, maxleafpage, maxleafitem);
if (g.c_bzip)
p += snprintf(p, (size_t)(end - p),
",block_compressor=\"bzip2_compress\"");
switch (g.type) {
case FIX:
p += snprintf(p, (size_t)(end - p),
",value_format=%dt", g.c_bitcnt);
break;
case ROW:
if (g.c_huffman_key)
p += snprintf(p, (size_t)(end - p),
",huffman_key=english");
if (g.c_reverse)
p += snprintf(p, (size_t)(end - p),
",collator=reverse");
/* FALLTHROUGH */
case VAR:
if (g.c_huffman_value)
p += snprintf(p, (size_t)(end - p),
",huffman_value=english");
if (g.c_dictionary)
p += snprintf(p, (size_t)(end - p),
",dictionary=%d", MMRAND(123, 517));
break;
}
if ((ret = session->create(session, g.uri, config)) != 0)
die(ret, "session.create: %s", g.uri);
if ((ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
g.wts_conn = conn;
}
static void *
ops(void *arg)
{
TINFO *tinfo;
WT_CONNECTION *conn;
WT_CURSOR *cursor, *cursor_insert;
WT_SESSION *session;
WT_ITEM key, value;
uint64_t cnt, keyno, sync_op, thread_ops;
uint32_t op;
uint8_t *keybuf, *valbuf;
u_int np;
int dir, insert, notfound, ret, sync_drop;
char sync_name[64];
conn = g.wts_conn;
tinfo = arg;
/* Set up the default key and value buffers. */
memset(&key, 0, sizeof(key));
key_gen_setup(&keybuf);
memset(&value, 0, sizeof(value));
val_gen_setup(&valbuf);
/* Open a session. */
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
die(ret, "connection.open_session");
/*
* Open two cursors: one configured for overwriting and one configured
* for append if we're dealing with a column-store.
*
* The reason is when testing with existing records, we don't track if
* a record was deleted or not, which means we must use cursor->insert
* with overwriting configured. But, in column-store files where we're
* testing with new, appended records, we don't want to have to specify
* the record number, which requires an append configuration.
*/
cursor = cursor_insert = NULL;
if ((ret = session->open_cursor(session,
WT_TABLENAME, NULL, "overwrite", &cursor)) != 0)
die(ret, "session.open_cursor");
if ((g.c_file_type == FIX || g.c_file_type == VAR) &&
(ret = session->open_cursor(session,
WT_TABLENAME, NULL, "append", &cursor_insert)) != 0)
die(ret, "session.open_cursor");
/* Each thread does its share of the total operations. */
thread_ops = g.c_ops / g.c_threads;
/* Pick an operation where we'll do a sync and create the name. */
sync_drop = 0;
sync_op = MMRAND(1, thread_ops);
snprintf(sync_name, sizeof(sync_name), "snapshot=thread-%d", tinfo->id);
for (cnt = 0; cnt < thread_ops; ++cnt) {
if (SINGLETHREADED && cnt % 100 == 0)
track("read/write ops", 0ULL, tinfo);
if (cnt == sync_op) {
if (sync_drop && (int)MMRAND(1, 4) == 1) {
if ((ret = session->drop(
session, WT_TABLENAME, sync_name)) != 0)
die(ret, "session.drop: %s: %s",
WT_TABLENAME, sync_name);
sync_drop = 0;
} else {
if ((ret = session->checkpoint(
session, sync_name)) != 0)
die(ret, "session.checkpoint: %s",
sync_name);
sync_drop = 1;
}
/*
* Pick the next sync operation, try for roughly five
* snapshot operations per thread run.
*/
sync_op += MMRAND(1, thread_ops) / 5;
}
insert = notfound = 0;
keyno = MMRAND(1, g.rows);
key.data = keybuf;
value.data = valbuf;
/*
* Perform some number of operations: the percentage of deletes,
* inserts and writes are specified, reads are the rest. The
* percentages don't have to add up to 100, a high percentage
* of deletes will mean fewer inserts and writes. A read
* operation always follows a modification to confirm it worked.
*/
op = (uint32_t)(wts_rand() % 100);
if (op < g.c_delete_pct) {
++tinfo->remove;
switch (g.c_file_type) {
case ROW:
/*
* If deleting a non-existent record, the cursor
* won't be positioned, and so can't do a next.
*/
row_remove(cursor, &key, keyno, ¬found);
break;
case FIX:
case VAR:
col_remove(cursor, &key, keyno, ¬found);
break;
}
} else if (op < g.c_delete_pct + g.c_insert_pct) {
++tinfo->insert;
switch (g.c_file_type) {
case ROW:
row_update(cursor, &key, &value, keyno, 1);
break;
case FIX:
case VAR:
/*
* Reset the standard cursor so it doesn't keep
* pages pinned.
*/
if ((ret = cursor->reset(cursor)) != 0)
die(ret, "cursor.reset");
col_insert(cursor_insert, &key, &value, &keyno);
insert = 1;
break;
}
} else if (
op < g.c_delete_pct + g.c_insert_pct + g.c_write_pct) {
++tinfo->update;
switch (g.c_file_type) {
case ROW:
row_update(cursor, &key, &value, keyno, 0);
break;
case FIX:
case VAR:
col_update(cursor, &key, &value, keyno);
break;
}
} else {
++tinfo->search;
read_row(cursor, &key, keyno);
continue;
}
/*
* If we did any operation, we've set the cursor, do a small
* number of next/prev cursor operations in a random direction.
*/
dir = (int)MMRAND(0, 1);
for (np = 0; np < MMRAND(1, 8); ++np) {
if (notfound)
break;
nextprev(
insert ? cursor_insert : cursor, dir, ¬found);
}
if (insert && (ret = cursor_insert->reset(cursor_insert)) != 0)
die(ret, "cursor.reset");
/* Read the value we modified to confirm the operation. */
read_row(cursor, &key, keyno);
}
if ((ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
free(keybuf);
free(valbuf);
tinfo->state = TINFO_COMPLETE;
return (NULL);
}
/*
* backup --
* Periodically do a backup and verify it.
*/
void *
backup(void *arg)
{
WT_CONNECTION *conn;
WT_CURSOR *backup_cursor;
WT_SESSION *session;
u_int period;
int ret;
const char *key;
(void)(arg);
conn = g.wts_conn;
/* Backups aren't supported for non-standard data sources. */
if (DATASOURCE("helium") || DATASOURCE("kvsbdb"))
return (NULL);
/* Open a session. */
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
die(ret, "connection.open_session");
/*
* Perform a backup at somewhere under 10 seconds (so we get at
* least one done), and then at 45 second intervals.
*/
for (period = MMRAND(1, 10);; period = 45) {
/* Sleep for short periods so we don't make the run wait. */
while (period > 0 && !g.workers_finished) {
--period;
sleep(1);
}
if (g.workers_finished)
break;
/* Lock out named checkpoints */
if ((ret = pthread_rwlock_wrlock(&g.backup_lock)) != 0)
die(ret, "pthread_rwlock_wrlock: backup lock");
/* Re-create the backup directory. */
if ((ret = system(g.home_backup_init)) != 0)
die(ret, "backup directory creation failed");
/*
* open_cursor can return EBUSY if a metadata operation is
* currently happening - retry in that case.
*/
while ((ret = session->open_cursor(session,
"backup:", NULL, NULL, &backup_cursor)) == EBUSY)
sleep(1);
if (ret != 0)
die(ret, "session.open_cursor: backup");
while ((ret = backup_cursor->next(backup_cursor)) == 0) {
if ((ret =
backup_cursor->get_key(backup_cursor, &key)) != 0)
die(ret, "cursor.get_key");
copy_file(key);
}
if ((ret = backup_cursor->close(backup_cursor)) != 0)
die(ret, "cursor.close");
if ((ret = pthread_rwlock_unlock(&g.backup_lock)) != 0)
die(ret, "pthread_rwlock_unlock: backup lock");
check_copy();
}
if ((ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
return (NULL);
}
/*
* config_setup --
* Initialize configuration for a run.
*/
void
config_setup(void)
{
CONFIG *cp;
/* Clear any temporary values. */
config_clear();
/*
* Choose a data source type and a file type: they're interrelated (LSM
* trees are only compatible with row-store) and other items depend on
* them.
*/
if (!config_find_is_perm("data_source", strlen("data_source")))
switch (MMRAND(0, 2)) {
case 0:
config_single("data_source=file", 0);
break;
case 1:
#if 0
config_single("data_source=lsm", 0);
break;
#endif
case 2:
config_single("data_source=table", 0);
break;
}
if (!config_find_is_perm("file_type", strlen("file_type"))) {
if (strcmp(g.c_data_source, "lsm") == 0)
config_single("file_type=row", 0);
else
switch (MMRAND(0, 2)) {
case 0:
config_single("file_type=fix", 0);
break;
case 1:
config_single("file_type=var", 0);
break;
case 2:
config_single("file_type=row", 0);
break;
}
}
g.type = config_translate(g.c_file_type);
/*
* If data_source and file_type were both "permanent", we may still
* have a mismatch.
*/
if (g.type != ROW && strcmp(g.c_data_source, "lsm") == 0) {
fprintf(stderr,
"%s: lsm data_source is only compatible with row file_type\n",
g.progname);
exit(EXIT_FAILURE);
}
/* Build the object name. */
if ((g.uri = malloc(
strlen(g.c_data_source) + strlen(WT_NAME) + 2)) == NULL)
syserr("malloc");
strcpy(g.uri, g.c_data_source);
strcat(g.uri, ":");
strcat(g.uri, WT_NAME);
/* Default single-threaded half of the time. */
cp = config_find("threads", strlen("threads"));
if (!(cp->flags & C_PERM))
*cp->v = MMRAND(0, 1) ? 1: CONF_RAND(cp);
/* Fill in random values for the rest of the run. */
for (cp = c; cp->name != NULL; ++cp) {
if (cp->flags & (C_IGNORE | C_PERM | C_TEMP))
continue;
/*
* Boolean flags are 0 or 1, but only set N in 100 where the
* variable's min value is N. Set the flag if we rolled >=
* the min, 0 otherwise.
*/
if (cp->flags & C_BOOL)
*cp->v = MMRAND(1, 100) <= cp->min ? 1 : 0;
else
*cp->v = CONF_RAND(cp);
}
config_compression();
/* Clear operations values if the whole run is read-only. */
if (g.c_ops == 0)
for (cp = c; cp->name != NULL; ++cp)
if (cp->flags & C_OPS)
*cp->v = 0;
/* Multi-threaded runs cannot be replayed. */
if (g.replay && !SINGLETHREADED)
die(0, "-r is incompatible with threaded runs");
/*
* Periodically, set the delete percentage to 0 so salvage gets run,
* as long as the delete percentage isn't nailed down.
*/
if (!g.replay && g.run_cnt % 10 == 0) {
cp = config_find("delete_pct", strlen("delete_pct"));
if (cp->name != NULL &&
!(cp->flags & (C_IGNORE | C_PERM | C_TEMP)))
g.c_delete_pct = 0;
}
/* Reset the key count. */
g.key_cnt = 0;
}
/*
* hot_backup --
* Periodically do a hot backup and verify it.
*/
void *
hot_backup(void *arg)
{
WT_CONNECTION *conn;
WT_CURSOR *backup_cursor;
WT_SESSION *session;
u_int period;
int ret;
const char *key;
(void)arg;
/* If hot backups aren't configured, we're done. */
if (!g.c_hot_backups)
return (NULL);
/* Hot backups aren't supported for non-standard data sources. */
if (DATASOURCE("kvsbdb") || DATASOURCE("memrata"))
return (NULL);
conn = g.wts_conn;
/* Open a session. */
if ((ret = conn->open_session(
conn, NULL, NULL, &session)) != 0)
die(ret, "connection.open_session");
/*
* Perform a hot backup at somewhere under 10 seconds (so we get at
* least one done), and then at 45 second intervals.
*/
for (period = MMRAND(1, 10);; period = 45) {
/* Sleep for short periods so we don't make the run wait. */
while (period > 0 && !g.threads_finished) {
--period;
sleep(1);
}
if (g.threads_finished)
break;
/* Lock out named checkpoints */
if ((ret = pthread_rwlock_wrlock(&g.backup_lock)) != 0)
die(ret, "pthread_rwlock_wrlock: hot-backup lock");
/* Re-create the backup directory. */
(void)system("cd RUNDIR && rm -rf BACKUP");
if (mkdir(RUNDIR_BACKUP, 0777) != 0)
die(errno, "mkdir: %s", RUNDIR_BACKUP);
/*
* open_cursor can return EBUSY if a metadata operation is
* currently happening - retry in that case.
*/
while ((ret = session->open_cursor(session,
"backup:", NULL, NULL, &backup_cursor)) == EBUSY)
sleep(1);
if (ret != 0)
die(ret, "session.open_cursor: backup");
while ((ret = backup_cursor->next(backup_cursor)) == 0) {
if ((ret =
backup_cursor->get_key(backup_cursor, &key)) != 0)
die(ret, "cursor.get_key");
hot_copy(key);
}
if ((ret = backup_cursor->close(backup_cursor)) != 0)
die(ret, "cursor.close");
if ((ret = pthread_rwlock_unlock(&g.backup_lock)) != 0)
die(ret, "pthread_rwlock_unlock: hot-backup lock");
check_copy();
}
if ((ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
return (NULL);
}
static void *
ops(void *arg)
{
TINFO *tinfo;
WT_CONNECTION *conn;
WT_CURSOR *cursor, *cursor_insert;
WT_SESSION *session;
WT_ITEM key, value;
uint64_t cnt, keyno, ckpt_op, session_op, thread_ops;
uint32_t op;
uint8_t *keybuf, *valbuf;
u_int np;
int dir, insert, intxn, notfound, ret;
char *ckpt_config, config[64];
tinfo = arg;
conn = g.wts_conn;
keybuf = valbuf = NULL;
/* Set up the default key and value buffers. */
key_gen_setup(&keybuf);
val_gen_setup(&valbuf);
/*
* Each thread does its share of the total operations, and make sure
* that it's not 0 (testing runs: threads might be larger than ops).
*/
thread_ops = 100 + g.c_ops / g.c_threads;
/*
* Select the first operation where we'll create sessions and cursors,
* perform checkpoint operations.
*/
ckpt_op = MMRAND(1, thread_ops);
session_op = 0;
session = NULL;
cursor = cursor_insert = NULL;
for (intxn = 0, cnt = 0; cnt < thread_ops; ++cnt) {
if (SINGLETHREADED && cnt % 100 == 0)
track("ops", 0ULL, tinfo);
/*
* We can't checkpoint or swap sessions/cursors while in a
* transaction, resolve any running transaction. Otherwise,
* reset the cursor: we may block waiting for a lock and there
* is no reason to keep pages pinned.
*/
if (cnt == ckpt_op || cnt == session_op) {
if (intxn) {
if ((ret = session->commit_transaction(
session, NULL)) != 0)
die(ret, "session.commit_transaction");
++tinfo->commit;
intxn = 0;
}
else if (cursor != NULL &&
(ret = cursor->reset(cursor)) != 0)
die(ret, "cursor.reset");
}
/* Open up a new session and cursors. */
if (cnt == session_op || session == NULL || cursor == NULL) {
if (session != NULL &&
(ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
if ((ret = conn->open_session(
conn, NULL, NULL, &session)) != 0)
die(ret, "connection.open_session");
/*
* Open two cursors: one configured for overwriting and
* one configured for append if we're dealing with a
* column-store.
*
* The reason is when testing with existing records, we
* don't track if a record was deleted or not, which
* means we must use cursor->insert with overwriting
* configured. But, in column-store files where we're
* testing with new, appended records, we don't want to
* have to specify the record number, which requires an
* append configuration.
*/
if ((ret = session->open_cursor(session,
g.uri, NULL, "overwrite", &cursor)) != 0)
die(ret, "session.open_cursor");
if ((g.type == FIX || g.type == VAR) &&
(ret = session->open_cursor(session,
g.uri, NULL, "append", &cursor_insert)) != 0)
die(ret, "session.open_cursor");
/* Pick the next session/cursor close/open. */
session_op += SINGLETHREADED ?
MMRAND(1, thread_ops) : 100 * MMRAND(1, 50);
}
/* Checkpoint the database. */
if (cnt == ckpt_op) {
/*
* LSM and data-sources don't support named checkpoints,
* else 25% of the time we name the checkpoint.
*/
if (DATASOURCE("lsm") || DATASOURCE("kvsbdb") ||
DATASOURCE("memrata") || MMRAND(1, 4) == 1)
ckpt_config = NULL;
else {
(void)snprintf(config, sizeof(config),
"name=thread-%d", tinfo->id);
ckpt_config = config;
}
/* Named checkpoints lock out hot backups */
if (ckpt_config != NULL &&
(ret = pthread_rwlock_wrlock(&g.backup_lock)) != 0)
die(ret,
"pthread_rwlock_wrlock: hot-backup lock");
if ((ret =
session->checkpoint(session, ckpt_config)) != 0)
die(ret, "session.checkpoint%s%s",
ckpt_config == NULL ? "" : ": ",
ckpt_config == NULL ? "" : ckpt_config);
if (ckpt_config != NULL &&
(ret = pthread_rwlock_unlock(&g.backup_lock)) != 0)
die(ret,
"pthread_rwlock_wrlock: hot-backup lock");
/*
* Pick the next checkpoint operation, try for roughly
* five checkpoint operations per thread run.
*/
ckpt_op += MMRAND(1, thread_ops) / 5;
}
/*
* If we're not single-threaded and we're not in a transaction,
* start a transaction 80% of the time.
*/
if (!SINGLETHREADED && !intxn && MMRAND(1, 10) >= 8) {
if ((ret =
session->begin_transaction(session, NULL)) != 0)
die(ret, "session.begin_transaction");
intxn = 1;
}
insert = notfound = 0;
keyno = MMRAND(1, g.rows);
key.data = keybuf;
value.data = valbuf;
/*
* Perform some number of operations: the percentage of deletes,
* inserts and writes are specified, reads are the rest. The
* percentages don't have to add up to 100, a high percentage
* of deletes will mean fewer inserts and writes. Modifications
* are always followed by a read to confirm it worked.
*/
op = (uint32_t)(rng() % 100);
if (op < g.c_delete_pct) {
++tinfo->remove;
switch (g.type) {
case ROW:
/*
* If deleting a non-existent record, the cursor
* won't be positioned, and so can't do a next.
*/
if (row_remove(cursor, &key, keyno, ¬found))
goto deadlock;
break;
case FIX:
case VAR:
if (col_remove(cursor, &key, keyno, ¬found))
goto deadlock;
break;
}
} else if (op < g.c_delete_pct + g.c_insert_pct) {
++tinfo->insert;
switch (g.type) {
case ROW:
if (row_insert(cursor, &key, &value, keyno))
goto deadlock;
insert = 1;
break;
case FIX:
case VAR:
/*
* We can only append so many new records, if
* we've reached that limit, update a record
* instead of doing an insert.
*/
if (g.append_cnt >= g.append_max)
goto skip_insert;
/*
* Reset the standard cursor so it doesn't keep
* pages pinned.
*/
if ((ret = cursor->reset(cursor)) != 0)
die(ret, "cursor.reset");
/* Insert, then reset the insert cursor. */
if (col_insert(
cursor_insert, &key, &value, &keyno))
goto deadlock;
if ((ret =
cursor_insert->reset(cursor_insert)) != 0)
die(ret, "cursor.reset");
insert = 1;
break;
}
} else if (
op < g.c_delete_pct + g.c_insert_pct + g.c_write_pct) {
++tinfo->update;
switch (g.type) {
case ROW:
if (row_update(cursor, &key, &value, keyno))
goto deadlock;
break;
case FIX:
case VAR:
skip_insert: if (col_update(cursor, &key, &value, keyno))
goto deadlock;
break;
}
} else {
++tinfo->search;
if (read_row(cursor, &key, keyno))
goto deadlock;
continue;
}
/*
* The cursor is positioned if we did any operation other than
* insert, do a small number of next/prev cursor operations in
* a random direction.
*/
if (!insert) {
dir = (int)MMRAND(0, 1);
for (np = 0; np < MMRAND(1, 8); ++np) {
if (notfound)
break;
if (nextprev(cursor, dir, ¬found))
goto deadlock;
}
}
/* Read the value we modified to confirm the operation. */
++tinfo->search;
if (read_row(cursor, &key, keyno))
goto deadlock;
/*
* If we're in the transaction, commit 40% of the time and
* rollback 10% of the time.
*/
if (intxn)
switch (MMRAND(1, 10)) {
case 1: case 2: case 3: case 4: /* 40% */
if ((ret = session->commit_transaction(
session, NULL)) != 0)
die(ret, "session.commit_transaction");
++tinfo->commit;
intxn = 0;
break;
case 5: /* 10% */
if (0) {
deadlock: ++tinfo->deadlock;
}
if ((ret = session->rollback_transaction(
session, NULL)) != 0)
die(ret, "session.commit_transaction");
++tinfo->rollback;
intxn = 0;
break;
default:
break;
}
}
if (session != NULL && (ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
free(keybuf);
free(valbuf);
tinfo->state = TINFO_COMPLETE;
return (NULL);
}
void
wts_open(void)
{
WT_CONNECTION *conn;
WT_SESSION *session;
uint32_t maxintlpage, maxintlitem, maxleafpage, maxleafitem;
int ret;
char config[2048], *end, *p;
/*
* Open configuration.
*
* Put configuration file configuration options second to last. Put
* command line configuration options at the end. Do this so they
* override the standard configuration.
*/
snprintf(config, sizeof(config),
"create,cache_size=%" PRIu32 "MB,"
"error_prefix=\"%s\","
"extensions=[\"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],%s,%s,"
"statistics=true,statistics_log=(wait=5),"
"sync=false,",
g.c_cache,
g.progname,
REVERSE_PATH,
access(BZIP_PATH, R_OK) == 0 ? BZIP_PATH : "",
access(LZO_PATH, R_OK) == 0 ? LZO_PATH : "",
(access(RAW_PATH, R_OK) == 0 &&
access(BZIP_PATH, R_OK) == 0) ? RAW_PATH : "",
access(SNAPPY_PATH, R_OK) == 0 ? SNAPPY_PATH : "",
g.c_config_open == NULL ? "" : g.c_config_open,
g.config_open == NULL ? "" : g.config_open);
if ((ret =
wiredtiger_open("RUNDIR", &event_handler, config, &conn)) != 0)
die(ret, "wiredtiger_open");
g.wts_conn = conn;
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
die(ret, "connection.open_session");
/*
* Create the object.
*
* Make sure at least 2 internal page per thread can fit in cache.
*/
maxintlpage = 1U << g.c_intl_page_max;
while (2 * g.c_threads * maxintlpage > g.c_cache << 20)
maxintlpage >>= 1;
maxintlitem = MMRAND(maxintlpage / 50, maxintlpage / 40);
if (maxintlitem < 40)
maxintlitem = 40;
/* Make sure at least one leaf page per thread can fit in cache. */
maxleafpage = 1U << g.c_leaf_page_max;
while (g.c_threads * (maxintlpage + maxleafpage) > g.c_cache << 20)
maxleafpage >>= 1;
maxleafitem = MMRAND(maxleafpage / 50, maxleafpage / 40);
if (maxleafitem < 40)
maxleafitem = 40;
p = config;
end = config + sizeof(config);
p += snprintf(p, (size_t)(end - p),
"key_format=%s,"
"internal_page_max=%d,internal_item_max=%d,"
"leaf_page_max=%d,leaf_item_max=%d",
(g.type == ROW) ? "u" : "r",
maxintlpage, maxintlitem, maxleafpage, maxleafitem);
switch (g.type) {
case FIX:
p += snprintf(p, (size_t)(end - p),
",value_format=%dt", g.c_bitcnt);
break;
case ROW:
if (g.c_huffman_key)
p += snprintf(p, (size_t)(end - p),
",huffman_key=english");
if (!g.c_prefix)
p += snprintf(p, (size_t)(end - p),
",prefix_compression=false");
if (g.c_reverse)
p += snprintf(p, (size_t)(end - p),
",collator=reverse");
/* FALLTHROUGH */
case VAR:
if (g.c_huffman_value)
p += snprintf(p, (size_t)(end - p),
",huffman_value=english");
if (g.c_dictionary)
p += snprintf(p, (size_t)(end - p),
",dictionary=%d", MMRAND(123, 517));
break;
}
/* Configure checksums (not configurable from the command line). */
switch MMRAND(1, 10) {
case 1: /* 10% */
p += snprintf(p, (size_t)(end - p), ",checksum=\"on\"");
break;
case 2: /* 10% */
p += snprintf(p, (size_t)(end - p), ",checksum=\"off\"");
break;
default: /* 80% */
p += snprintf(
p, (size_t)(end - p), ",checksum=\"uncompressed\"");
break;
}
/* Configure compression. */
switch (g.compression) {
case COMPRESS_NONE:
break;
case COMPRESS_BZIP:
p += snprintf(p, (size_t)(end - p),
",block_compressor=\"bzip2\"");
break;
case COMPRESS_LZO:
p += snprintf(p, (size_t)(end - p),
",block_compressor=\"LZO1B-6\"");
break;
case COMPRESS_RAW:
p += snprintf(p, (size_t)(end - p),
",block_compressor=\"raw\"");
break;
case COMPRESS_SNAPPY:
p += snprintf(p, (size_t)(end - p),
",block_compressor=\"snappy\"");
break;
}
/* Configure internal key truncation. */
p += snprintf(
p, (size_t)(end - p), ",internal_key_truncate=%s",
g.c_internal_key_truncation ? "true" : "false");
/* Configure Btree page key gap. */
p += snprintf(p, (size_t)(end - p), ",key_gap=%u", g.c_key_gap);
/* Configure Btree split page percentage. */
p += snprintf(p, (size_t)(end - p), ",split_pct=%u", g.c_split_pct);
if ((ret = session->create(session, g.uri, config)) != 0)
die(ret, "session.create: %s", g.uri);
if ((ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
}
/*
* config_setup --
* Initialize configuration for a run.
*/
void
config_setup(void)
{
CONFIG *cp;
/* Clear any temporary values. */
config_clear();
/*
* Choose a data source type and a file type: they're interrelated (LSM
* trees are only compatible with row-store) and other items depend on
* them.
*/
if (!config_find_is_perm("data_source", strlen("data_source")))
switch (MMRAND(1, 3)) {
case 1:
config_single("data_source=file", 0);
break;
case 2:
config_single("data_source=lsm", 0);
break;
case 3:
config_single("data_source=table", 0);
break;
}
if (!config_find_is_perm("file_type", strlen("file_type")))
switch (DATASOURCE("lsm") ? 3 : MMRAND(1, 3)) {
case 1:
config_single("file_type=fix", 0);
break;
case 2:
config_single("file_type=var", 0);
break;
case 3:
config_single("file_type=row", 0);
break;
}
config_map_file_type(g.c_file_type, &g.type);
/*
* If data_source and file_type were both "permanent", we may still
* have a mismatch.
*/
if (DATASOURCE("lsm") && g.type != ROW) {
fprintf(stderr,
"%s: lsm data_source is only compatible with row file_type\n",
g.progname);
exit(EXIT_FAILURE);
}
/*
* Build the top-level object name: we're overloading data_source in
* our configuration, LSM or KVS devices are "tables", but files are
* tested as well.
*/
if ((g.uri = malloc(256)) == NULL)
syserr("malloc");
strcpy(g.uri, DATASOURCE("file") ? "file:" : "table:");
if (DATASOURCE("helium"))
strcat(g.uri, "dev1/");
strcat(g.uri, WT_NAME);
/* Default single-threaded 10% of the time. */
cp = config_find("threads", strlen("threads"));
if (!(cp->flags & C_PERM))
*cp->v = MMRAND(1, 100) < 10 ? 1: CONF_RAND(cp);
/* Fill in random values for the rest of the run. */
for (cp = c; cp->name != NULL; ++cp) {
if (cp->flags & (C_IGNORE | C_PERM | C_TEMP))
continue;
/*
* Boolean flags are 0 or 1, but only set N in 100 where the
* variable's min value is N. Set the flag if we rolled >=
* the min, 0 otherwise.
*/
if (cp->flags & C_BOOL)
*cp->v = MMRAND(1, 100) <= cp->min ? 1 : 0;
else
*cp->v = CONF_RAND(cp);
}
/* Required shared libraries. */
if (DATASOURCE("helium") && access(HELIUM_PATH, R_OK) != 0)
die(errno, "Levyx/helium shared library: %s", HELIUM_PATH);
if (DATASOURCE("kvsbdb") && access(KVS_BDB_PATH, R_OK) != 0)
die(errno, "kvsbdb shared library: %s", KVS_BDB_PATH);
/* Some data-sources don't support user-specified collations. */
if (DATASOURCE("helium") || DATASOURCE("kvsbdb"))
g.c_reverse = 0;
config_checksum();
config_compression();
/* Clear operations values if the whole run is read-only. */
if (g.c_ops == 0)
for (cp = c; cp->name != NULL; ++cp)
if (cp->flags & C_OPS)
*cp->v = 0;
/*
* Periodically, set the delete percentage to 0 so salvage gets run,
* as long as the delete percentage isn't nailed down.
*/
if (!g.replay && g.run_cnt % 10 == 0) {
cp = config_find("delete_pct", strlen("delete_pct"));
if (cp->name != NULL &&
!(cp->flags & (C_IGNORE | C_PERM | C_TEMP)))
g.c_delete_pct = 0;
}
/*
* If this is an LSM run, set the cache size and crank up the insert
* percentage.
*/
if (DATASOURCE("lsm")) {
cp = config_find("cache", strlen("cache"));
if (!(cp->flags & C_PERM))
g.c_cache = 30 * g.c_chunk_size;
cp = config_find("insert_pct", strlen("insert_pct"));
if (cp->name != NULL &&
!(cp->flags & (C_IGNORE | C_PERM | C_TEMP)))
g.c_insert_pct = MMRAND(50, 85);
}
/*
* Key/value minimum/maximum are related, correct unless specified by
* the configuration.
*/
cp = config_find("key_min", strlen("key_min"));
if (!(cp->flags & C_PERM) && g.c_key_min > g.c_key_max)
g.c_key_min = g.c_key_max;
cp = config_find("key_max", strlen("key_max"));
if (!(cp->flags & C_PERM) && g.c_key_max < g.c_key_min)
g.c_key_max = g.c_key_min;
if (g.c_key_min > g.c_key_max)
die(EINVAL, "key_min may not be larger than key_max");
cp = config_find("value_min", strlen("value_min"));
if (!(cp->flags & C_PERM) && g.c_value_min > g.c_value_max)
g.c_value_min = g.c_value_max;
cp = config_find("value_max", strlen("value_max"));
if (!(cp->flags & C_PERM) && g.c_value_max < g.c_value_min)
g.c_value_max = g.c_value_min;
if (g.c_value_min > g.c_value_max)
die(EINVAL, "value_min may not be larger than value_max");
/* Reset the key count. */
g.key_cnt = 0;
}
