/*
* config_checksum --
* Checksum configuration.
*/
static void
config_checksum(void)
{
CONFIG *cp;
/* Choose a checksum mode if nothing was specified. */
cp = config_find("checksum", strlen("checksum"));
if (!(cp->flags & C_PERM))
switch (MMRAND(1, 10)) {
case 1: /* 10% */
config_single("checksum=on", 0);
break;
case 2: /* 10% */
config_single("checksum=off", 0);
break;
default: /* 80% */
config_single("checksum=uncompressed", 0);
break;
}
}
/*
* config_compression --
* Compression configuration.
*/
static void
config_compression(void)
{
CONFIG *cp;
const char *cstr;
/*
* Compression: choose something if compression wasn't specified,
* otherwise confirm the appropriate shared library is available.
* We don't include LZO in the test compression choices, we don't
* yet have an LZO module of our own.
*/
cp = config_find("compression", strlen("compression"));
if (!(cp->flags & C_PERM)) {
cstr = "compression=none";
switch (MMRAND(0, 9)) {
case 0: /* 10% */
break;
case 1: case 2: case 3: case 4: /* 40% */
if (access(BZIP_PATH, R_OK) == 0)
cstr = "compression=bzip";
break;
case 5: /* 10% */
if (access(BZIP_PATH, R_OK) == 0)
cstr = "compression=raw";
break;
case 6: case 7: case 8: case 9: /* 40% */
if (access(SNAPPY_PATH, R_OK) == 0)
cstr = "compression=snappy";
break;
}
config_single(cstr, 0);
}
g.compression = config_translate(g.c_compression);
if (!(cp->flags & C_PERM))
return;
switch (g.compression) {
case COMPRESS_BZIP:
case COMPRESS_RAW:
if (access(BZIP_PATH, R_OK) != 0)
die(0, "bzip library not found or not readable");
break;
case COMPRESS_LZO:
if (access(LZO_PATH, R_OK) != 0)
die(0, "LZO library not found or not readable");
break;
case COMPRESS_SNAPPY:
if (access(SNAPPY_PATH, R_OK) != 0)
die(0, "snappy library not found or not readable");
}
}
/*
* config_file --
* Read configuration values from a file.
*/
void
config_file(const char *name)
{
FILE *fp;
char *p, buf[256];
if ((fp = fopen(name, "r")) == NULL)
die(errno, "fopen: %s", name);
while (fgets(buf, sizeof(buf), fp) != NULL) {
for (p = buf; *p != '\0' && *p != '\n'; ++p)
;
*p = '\0';
if (buf[0] == '\0' || buf[0] == '#')
continue;
config_single(buf, 1);
}
(void)fclose(fp);
}
int
main(int argc, char *argv[])
{
time_t start;
int ch, onerun, reps;
const char *config, *home;
config = NULL;
#ifdef _WIN32
g.progname = "t_format.exe";
#else
if ((g.progname = strrchr(argv[0], DIR_DELIM)) == NULL)
g.progname = argv[0];
else
++g.progname;
#endif
#if 0
/* Configure the GNU malloc for debugging. */
(void)setenv("MALLOC_CHECK_", "2", 1);
#endif
#if 0
/* Configure the FreeBSD malloc for debugging. */
(void)setenv("MALLOC_OPTIONS", "AJ", 1);
#endif
/* Track progress unless we're re-directing output to a file. */
g.c_quiet = isatty(1) ? 0 : 1;
/* Set values from the command line. */
home = NULL;
onerun = 0;
while ((ch = __wt_getopt(
g.progname, argc, argv, "1C:c:H:h:Llqrt:")) != EOF)
switch (ch) {
case '1': /* One run */
onerun = 1;
break;
case 'C': /* wiredtiger_open config */
g.config_open = __wt_optarg;
break;
case 'c': /* Configuration from a file */
config = __wt_optarg;
break;
case 'H':
g.helium_mount = __wt_optarg;
break;
case 'h':
home = __wt_optarg;
break;
case 'L': /* Re-direct output to a log */
/*
* The -l option is a superset of -L, ignore -L if we
* have already configured logging for operations.
*/
if (g.logging == 0)
g.logging = LOG_FILE;
break;
case 'l': /* Turn on operation logging */
g.logging = LOG_OPS;
break;
case 'q': /* Quiet */
g.c_quiet = 1;
break;
case 'r': /* Replay a run */
g.replay = 1;
break;
default:
usage();
}
argc -= __wt_optind;
argv += __wt_optind;
/* Initialize the global RNG. */
testutil_check(__wt_random_init_seed(NULL, &g.rnd));
/* Set up paths. */
path_setup(home);
/* If it's a replay, use the home directory's CONFIG file. */
if (g.replay) {
if (config != NULL)
testutil_die(EINVAL, "-c incompatible with -r");
if (access(g.home_config, R_OK) != 0)
testutil_die(ENOENT, "%s", g.home_config);
config = g.home_config;
}
/*
* If we weren't given a configuration file, set values from "CONFIG",
* if it exists.
*
* Small hack to ignore any CONFIG file named ".", that just makes it
* possible to ignore any local CONFIG file, used when running checks.
*/
if (config == NULL && access("CONFIG", R_OK) == 0)
config = "CONFIG";
if (config != NULL && strcmp(config, ".") != 0)
config_file(config);
/*
* The rest of the arguments are individual configurations that modify
* the base configuration.
*/
for (; *argv != NULL; ++argv)
config_single(*argv, 1);
/*
* Multithreaded runs can be replayed: it's useful and we'll get the
* configuration correct. Obviously the order of operations changes,
* warn the user.
*/
if (g.replay && !SINGLETHREADED)
printf("Warning: replaying a threaded run\n");
/*
* Single-threaded runs historically exited after a single replay, which
* makes sense when you're debugging, leave that semantic in place.
*/
if (g.replay && SINGLETHREADED)
g.c_runs = 1;
/*
* Let the command line -1 flag override runs configured from other
* sources.
*/
if (onerun)
g.c_runs = 1;
/*
* Initialize locks to single-thread named checkpoints and backups, last
* last-record updates, and failures.
*/
testutil_check(pthread_rwlock_init(&g.append_lock, NULL));
testutil_check(pthread_rwlock_init(&g.backup_lock, NULL));
testutil_check(pthread_rwlock_init(&g.checkpoint_lock, NULL));
testutil_check(pthread_rwlock_init(&g.death_lock, NULL));
printf("%s: process %" PRIdMAX "\n", g.progname, (intmax_t)getpid());
while (++g.run_cnt <= g.c_runs || g.c_runs == 0 ) {
startup(); /* Start a run */
config_setup(); /* Run configuration */
config_print(0); /* Dump run configuration */
key_len_setup(); /* Setup keys */
start = time(NULL);
track("starting up", 0ULL, NULL);
#ifdef HAVE_BERKELEY_DB
if (SINGLETHREADED)
bdb_open(); /* Initial file config */
#endif
wts_open(g.home, true, &g.wts_conn);
wts_init();
wts_load(); /* Load initial records */
wts_verify("post-bulk verify"); /* Verify */
/*
* If we're not doing any operations, scan the bulk-load, copy
* the statistics and we're done. Otherwise, loop reading and
* operations, with a verify after each set.
*/
if (g.c_timer == 0 && g.c_ops == 0) {
wts_read_scan(); /* Read scan */
wts_stats(); /* Statistics */
} else
for (reps = 1; reps <= FORMAT_OPERATION_REPS; ++reps) {
wts_read_scan(); /* Read scan */
/* Operations */
wts_ops(reps == FORMAT_OPERATION_REPS);
/*
* Copy out the run's statistics after the last
* set of operations.
*
* XXX
* Verify closes the underlying handle and
* discards the statistics, read them first.
*/
if (reps == FORMAT_OPERATION_REPS)
wts_stats();
/* Verify */
wts_verify("post-ops verify");
}
track("shutting down", 0ULL, NULL);
#ifdef HAVE_BERKELEY_DB
if (SINGLETHREADED)
bdb_close();
#endif
wts_close();
/*
* Rebalance testing.
*/
wts_rebalance();
/*
* If single-threaded, we can dump and compare the WiredTiger
* and Berkeley DB data sets.
*/
if (SINGLETHREADED)
wts_dump("standard", 1);
/*
* Salvage testing.
*/
wts_salvage();
/* Overwrite the progress line with a completion line. */
if (!g.c_quiet)
printf("\r%78s\r", " ");
printf("%4d: %s, %s (%.0f seconds)\n",
g.run_cnt, g.c_data_source,
g.c_file_type, difftime(time(NULL), start));
fflush(stdout);
}
/* Flush/close any logging information. */
fclose_and_clear(&g.logfp);
fclose_and_clear(&g.randfp);
config_print(0);
testutil_check(pthread_rwlock_destroy(&g.append_lock));
testutil_check(pthread_rwlock_destroy(&g.backup_lock));
testutil_check(pthread_rwlock_destroy(&g.checkpoint_lock));
testutil_check(pthread_rwlock_destroy(&g.death_lock));
config_clear();
return (EXIT_SUCCESS);
}
int
main(int argc, char *argv[])
{
int ch, reps, ret;
const char *config, *home;
config = NULL;
if ((g.progname = strrchr(argv[0], '/')) == NULL)
g.progname = argv[0];
else
++g.progname;
#if 0
/* Configure the GNU malloc for debugging. */
(void)setenv("MALLOC_CHECK_", "2", 1);
#endif
#if 0
/* Configure the FreeBSD malloc for debugging. */
(void)setenv("MALLOC_OPTIONS", "AJ", 1);
#endif
/* Track progress unless we're re-directing output to a file. */
g.track = isatty(STDOUT_FILENO) ? 1 : 0;
/* Set values from the command line. */
home = NULL;
while ((ch = getopt(argc, argv, "1C:c:h:Llqrt:")) != EOF)
switch (ch) {
case '1': /* One run */
g.c_runs = 1;
break;
case 'C': /* wiredtiger_open config */
g.config_open = optarg;
break;
case 'c': /* Configuration from a file */
config = optarg;
break;
case 'h':
home = optarg;
break;
case 'L': /* Re-direct output to a log */
/*
* The -l option is a superset of -L, ignore -L if we
* have already configured logging for operations.
*/
if (g.logging == 0)
g.logging = LOG_FILE;
break;
case 'l': /* Turn on operation logging */
g.logging = LOG_OPS;
break;
case 'q': /* Quiet */
g.track = 0;
break;
case 'r': /* Replay a run */
g.replay = 1;
break;
default:
usage();
}
argc -= optind;
argv += optind;
/*
* If we weren't given a configuration file, set values from "CONFIG",
* if it exists.
*
* Small hack to ignore any CONFIG file named ".", that just makes it
* possible to ignore any local CONFIG file, used when running checks.
*/
if (config == NULL && access("CONFIG", R_OK) == 0)
config = "CONFIG";
if (config != NULL && strcmp(config, ".") != 0)
config_file(config);
/*
* The rest of the arguments are individual configurations that modify
* the base configuration.
*/
for (; *argv != NULL; ++argv)
config_single(*argv, 1);
/*
* Multithreaded runs can be replayed: it's useful and we'll get the
* configuration correct. Obviously the order of operations changes,
* warn the user.
*/
if (g.replay && !SINGLETHREADED)
printf("Warning: replaying a threaded run\n");
/*
* Single-threaded runs historically exited after a single replay, which
* makes sense when you're debugging, leave that semantic in place.
*/
if (g.replay && SINGLETHREADED)
g.c_runs = 1;
/* Use line buffering on stdout so status updates aren't buffered. */
(void)setvbuf(stdout, NULL, _IOLBF, 0);
/*
* Initialize locks to single-thread named checkpoints and hot backups
* and to single-thread last-record updates.
*/
if ((ret = pthread_rwlock_init(&g.append_lock, NULL)) != 0)
die(ret, "pthread_rwlock_init: append lock");
if ((ret = pthread_rwlock_init(&g.backup_lock, NULL)) != 0)
die(ret, "pthread_rwlock_init: hot-backup lock");
/* Clean up on signal. */
(void)signal(SIGINT, onint);
/* Seed the random number generator. */
srand((u_int)(0xdeadbeef ^ (u_int)time(NULL)));
/* Set up paths. */
path_setup(home);
printf("%s: process %" PRIdMAX "\n", g.progname, (intmax_t)getpid());
while (++g.run_cnt <= g.c_runs || g.c_runs == 0 ) {
startup(); /* Start a run */
config_setup(); /* Run configuration */
config_print(0); /* Dump run configuration */
key_len_setup(); /* Setup keys */
track("starting up", 0ULL, NULL);
if (SINGLETHREADED)
bdb_open(); /* Initial file config */
wts_open(g.home, 1, &g.wts_conn);
wts_create();
wts_load(); /* Load initial records */
wts_verify("post-bulk verify"); /* Verify */
/* Loop reading & operations */
for (reps = 0; reps < 3; ++reps) {
wts_read_scan(); /* Read scan */
if (g.c_ops != 0) /* Random operations */
wts_ops();
/*
* Statistics.
*
* XXX
* Verify closes the underlying handle and discards the
* statistics, read them first.
*/
if (g.c_ops == 0 || reps == 2)
wts_stats();
/* Verify */
wts_verify("post-ops verify");
/*
* If no operations scheduled, quit after a single
* read pass.
*/
if (g.c_ops == 0)
break;
}
track("shutting down", 0ULL, NULL);
if (SINGLETHREADED)
bdb_close();
wts_close();
/*
* If single-threaded, we can dump and compare the WiredTiger
* and Berkeley DB data sets.
*/
if (SINGLETHREADED)
wts_dump("standard", 1);
/*
* If no records are deleted, we can salvage the file and test
* the result. (The problem with deleting records is salvage
* restores deleted records if a page splits leaving a deleted
* record on one side of the split.)
*
* Salvage, verify the salvaged files, then dump (comparing
* against the Berkeley DB data set again, if possible).
*/
if (g.c_delete_pct == 0) {
wts_open(g.home, 1, &g.wts_conn);
wts_salvage();
wts_verify("post-salvage verify");
wts_close();
wts_dump("salvage", SINGLETHREADED);
}
/* Overwrite the progress line with a completion line. */
if (g.track)
printf("\r%78s\r", " ");
printf("%4d: %s, %s\n",
g.run_cnt, g.c_data_source, g.c_file_type);
}
/* Flush/close any logging information. */
if (g.logfp != NULL)
(void)fclose(g.logfp);
if (g.rand_log != NULL)
(void)fclose(g.rand_log);
config_print(0);
if ((ret = pthread_rwlock_destroy(&g.append_lock)) != 0)
die(ret, "pthread_rwlock_destroy: append lock");
if ((ret = pthread_rwlock_destroy(&g.backup_lock)) != 0)
die(ret, "pthread_rwlock_destroy: hot-backup lock");
config_clear();
return (EXIT_SUCCESS);
}
/*
* config_setup --
* Initialize configuration for a run.
*/
void
config_setup(void)
{
CONFIG *cp;
/* Clear any temporary values. */
config_clear();
/*
* Choose a data source type and a file type: they're interrelated (LSM
* trees are only compatible with row-store) and other items depend on
* them.
*/
if (!config_find_is_perm("data_source", strlen("data_source")))
switch (MMRAND(0, 2)) {
case 0:
config_single("data_source=file", 0);
break;
case 1:
#if 0
config_single("data_source=lsm", 0);
break;
#endif
case 2:
config_single("data_source=table", 0);
break;
}
if (!config_find_is_perm("file_type", strlen("file_type"))) {
if (strcmp(g.c_data_source, "lsm") == 0)
config_single("file_type=row", 0);
else
switch (MMRAND(0, 2)) {
case 0:
config_single("file_type=fix", 0);
break;
case 1:
config_single("file_type=var", 0);
break;
case 2:
config_single("file_type=row", 0);
break;
}
}
g.type = config_translate(g.c_file_type);
/*
* If data_source and file_type were both "permanent", we may still
* have a mismatch.
*/
if (g.type != ROW && strcmp(g.c_data_source, "lsm") == 0) {
fprintf(stderr,
"%s: lsm data_source is only compatible with row file_type\n",
g.progname);
exit(EXIT_FAILURE);
}
/* Build the object name. */
if ((g.uri = malloc(
strlen(g.c_data_source) + strlen(WT_NAME) + 2)) == NULL)
syserr("malloc");
strcpy(g.uri, g.c_data_source);
strcat(g.uri, ":");
strcat(g.uri, WT_NAME);
/* Default single-threaded half of the time. */
cp = config_find("threads", strlen("threads"));
if (!(cp->flags & C_PERM))
*cp->v = MMRAND(0, 1) ? 1: CONF_RAND(cp);
/* Fill in random values for the rest of the run. */
for (cp = c; cp->name != NULL; ++cp) {
if (cp->flags & (C_IGNORE | C_PERM | C_TEMP))
continue;
/*
* Boolean flags are 0 or 1, but only set N in 100 where the
* variable's min value is N. Set the flag if we rolled >=
* the min, 0 otherwise.
*/
if (cp->flags & C_BOOL)
*cp->v = MMRAND(1, 100) <= cp->min ? 1 : 0;
else
*cp->v = CONF_RAND(cp);
}
config_compression();
/* Clear operations values if the whole run is read-only. */
if (g.c_ops == 0)
for (cp = c; cp->name != NULL; ++cp)
if (cp->flags & C_OPS)
*cp->v = 0;
/* Multi-threaded runs cannot be replayed. */
if (g.replay && !SINGLETHREADED)
die(0, "-r is incompatible with threaded runs");
/*
* Periodically, set the delete percentage to 0 so salvage gets run,
* as long as the delete percentage isn't nailed down.
*/
if (!g.replay && g.run_cnt % 10 == 0) {
cp = config_find("delete_pct", strlen("delete_pct"));
if (cp->name != NULL &&
!(cp->flags & (C_IGNORE | C_PERM | C_TEMP)))
g.c_delete_pct = 0;
}
/* Reset the key count. */
g.key_cnt = 0;
}
/*
* config_setup --
* Initialize configuration for a run.
*/
void
config_setup(void)
{
CONFIG *cp;
/* Clear any temporary values. */
config_clear();
/*
* Choose a data source type and a file type: they're interrelated (LSM
* trees are only compatible with row-store) and other items depend on
* them.
*/
if (!config_find_is_perm("data_source", strlen("data_source")))
switch (MMRAND(1, 3)) {
case 1:
config_single("data_source=file", 0);
break;
case 2:
config_single("data_source=lsm", 0);
break;
case 3:
config_single("data_source=table", 0);
break;
}
if (!config_find_is_perm("file_type", strlen("file_type")))
switch (DATASOURCE("lsm") ? 3 : MMRAND(1, 3)) {
case 1:
config_single("file_type=fix", 0);
break;
case 2:
config_single("file_type=var", 0);
break;
case 3:
config_single("file_type=row", 0);
break;
}
config_map_file_type(g.c_file_type, &g.type);
/*
* If data_source and file_type were both "permanent", we may still
* have a mismatch.
*/
if (DATASOURCE("lsm") && g.type != ROW) {
fprintf(stderr,
"%s: lsm data_source is only compatible with row file_type\n",
g.progname);
exit(EXIT_FAILURE);
}
/*
* Build the top-level object name: we're overloading data_source in
* our configuration, LSM or KVS devices are "tables", but files are
* tested as well.
*/
if ((g.uri = malloc(256)) == NULL)
syserr("malloc");
strcpy(g.uri, DATASOURCE("file") ? "file:" : "table:");
if (DATASOURCE("helium"))
strcat(g.uri, "dev1/");
strcat(g.uri, WT_NAME);
/* Default single-threaded 10% of the time. */
cp = config_find("threads", strlen("threads"));
if (!(cp->flags & C_PERM))
*cp->v = MMRAND(1, 100) < 10 ? 1: CONF_RAND(cp);
/* Fill in random values for the rest of the run. */
for (cp = c; cp->name != NULL; ++cp) {
if (cp->flags & (C_IGNORE | C_PERM | C_TEMP))
continue;
/*
* Boolean flags are 0 or 1, but only set N in 100 where the
* variable's min value is N. Set the flag if we rolled >=
* the min, 0 otherwise.
*/
if (cp->flags & C_BOOL)
*cp->v = MMRAND(1, 100) <= cp->min ? 1 : 0;
else
*cp->v = CONF_RAND(cp);
}
/* Required shared libraries. */
if (DATASOURCE("helium") && access(HELIUM_PATH, R_OK) != 0)
die(errno, "Levyx/helium shared library: %s", HELIUM_PATH);
if (DATASOURCE("kvsbdb") && access(KVS_BDB_PATH, R_OK) != 0)
die(errno, "kvsbdb shared library: %s", KVS_BDB_PATH);
/* Some data-sources don't support user-specified collations. */
if (DATASOURCE("helium") || DATASOURCE("kvsbdb"))
g.c_reverse = 0;
config_checksum();
config_compression();
/* Clear operations values if the whole run is read-only. */
if (g.c_ops == 0)
for (cp = c; cp->name != NULL; ++cp)
if (cp->flags & C_OPS)
*cp->v = 0;
/*
* Periodically, set the delete percentage to 0 so salvage gets run,
* as long as the delete percentage isn't nailed down.
*/
if (!g.replay && g.run_cnt % 10 == 0) {
cp = config_find("delete_pct", strlen("delete_pct"));
if (cp->name != NULL &&
!(cp->flags & (C_IGNORE | C_PERM | C_TEMP)))
g.c_delete_pct = 0;
}
/*
* If this is an LSM run, set the cache size and crank up the insert
* percentage.
*/
if (DATASOURCE("lsm")) {
cp = config_find("cache", strlen("cache"));
if (!(cp->flags & C_PERM))
g.c_cache = 30 * g.c_chunk_size;
cp = config_find("insert_pct", strlen("insert_pct"));
if (cp->name != NULL &&
!(cp->flags & (C_IGNORE | C_PERM | C_TEMP)))
g.c_insert_pct = MMRAND(50, 85);
}
/*
* Key/value minimum/maximum are related, correct unless specified by
* the configuration.
*/
cp = config_find("key_min", strlen("key_min"));
if (!(cp->flags & C_PERM) && g.c_key_min > g.c_key_max)
g.c_key_min = g.c_key_max;
cp = config_find("key_max", strlen("key_max"));
if (!(cp->flags & C_PERM) && g.c_key_max < g.c_key_min)
g.c_key_max = g.c_key_min;
if (g.c_key_min > g.c_key_max)
die(EINVAL, "key_min may not be larger than key_max");
cp = config_find("value_min", strlen("value_min"));
if (!(cp->flags & C_PERM) && g.c_value_min > g.c_value_max)
g.c_value_min = g.c_value_max;
cp = config_find("value_max", strlen("value_max"));
if (!(cp->flags & C_PERM) && g.c_value_max < g.c_value_min)
g.c_value_max = g.c_value_min;
if (g.c_value_min > g.c_value_max)
die(EINVAL, "value_min may not be larger than value_max");
/* Reset the key count. */
g.key_cnt = 0;
}
