Exemple #1
0
/*
 * fop --
 *	File operation function.
 */
static void *
fop(void *arg)
{
	STATS *s;
	uintptr_t id;
	WT_RAND_STATE rnd;
	u_int i;

	id = (uintptr_t)arg;
	__wt_yield();		/* Get all the threads created. */

	s = &run_stats[id];
	__wt_random_init(&rnd);

	for (i = 0; i < nops; ++i, __wt_yield())
		switch (__wt_random(&rnd) % 10) {
		case 0:
			++s->bulk;
			obj_bulk();
			break;
		case 1:
			++s->create;
			obj_create();
			break;
		case 2:
			++s->cursor;
			obj_cursor();
			break;
		case 3:
			++s->drop;
			obj_drop(__wt_random(&rnd) & 1);
			break;
		case 4:
			++s->ckpt;
			obj_checkpoint();
			break;
		case 5:
			++s->upgrade;
			obj_upgrade();
			break;
		case 6:
			++s->rebalance;
			obj_rebalance();
			break;
		case 7:
			++s->verify;
			obj_verify();
			break;
		case 8:
			++s->bulk_unique;
			obj_bulk_unique(__wt_random(&rnd) & 1);
			break;
		case 9:
			++s->create_unique;
			obj_create_unique(__wt_random(&rnd) & 1);
			break;
		}

	return (NULL);
}
Exemple #2
0
/*
 * modify_build --
 *	Generate a set of modify vectors.
 */
static void
modify_build(void)
{
	int i;

	/* Mess up the entries. */
	memset(entries, 0xff, MAX_MODIFY_ENTRIES * sizeof(entries[0]));

	/*
	 * Randomly select a number of byte changes, offsets and lengths.
	 * Allow a value of 0, the API should accept it.
	 */
	nentries = (int)(__wt_random(&rnd) % MAX_MODIFY_ENTRIES);
	for (i = 0; i < nentries; ++i) {
		entries[i].data.data =
		    modify_repl + __wt_random(&rnd) % MAX_REPL_BYTES;
		entries[i].data.size =
		    (size_t)(__wt_random(&rnd) % MAX_REPL_BYTES);
		entries[i].offset = (size_t)(__wt_random(&rnd) % DATASIZE);
		entries[i].size = (size_t)(__wt_random(&rnd) % MAX_REPL_BYTES);
	}
#if DEBUG
	for (i = 0; i < nentries; ++i)
		printf(
		    "%d: {%.*s} %" WT_SIZET_FMT " bytes replacing %"
		    WT_SIZET_FMT " bytes @ %" WT_SIZET_FMT "\n",
		    i, (int)entries[i].data.size, entries[i].data.data,
		    entries[i].data.size, entries[i].size, entries[i].offset);
#endif
}
Exemple #3
0
static void
op(WT_SESSION *session, WT_RAND_STATE *rnd, WT_CURSOR **cpp)
{
	WT_CURSOR *cursor;
	WT_DECL_RET;
	u_int i, key;
	char buf[128];
	bool readonly;

	/* Close any open cursor in the slot we're about to reuse. */
	if (*cpp != NULL) {
		testutil_check((*cpp)->close(*cpp));
		*cpp = NULL;
	}

	cursor = NULL;
	readonly = __wt_random(rnd) % 2 == 0;

	/* Loop to open an object handle. */
	for (i = __wt_random(rnd) % uris; !done; __wt_yield()) {
		/* Use a checkpoint handle for 50% of reads. */
		ret = session->open_cursor(session, uri_list[i], NULL,
		    readonly && (i % 2 == 0) ?
		    "checkpoint=WiredTigerCheckpoint" : NULL, &cursor);
		if (ret != EBUSY) {
			testutil_check(ret);
			break;
		}
		(void)__wt_atomic_add64(&worker_busy, 1);
	}
	if (cursor == NULL)
		return;

	/* Operate on some number of key/value pairs. */
	for (key = 1;
	    !done && key < MAXKEY; key += __wt_random(rnd) % 37, __wt_yield()) {
		testutil_check(
		    __wt_snprintf(buf, sizeof(buf), "key:%020u", key));
		cursor->set_key(cursor, buf);
		if (readonly)
			testutil_check(cursor->search(cursor));
		else {
			cursor->set_value(cursor, buf);
			testutil_check(cursor->insert(cursor));
		}
	}

	/* Close the cursor half the time, otherwise cache it. */
	if (__wt_random(rnd) % 2 == 0)
		testutil_check(cursor->close(cursor));
	else
		*cpp = cursor;

	(void)__wt_atomic_add64(&worker, 1);
}
Exemple #4
0
/*
 * rng --
 *	Return a random number.
 */
uint32_t
rng(WT_RAND_STATE *rnd)
{
	char buf[64];
	uint32_t r;

	/*
	 * Threaded operations have their own RNG information, otherwise we
	 * use the default.
	 */
	if (rnd == NULL)
		rnd = &g.rnd;

	/*
	 * We can reproduce a single-threaded run based on the random numbers
	 * used in the initial run, plus the configuration files.
	 *
	 * Check g.replay and g.rand_log_stop: multithreaded runs log/replay
	 * until they get to the operations phase, then turn off log/replay,
	 * threaded operation order can't be replayed.
	 */
	if (g.rand_log_stop)
		return (__wt_random(rnd));

	if (g.replay) {
		if (fgets(buf, sizeof(buf), g.randfp) == NULL) {
			if (feof(g.randfp)) {
				fprintf(stderr,
				    "\n" "end of random number log reached\n");
				exit(EXIT_SUCCESS);
			}
			testutil_die(errno, "random number log");
		}

		return ((uint32_t)strtoul(buf, NULL, 10));
	}

	r = __wt_random(rnd);

	/* Save and flush the random number so we're up-to-date on error. */
	(void)fprintf(g.randfp, "%" PRIu32 "\n", r);
	(void)fflush(g.randfp);

	return (r);
}
Exemple #5
0
/*
 * Create a guaranteed unique table and open and close a bulk cursor on it.
 */
void
op_bulk_unique(void *arg)
{
	TEST_OPTS *opts;
	TEST_PER_THREAD_OPTS *args;
	WT_CURSOR *c;
	WT_RAND_STATE rnd;
	WT_SESSION *session;
	int ret;
	char new_uri[64];

	args = (TEST_PER_THREAD_OPTS *)arg;
	opts = args->testopts;
	__wt_random_init_seed(NULL, &rnd);

	testutil_check(
	    opts->conn->open_session(opts->conn, NULL, NULL, &session));

	/* Generate a unique object name. */
	testutil_check(__wt_snprintf(
	    new_uri, sizeof(new_uri), "%s.%" PRIu64,
	    opts->uri, __wt_atomic_add64(&opts->unique_id, 1)));
	testutil_check(session->create(session, new_uri, DEFAULT_TABLE_SCHEMA));

	__wt_yield();

	/*
	 * Opening a bulk cursor may have raced with a forced checkpoint
	 * which created a checkpoint of the empty file, and triggers an EINVAL.
	 */
	if ((ret = session->open_cursor(
	    session, new_uri, NULL, "bulk,checkpoint_wait=false", &c)) == 0) {
		testutil_check(c->close(c));
	} else if (ret != EINVAL && ret != EBUSY)
		testutil_die(ret,
		    "session.open_cursor bulk unique: %s", new_uri);

	while ((ret = session->drop(session, new_uri, __wt_random(&rnd) & 1 ?
	    "force,checkpoint_wait=false" : "checkpoint_wait=false")) != 0)
		if (ret != EBUSY)
			testutil_die(ret, "session.drop: %s", new_uri);
		else
			/*
			 * The EBUSY is expected when we run with
			 * checkpoint_wait set to false, so we increment the
			 * counter while in this loop to avoid false positives.
			 */
			args->thread_counter++;

	testutil_check(session->close(session, NULL));
	args->thread_counter++;
}
Exemple #6
0
void
populate(TEST_OPTS *opts)
{
	WT_CURSOR *maincur;
	WT_RAND_STATE rnd;
	WT_SESSION *session;
	uint32_t key;
	int balance, i, flag, post;

	__wt_random_init_seed(NULL, &rnd);

	testutil_check(opts->conn->open_session(
	    opts->conn, NULL, NULL, &session));

	testutil_check(session->open_cursor(session, opts->uri, NULL, NULL,
	    &maincur));

	for (i = 0; i < N_INSERT; i++) {
		testutil_check(session->begin_transaction(session, NULL));
		key = (__wt_random(&rnd) % (N_RECORDS));
		maincur->set_key(maincur, key);
		if (__wt_random(&rnd) % 11 == 0)
			post = 54321;
		else
			post = i % 100000;
		if (__wt_random(&rnd) % 4 == 0) {
			balance = -100;
			flag = 1;
		} else {
			balance = 100 * (i + 1);
			flag = 0;
		}
		maincur->set_value(maincur, post, balance, flag, key);
		testutil_check(maincur->insert(maincur));
		testutil_check(session->commit_transaction(session, NULL));
	}
	testutil_check(maincur->close(maincur));
	testutil_check(session->close(session, NULL));
}
Exemple #7
0
/*
 * thread_ckpt_run --
 *	Runner function for the checkpoint thread.
 */
static WT_THREAD_RET
thread_ckpt_run(void *arg)
{
	FILE *fp;
	WT_RAND_STATE rnd;
	WT_SESSION *session;
	WT_THREAD_DATA *td;
	uint64_t ts;
	uint32_t sleep_time;
	int i, ret;
	bool first_ckpt;

	__wt_random_init(&rnd);

	td = (WT_THREAD_DATA *)arg;
	/*
	 * Keep a separate file with the records we wrote for checking.
	 */
	(void)unlink(ckpt_file);
	if ((ret = td->conn->open_session(td->conn, NULL, NULL, &session)) != 0)
		testutil_die(ret, "WT_CONNECTION:open_session");
	first_ckpt = true;
	ts = 0;
	for (i = 0; ;++i) {
		sleep_time = __wt_random(&rnd) % MAX_CKPT_INTERVAL;
		sleep(sleep_time);
		if (use_ts)
			ts = global_ts;
		/*
		 * Since this is the default, send in this string even if
		 * running without timestamps.
		 */
		testutil_check(session->checkpoint(
		    session, "use_timestamp=true"));
		printf("Checkpoint %d complete.  Minimum ts %" PRIu64 "\n",
		    i, ts);
		fflush(stdout);
		/*
		 * Create the checkpoint file so that the parent process knows
		 * at least one checkpoint has finished and can start its
		 * timer.
		 */
		if (first_ckpt) {
			testutil_checksys((fp = fopen(ckpt_file, "w")) == NULL);
			first_ckpt = false;
			testutil_checksys(fclose(fp) != 0);
		}
	}
	/* NOTREACHED */
}
Exemple #8
0
/*
 * thread_ckpt_run --
 *	Runner function for the checkpoint thread.
 */
static WT_THREAD_RET
thread_ckpt_run(void *arg)
{
	FILE *fp;
	WT_RAND_STATE rnd;
	WT_SESSION *session;
	THREAD_DATA *td;
	uint64_t stable;
	uint32_t sleep_time;
	int i;
	bool first_ckpt;
	char ts_string[WT_TS_HEX_STRING_SIZE];

	__wt_random_init(&rnd);

	td = (THREAD_DATA *)arg;
	/*
	 * Keep a separate file with the records we wrote for checking.
	 */
	(void)unlink(ckpt_file);
	testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session));
	first_ckpt = true;
	for (i = 0; ;++i) {
		sleep_time = __wt_random(&rnd) % MAX_CKPT_INVL;
		sleep(sleep_time);
		/*
		 * Since this is the default, send in this string even if
		 * running without timestamps.
		 */
		testutil_check(session->checkpoint(
		    session, "use_timestamp=true"));
		testutil_check(td->conn->query_timestamp(
		    td->conn, ts_string, "get=last_checkpoint"));
		testutil_assert(sscanf(ts_string, "%" SCNx64, &stable) == 1);
		printf("Checkpoint %d complete at stable %"
		    PRIu64 ".\n", i, stable);
		fflush(stdout);
		/*
		 * Create the checkpoint file so that the parent process knows
		 * at least one checkpoint has finished and can start its
		 * timer.
		 */
		if (first_ckpt) {
			testutil_checksys((fp = fopen(ckpt_file, "w")) == NULL);
			first_ckpt = false;
			testutil_checksys(fclose(fp) != 0);
		}
	}
	/* NOTREACHED */
}
Exemple #9
0
/*
 * Create and drop a unique guaranteed table.
 */
void
op_create_unique(void *arg)
{
	TEST_OPTS *opts;
	TEST_PER_THREAD_OPTS *args;
	WT_RAND_STATE rnd;
	WT_SESSION *session;
	int ret;
	char new_uri[64];

	args = (TEST_PER_THREAD_OPTS *)arg;
	opts = args->testopts;
	__wt_random_init_seed(NULL, &rnd);

	testutil_check(
	    opts->conn->open_session(opts->conn, NULL, NULL, &session));

	/* Generate a unique object name. */
	testutil_check(__wt_snprintf(
	    new_uri, sizeof(new_uri), "%s.%" PRIu64,
	    opts->uri, __wt_atomic_add64(&opts->unique_id, 1)));
	testutil_check(session->create(session, new_uri, DEFAULT_TABLE_SCHEMA));

	__wt_yield();
	while ((ret = session->drop(session, new_uri, __wt_random(&rnd) & 1 ?
	    "force,checkpoint_wait=false" : "checkpoint_wait=false")) != 0)
		if (ret != EBUSY)
			testutil_die(ret, "session.drop: %s", new_uri);
		else
			/*
			 * The EBUSY is expected when we run with
			 * checkpoint_wait set to false, so we increment the
			 * counter while in this loop to avoid false positives.
			 */
			args->thread_counter++;

	testutil_check(session->close(session, NULL));
	args->thread_counter++;
}
Exemple #10
0
static void *
vthread(void *arg)
{
	WT_CURSOR *cursor_list[10];
	WT_DECL_RET;
	WT_RAND_STATE rnd;
	WT_SESSION *session;
	u_int i, next;

	(void)arg;

	memset(cursor_list, 0, sizeof(cursor_list));

	testutil_check(conn->open_session(conn, NULL, NULL, &session));
	__wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd);

	for (next = 0; !done;) {
		if (++next == WT_ELEMENTS(cursor_list))
			next = 0;
		op(session, &rnd, &cursor_list[next]);

		while (!done) {
			i = __wt_random(&rnd) % uris;
			ret = session->verify(session, uri_list[i], NULL);
			if (ret == EBUSY) {
				(void)__wt_atomic_add64(&verify_busy, 1);
				continue;
			}

			testutil_check(ret);
			(void)__wt_atomic_add64(&verify, 1);
			break;
		}
	}

	return (NULL);
}
Exemple #11
0
/*
 * Worker thread. Executes random operations from the set of 6.
 */
static WT_THREAD_RET
do_ops(void *args)
{
	WT_RAND_STATE rnd;
	time_t now, start;

	__wt_random_init_seed(NULL, &rnd);
	(void)time(&start);
	(void)time(&now);

	while (difftime(now, start) < RUNTIME) {
		switch (__wt_random(&rnd) % 6) {
		case 0:
			op_bulk(args);
			break;
		case 1:
			op_create(args);
			break;
		case 2:
			op_cursor(args);
			break;
		case 3:
			op_drop(args);
			break;
		case 4:
			op_bulk_unique(args);
			break;
		case 5:
			op_create_unique(args);
			break;
		}
		(void)time(&now);
	}

	return (WT_THREAD_RET_VALUE);
}
Exemple #12
0
/*
 * Drop a table.
 */
void
op_drop(void *arg)
{
	TEST_OPTS *opts;
	TEST_PER_THREAD_OPTS *args;
	WT_RAND_STATE rnd;
	WT_SESSION *session;
	int ret;

	args = (TEST_PER_THREAD_OPTS *)arg;
	opts = args->testopts;
	__wt_random_init_seed(NULL, &rnd);

	testutil_check(
	    opts->conn->open_session(opts->conn, NULL, NULL, &session));

	if ((ret = session->drop(session, opts->uri, __wt_random(&rnd) & 1 ?
	    "force,checkpoint_wait=false" : "checkpoint_wait=false")) != 0)
		if (ret != ENOENT && ret != EBUSY)
			testutil_die(ret, "session.drop");

	testutil_check(session->close(session, NULL));
	args->thread_counter++;
}
Exemple #13
0
/*
 * modify_run
 *	Run some tests:
 *	1. Create an initial value, a copy and a fake cursor to use with the
 *	WiredTiger routines. Generate a set of modify vectors and apply them to
 *	the item stored in the cursor using the modify apply API. Also apply the
 *	same modify vector to one of the copies using a helper routine written
 *	to test the modify API. The final value generated with the modify API
 *	and the helper routine should match.
 *
 *	2. Use the initial value and the modified value generated above as
 *	inputs into the calculate-modify API to generate a set of modify
 *	vectors. Apply this generated vector to the initial value using the
 *	modify apply API to obtain a final value. The final value generated
 *	should match the modified value that was used as input to the
 *	calculate-modify API.
 */
static void
modify_run(bool verbose)
{
	WT_CURSOR *cursor, _cursor;
	WT_DECL_RET;
	WT_ITEM *localA, _localA, *localB, _localB;
	size_t len;
	int i, j;

	/* Initialize the RNG. */
	__wt_random_init_seed(NULL, &rnd);

	/* Set up replacement information. */
	modify_repl_init();

	/* We need three WT_ITEMs, one of them part of a fake cursor. */
	localA = &_localA;
	memset(&_localA, 0, sizeof(_localA));
	localB = &_localB;
	memset(&_localB, 0, sizeof(_localB));
	cursor = &_cursor;
	memset(&_cursor, 0, sizeof(_cursor));
	cursor->value_format = "u";

#define	NRUNS	10000
	for (i = 0; i < NRUNS; ++i) {
		/* Create an initial value. */
		len = (size_t)(__wt_random(&rnd) % MAX_REPL_BYTES);
		testutil_check(__wt_buf_set(NULL, localA, modify_repl, len));

		for (j = 0; j < 1000; ++j) {
			/* Copy the current value into the second item. */
			testutil_check(__wt_buf_set(
			    NULL, localB, localA->data, localA->size));

			/*
			 * Create a random set of modify vectors, run the
			 * underlying library modification function, then
			 * compare the result against our implementation
			 * of modify.
			 */
			modify_build();
			testutil_check(__wt_buf_set(
			    NULL, &cursor->value, localA->data, localA->size));
			testutil_check(__wt_modify_apply_api(
			    NULL, cursor, entries, nentries));
			slow_apply_api(localA);
			compare(localA, &cursor->value);

			/*
			 * Call the WiredTiger function to build a modification
			 * vector for the change, and repeat the test using the
			 * WiredTiger modification vector, then compare results
			 * against our implementation of modify.
			 */
			nentries = WT_ELEMENTS(entries);
			ret = wiredtiger_calc_modify(NULL,
			    localB, localA,
			    WT_MAX(localB->size, localA->size) + 100,
			    entries, &nentries);
			if (ret == WT_NOTFOUND)
				continue;
			testutil_check(ret);
			testutil_check(__wt_buf_set(
			    NULL, &cursor->value, localB->data, localB->size));
			testutil_check(__wt_modify_apply_api(
			    NULL, cursor, entries, nentries));
			compare(localA, &cursor->value);
		}
		if (verbose) {
			printf("%d (%d%%)\r", i, (i * 100) / NRUNS);
			fflush(stdout);
		}
	}
	if (verbose)
		printf("%d (100%%)\n", i);

	__wt_buf_free(NULL, localA);
	__wt_buf_free(NULL, localB);
	__wt_buf_free(NULL, &cursor->value);
}
Exemple #14
0
/*
 * Child process creates the database and table, and then writes data into
 * the table until it is killed by the parent.
 */
static void
fill_db(void)
{
	FILE *fp;
	WT_CONNECTION *conn;
	WT_CURSOR *cursor;
	WT_ITEM data;
	WT_RAND_STATE rnd;
	WT_SESSION *session;
	uint64_t i;
	int ret;
	uint8_t buf[MAX_VAL];

	__wt_random_init(&rnd);
	memset(buf, 0, sizeof(buf));
	/*
	 * Initialize the first 25% to random values.  Leave a bunch of data
	 * space at the end to emphasize zero data.
	 */
	for (i = 0; i < MAX_VAL/4; i++)
		buf[i] = (uint8_t)__wt_random(&rnd);

	/*
	 * Run in the home directory so that the records file is in there too.
	 */
	chdir(home);
	if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG, &conn)) != 0)
		testutil_die(ret, "wiredtiger_open");
	if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
		testutil_die(ret, "WT_CONNECTION:open_session");
	if ((ret = session->create(session,
	    uri, "key_format=Q,value_format=u")) != 0)
		testutil_die(ret, "WT_SESSION.create: %s", uri);
	if ((ret =
	    session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0)
		testutil_die(ret, "WT_SESSION.open_cursor: %s", uri);

	/*
	 * Keep a separate file with the records we wrote for checking.
	 */
	(void)unlink(RECORDS_FILE);
	if ((fp = fopen(RECORDS_FILE, "w")) == NULL)
		testutil_die(errno, "fopen");
	/*
	 * Set to no buffering.
	 */
	setvbuf(fp, NULL, _IONBF, 0);

	/*
	 * Write data into the table until we are killed by the parent.
	 * The data in the buffer is already set to random content.
	 */
	data.data = buf;
	for (i = 0;; ++i) {
		data.size = __wt_random(&rnd) % MAX_VAL;
		cursor->set_key(cursor, i);
		cursor->set_value(cursor, &data);
		if ((ret = cursor->insert(cursor)) != 0)
			testutil_die(ret, "WT_CURSOR.insert");
		/*
		 * Save the key separately for checking later.
		 */
		if (fprintf(fp, "%" PRIu64 "\n", i) == -1)
			testutil_die(errno, "fprintf");
		if (i % 5000)
			__wt_yield();
	}
}
Exemple #15
0
int
main(int argc, char *argv[])
{
	enum { CACHE_SHARED, CACHE_SET, CACHE_NONE } cache;
	TEST_OPTS *opts, _opts;
	WT_RAND_STATE rnd;
	WT_SESSION *session;
	size_t len;
	u_int i, j;
	const char *p;
	char *config;

	opts = &_opts;
	memset(opts, 0, sizeof(*opts));
	opts->table_type = TABLE_ROW;
	testutil_check(testutil_parse_opts(argc, argv, opts));
	testutil_make_work_dir(opts->home);

	testutil_check(
	    wiredtiger_open(opts->home, &event_handler, "create", &opts->conn));

	/* Open an LSM file so the LSM reconfiguration options make sense. */
	testutil_check(
	    opts->conn->open_session(opts->conn, NULL, NULL, &session));
	testutil_check(session->create(
	    session, opts->uri, "type=lsm,key_format=S,value_format=S"));

	/* Initialize the RNG. */
	__wt_random_init_seed(NULL, &rnd);

	/* Allocate memory for the config. */
	len = WT_ELEMENTS(list) * 64;
	config = dmalloc(len);

	/* Set an alarm so we can debug hangs. */
	(void)signal(SIGALRM, on_alarm);

	/* A linear pass through the list. */
	for (i = 0; i < WT_ELEMENTS(list); ++i)
		reconfig(opts, session, list[i]);

	/*
	 * A linear pass through the list, adding random elements.
	 *
	 * WiredTiger configurations are usually "the last one set wins", but
	 * "shared_cache" and "cache_set" options aren't allowed in the same
	 * configuration string.
	 */
	for (i = 0; i < WT_ELEMENTS(list); ++i) {
		p = list[i];
		cache = CACHE_NONE;
		if (WT_PREFIX_MATCH(p, ",shared_cache"))
			cache = CACHE_SHARED;
		else if (WT_PREFIX_MATCH(p, ",cache_size"))
			cache = CACHE_SET;
		strcpy(config, p);

		for (j =
		    (__wt_random(&rnd) % WT_ELEMENTS(list)) + 1; j > 0; --j) {
			p = list[__wt_random(&rnd) % WT_ELEMENTS(list)];
			if (WT_PREFIX_MATCH(p, ",shared_cache")) {
				if (cache == CACHE_SET)
					continue;
				cache = CACHE_SHARED;
			} else if (WT_PREFIX_MATCH(p, ",cache_size")) {
				if (cache == CACHE_SHARED)
					continue;
				cache = CACHE_SET;
			}
			strcat(config, p);
		}
		reconfig(opts, session, config);
	}

	/*
	 * Turn on-close statistics off, if on-close is on and statistics were
	 * randomly turned off during the run, close would fail.
	 */
	testutil_check(opts->conn->reconfigure(
	    opts->conn, "statistics_log=(on_close=0)"));

	free(config);
	testutil_cleanup(opts);
	return (EXIT_SUCCESS);
}
Exemple #16
0
/*
 * subtest_populate --
 *	Populate the tables.
 */
static void
subtest_populate(TEST_OPTS *opts, bool close_test)
{
	WT_CURSOR *maincur, *maincur2;
	WT_RAND_STATE rnd;
	WT_SESSION *session;
	uint64_t i, nrecords;
	uint32_t rndint;
	int key, v0, v1, v2;
	char *big, *bigref;
	bool failed;

	failed = false;
	__wt_random_init_seed(NULL, &rnd);
	CHECK(create_big_string(&bigref), false);
	nrecords = opts->nrecords;

	CHECK(opts->conn->open_session(
	    opts->conn, NULL, NULL, &session), false);

	CHECK(session->open_cursor(session, "table:subtest", NULL,
	    NULL, &maincur), false);

	CHECK(session->open_cursor(session, "table:subtest2", NULL,
	    NULL, &maincur2), false);

	for (i = 0; i < nrecords && !failed; i++) {
		rndint = __wt_random(&rnd);
		generate_key(i, &key);
		generate_value(rndint, i, bigref, &v0, &v1, &v2, &big);
		CHECK(session->begin_transaction(session, NULL), false);
		maincur->set_key(maincur, key);
		maincur->set_value(maincur, v0, v1, v2, big);
		CHECK(maincur->insert(maincur), false);

		maincur2->set_key(maincur2, key);
		maincur2->set_value(maincur2, rndint);
		CHECK(maincur2->insert(maincur2), false);
		CHECK(session->commit_transaction(session, NULL), false);

		if (i == 0)
			/*
			 * Force an initial checkpoint, that helps to
			 * distinguish a clear failure from just not running
			 * long enough.
			 */
			CHECK(session->checkpoint(session, NULL), false);

		if ((i + 1) % VERBOSE_PRINT == 0 && opts->verbose)
			printf("  %" PRIu64 "/%" PRIu64 "\n",
			    (i + 1), nrecords);
		/* Attempt to isolate the failures to checkpointing. */
		if (i == (nrecords/100)) {
			enable_failures(opts->nops, 1000000);
			/* CHECK should expect failures. */
			CHECK(session->checkpoint(session, NULL), true);
			disable_failures();
			if (failed && opts->verbose)
				printf("checkpoint failed (expected).\n");
		}
	}

	/*
	 * Closing handles after an extreme fail is likely to cause
	 * cascading failures (or crashes), so recommended practice is
	 * to immediately exit. We're interested in testing both with
	 * and without the recommended practice.
	 */
	if (failed) {
		if (!close_test) {
			fprintf(stderr, "exit early.\n");
			exit(0);
		} else
			fprintf(stderr, "closing after failure.\n");
	}

	free(bigref);
	CHECK(maincur->close(maincur), false);
	CHECK(maincur2->close(maincur2), false);
	CHECK(session->close(session, NULL), false);
}
Exemple #17
0
/*
 * __wt_log_slot_join --
 *	Join a consolidated logging slot. Callers should be prepared to deal
 *	with a ENOMEM return - which indicates no slots could accommodate
 *	the log record.
 */
int
__wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize,
    uint32_t flags, WT_MYSLOT *myslotp)
{
	WT_CONNECTION_IMPL *conn;
	WT_LOG *log;
	WT_LOGSLOT *slot;
	int64_t cur_state, new_state, old_state;
	uint32_t allocated_slot, slot_grow_attempts;

	conn = S2C(session);
	log = conn->log;
	slot_grow_attempts = 0;
find_slot:
	allocated_slot = __wt_random(session->rnd) % SLOT_ACTIVE;
	slot = log->slot_array[allocated_slot];
	old_state = slot->slot_state;
join_slot:
	/*
	 * WT_LOG_SLOT_READY and higher means the slot is available for
	 * joining.  Any other state means it is in use and transitioning
	 * from the active array.
	 */
	if (old_state < WT_LOG_SLOT_READY) {
		WT_STAT_FAST_CONN_INCR(session, log_slot_transitions);
		goto find_slot;
	}
	/*
	 * Add in our size to the state and then atomically swap that
	 * into place if it is still the same value.
	 */
	new_state = old_state + (int64_t)mysize;
	if (new_state < old_state) {
		/* Our size doesn't fit here. */
		WT_STAT_FAST_CONN_INCR(session, log_slot_toobig);
		goto find_slot;
	}
	/*
	 * If the slot buffer isn't big enough to hold this update, mark
	 * the slot for a buffer size increase and find another slot.
	 */
	if (new_state > (int64_t)slot->slot_buf.memsize) {
		F_SET(slot, SLOT_BUF_GROW);
		if (++slot_grow_attempts > 5) {
			WT_STAT_FAST_CONN_INCR(session, log_slot_toosmall);
			return (ENOMEM);
		}
		goto find_slot;
	}
	cur_state = WT_ATOMIC_CAS_VAL8(slot->slot_state, old_state, new_state);
	/*
	 * We lost a race to add our size into this slot.  Check the state
	 * and try again.
	 */
	if (cur_state != old_state) {
		old_state = cur_state;
		WT_STAT_FAST_CONN_INCR(session, log_slot_races);
		goto join_slot;
	}
	WT_ASSERT(session, myslotp != NULL);
	/*
	 * We joined this slot.  Fill in our information to return to
	 * the caller.
	 */
	WT_STAT_FAST_CONN_INCR(session, log_slot_joins);
	if (LF_ISSET(WT_LOG_DSYNC | WT_LOG_FSYNC))
		F_SET(slot, SLOT_SYNC_DIR);
	if (LF_ISSET(WT_LOG_FSYNC))
		F_SET(slot, SLOT_SYNC);
	myslotp->slot = slot;
	myslotp->offset = (wt_off_t)old_state - WT_LOG_SLOT_READY;
	return (0);
}
Exemple #18
0
int
main(int argc, char *argv[])
{
	struct sigaction sa;
	struct stat sb;
	FILE *fp;
	REPORT c_rep[MAX_TH], l_rep[MAX_TH], o_rep[MAX_TH];
	WT_CONNECTION *conn;
	WT_CURSOR *cur_coll, *cur_local, *cur_oplog;
	WT_RAND_STATE rnd;
	WT_SESSION *session;
	pid_t pid;
	uint64_t absent_coll, absent_local, absent_oplog, count, key, last_key;
	uint64_t stable_fp, stable_val;
	uint32_t i, nth, timeout;
	int ch, status, ret;
	const char *working_dir;
	char buf[512], fname[64], kname[64], statname[1024];
	char ts_string[WT_TS_HEX_STRING_SIZE];
	bool fatal, rand_th, rand_time, verify_only;

	(void)testutil_set_progname(argv);

	compat = inmem = false;
	use_ts = true;
	nth = MIN_TH;
	rand_th = rand_time = true;
	timeout = MIN_TIME;
	verify_only = false;
	working_dir = "WT_TEST.timestamp-abort";

	while ((ch = __wt_getopt(progname, argc, argv, "Ch:LmT:t:vz")) != EOF)
		switch (ch) {
		case 'C':
			compat = true;
			break;
		case 'h':
			working_dir = __wt_optarg;
			break;
		case 'L':
			table_pfx = "lsm";
			break;
		case 'm':
			inmem = true;
			break;
		case 'T':
			rand_th = false;
			nth = (uint32_t)atoi(__wt_optarg);
			if (nth > MAX_TH) {
				fprintf(stderr,
				    "Number of threads is larger than the"
				    " maximum %" PRId32 "\n", MAX_TH);
				return (EXIT_FAILURE);
			}
			break;
		case 't':
			rand_time = false;
			timeout = (uint32_t)atoi(__wt_optarg);
			break;
		case 'v':
			verify_only = true;
			break;
		case 'z':
			use_ts = false;
			break;
		default:
			usage();
		}
	argc -= __wt_optind;
	if (argc != 0)
		usage();

	testutil_work_dir_from_path(home, sizeof(home), working_dir);
	testutil_check(pthread_rwlock_init(&ts_lock, NULL));

	/*
	 * If the user wants to verify they need to tell us how many threads
	 * there were so we can find the old record files.
	 */
	if (verify_only && rand_th) {
		fprintf(stderr,
		    "Verify option requires specifying number of threads\n");
		exit (EXIT_FAILURE);
	}
	if (!verify_only) {
		testutil_make_work_dir(home);

		__wt_random_init_seed(NULL, &rnd);
		if (rand_time) {
			timeout = __wt_random(&rnd) % MAX_TIME;
			if (timeout < MIN_TIME)
				timeout = MIN_TIME;
		}
		if (rand_th) {
			nth = __wt_random(&rnd) % MAX_TH;
			if (nth < MIN_TH)
				nth = MIN_TH;
		}

		printf("Parent: compatibility: %s, "
		    "in-mem log sync: %s, timestamp in use: %s\n",
		    compat ? "true" : "false",
		    inmem ? "true" : "false",
		    use_ts ? "true" : "false");
		printf("Parent: Create %" PRIu32
		    " threads; sleep %" PRIu32 " seconds\n", nth, timeout);
		printf("CONFIG: %s%s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n",
		    progname,
		    compat ? " -C" : "",
		    inmem ? " -m" : "",
		    !use_ts ? " -z" : "",
		    working_dir, nth, timeout);
		/*
		 * Fork a child to insert as many items.  We will then randomly
		 * kill the child, run recovery and make sure all items we wrote
		 * exist after recovery runs.
		 */
		memset(&sa, 0, sizeof(sa));
		sa.sa_handler = handler;
		testutil_checksys(sigaction(SIGCHLD, &sa, NULL));
		testutil_checksys((pid = fork()) < 0);

		if (pid == 0) { /* child */
			run_workload(nth);
			return (EXIT_SUCCESS);
		}

		/* parent */
		/*
		 * Sleep for the configured amount of time before killing
		 * the child.  Start the timeout from the time we notice that
		 * the file has been created.  That allows the test to run
		 * correctly on really slow machines.
		 */
		testutil_check(__wt_snprintf(
		    statname, sizeof(statname), "%s/%s", home, ckpt_file));
		while (stat(statname, &sb) != 0)
			testutil_sleep_wait(1, pid);
		sleep(timeout);
		sa.sa_handler = SIG_DFL;
		testutil_checksys(sigaction(SIGCHLD, &sa, NULL));

		/*
		 * !!! It should be plenty long enough to make sure more than
		 * one log file exists.  If wanted, that check would be added
		 * here.
		 */
		printf("Kill child\n");
		testutil_checksys(kill(pid, SIGKILL) != 0);
		testutil_checksys(waitpid(pid, &status, 0) == -1);
	}
	/*
	 * !!! If we wanted to take a copy of the directory before recovery,
	 * this is the place to do it. Don't do it all the time because
	 * it can use a lot of disk space, which can cause test machine
	 * issues.
	 */
	if (chdir(home) != 0)
		testutil_die(errno, "parent chdir: %s", home);
	/*
	 * The tables can get very large, so while we'd ideally like to
	 * copy the entire database, we only copy the log files for now.
	 * Otherwise it can take far too long to run the test, particularly
	 * in automated testing.
	 */
	testutil_check(__wt_snprintf(buf, sizeof(buf),
	    "rm -rf ../%s.SAVE && mkdir ../%s.SAVE && "
	    "cp -p * ../%s.SAVE",
	     home, home, home));
	if ((status = system(buf)) < 0)
		testutil_die(status, "system: %s", buf);
	printf("Open database, run recovery and verify content\n");

	/*
	 * Open the connection which forces recovery to be run.
	 */
	testutil_check(wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn));
	testutil_check(conn->open_session(conn, NULL, NULL, &session));
	/*
	 * Open a cursor on all the tables.
	 */
	testutil_check(__wt_snprintf(
	    buf, sizeof(buf), "%s:%s", table_pfx, uri_collection));
	testutil_check(session->open_cursor(session,
	    buf, NULL, NULL, &cur_coll));
	testutil_check(__wt_snprintf(
	    buf, sizeof(buf), "%s:%s", table_pfx, uri_local));
	testutil_check(session->open_cursor(session,
	    buf, NULL, NULL, &cur_local));
	testutil_check(__wt_snprintf(
	    buf, sizeof(buf), "%s:%s", table_pfx, uri_oplog));
	testutil_check(session->open_cursor(session,
	    buf, NULL, NULL, &cur_oplog));

	/*
	 * Find the biggest stable timestamp value that was saved.
	 */
	stable_val = 0;
	if (use_ts) {
		testutil_check(
		    conn->query_timestamp(conn, ts_string, "get=recovery"));
		testutil_assert(
		    sscanf(ts_string, "%" SCNx64, &stable_val) == 1);
		printf("Got stable_val %" PRIu64 "\n", stable_val);
	}

	count = 0;
	absent_coll = absent_local = absent_oplog = 0;
	fatal = false;
	for (i = 0; i < nth; ++i) {
		initialize_rep(&c_rep[i]);
		initialize_rep(&l_rep[i]);
		initialize_rep(&o_rep[i]);
		testutil_check(__wt_snprintf(
		    fname, sizeof(fname), RECORDS_FILE, i));
		if ((fp = fopen(fname, "r")) == NULL)
			testutil_die(errno, "fopen: %s", fname);

		/*
		 * For every key in the saved file, verify that the key exists
		 * in the table after recovery.  If we're doing in-memory
		 * log buffering we never expect a record missing in the middle,
		 * but records may be missing at the end.  If we did
		 * write-no-sync, we expect every key to have been recovered.
		 */
		for (last_key = INVALID_KEY;; ++count, last_key = key) {
			ret = fscanf(fp, "%" SCNu64 "%" SCNu64 "\n",
			    &stable_fp, &key);
			if (last_key == INVALID_KEY) {
				c_rep[i].first_key = key;
				l_rep[i].first_key = key;
				o_rep[i].first_key = key;
			}
			if (ret != EOF && ret != 2) {
				/*
				 * If we find a partial line, consider it
				 * like an EOF.
				 */
				if (ret == 1 || ret == 0)
					break;
				testutil_die(errno, "fscanf");
			}
			if (ret == EOF)
				break;
			/*
			 * If we're unlucky, the last line may be a partially
			 * written key at the end that can result in a false
			 * negative error for a missing record.  Detect it.
			 */
			if (last_key != INVALID_KEY && key != last_key + 1) {
				printf("%s: Ignore partial record %" PRIu64
				    " last valid key %" PRIu64 "\n",
				    fname, key, last_key);
				break;
			}
			testutil_check(__wt_snprintf(
			    kname, sizeof(kname), "%" PRIu64, key));
			cur_coll->set_key(cur_coll, kname);
			cur_local->set_key(cur_local, kname);
			cur_oplog->set_key(cur_oplog, kname);
			/*
			 * The collection table should always only have the
			 * data as of the checkpoint.
			 */
			if ((ret = cur_coll->search(cur_coll)) != 0) {
				if (ret != WT_NOTFOUND)
					testutil_die(ret, "search");
				/*
				 * If we don't find a record, the stable
				 * timestamp written to our file better be
				 * larger than the saved one.
				 */
				if (!inmem &&
				    stable_fp != 0 && stable_fp <= stable_val) {
					printf("%s: COLLECTION no record with "
					    "key %" PRIu64 " record ts %" PRIu64
					    " <= stable ts %" PRIu64 "\n",
					    fname, key, stable_fp, stable_val);
					absent_coll++;
				}
				if (c_rep[i].first_miss == INVALID_KEY)
					c_rep[i].first_miss = key;
				c_rep[i].absent_key = key;
			} else if (c_rep[i].absent_key != INVALID_KEY &&
			    c_rep[i].exist_key == INVALID_KEY) {
				/*
				 * If we get here we found a record that exists
				 * after absent records, a hole in our data.
				 */
				c_rep[i].exist_key = key;
				fatal = true;
			} else if (!inmem &&
			    stable_fp != 0 && stable_fp > stable_val) {
				/*
				 * If we found a record, the stable timestamp
				 * written to our file better be no larger
				 * than the checkpoint one.
				 */
				printf("%s: COLLECTION record with "
				    "key %" PRIu64 " record ts %" PRIu64
				    " > stable ts %" PRIu64 "\n",
				    fname, key, stable_fp, stable_val);
				fatal = true;
			}
			/*
			 * The local table should always have all data.
			 */
			if ((ret = cur_local->search(cur_local)) != 0) {
				if (ret != WT_NOTFOUND)
					testutil_die(ret, "search");
				if (!inmem)
					printf("%s: LOCAL no record with key %"
					    PRIu64 "\n", fname, key);
				absent_local++;
				if (l_rep[i].first_miss == INVALID_KEY)
					l_rep[i].first_miss = key;
				l_rep[i].absent_key = key;
			} else if (l_rep[i].absent_key != INVALID_KEY &&
			    l_rep[i].exist_key == INVALID_KEY) {
				/*
				 * We should never find an existing key after
				 * we have detected one missing.
				 */
				l_rep[i].exist_key = key;
				fatal = true;
			}
			/*
			 * The oplog table should always have all data.
			 */
			if ((ret = cur_oplog->search(cur_oplog)) != 0) {
				if (ret != WT_NOTFOUND)
					testutil_die(ret, "search");
				if (!inmem)
					printf("%s: OPLOG no record with key %"
					    PRIu64 "\n", fname, key);
				absent_oplog++;
				if (o_rep[i].first_miss == INVALID_KEY)
					o_rep[i].first_miss = key;
				o_rep[i].absent_key = key;
			} else if (o_rep[i].absent_key != INVALID_KEY &&
			    o_rep[i].exist_key == INVALID_KEY) {
				/*
				 * We should never find an existing key after
				 * we have detected one missing.
				 */
				o_rep[i].exist_key = key;
				fatal = true;
			}
		}
		c_rep[i].last_key = last_key;
		l_rep[i].last_key = last_key;
		o_rep[i].last_key = last_key;
		testutil_checksys(fclose(fp) != 0);
		print_missing(&c_rep[i], fname, "COLLECTION");
		print_missing(&l_rep[i], fname, "LOCAL");
		print_missing(&o_rep[i], fname, "OPLOG");
	}
	testutil_check(conn->close(conn, NULL));
	if (!inmem && absent_coll) {
		printf("COLLECTION: %" PRIu64
		    " record(s) absent from %" PRIu64 "\n",
		    absent_coll, count);
		fatal = true;
	}
	if (!inmem && absent_local) {
		printf("LOCAL: %" PRIu64 " record(s) absent from %" PRIu64 "\n",
		    absent_local, count);
		fatal = true;
	}
	if (!inmem && absent_oplog) {
		printf("OPLOG: %" PRIu64 " record(s) absent from %" PRIu64 "\n",
		    absent_oplog, count);
		fatal = true;
	}
	testutil_check(pthread_rwlock_destroy(&ts_lock));
	if (fatal)
		return (EXIT_FAILURE);
	printf("%" PRIu64 " records verified\n", count);
	return (EXIT_SUCCESS);
}
Exemple #19
0
int
main(int argc, char *argv[])
{
	struct stat sb;
	FILE *fp;
	WT_CONNECTION *conn;
	WT_CURSOR *cur_coll, *cur_local, *cur_oplog, *cur_stable;
	WT_RAND_STATE rnd;
	WT_SESSION *session;
	pid_t pid;
	uint64_t absent_coll, absent_local, absent_oplog, count, key, last_key;
	uint64_t first_miss, middle_coll, middle_local, middle_oplog;
	uint64_t stable_fp, stable_val, val[MAX_TH+1];
	uint32_t i, nth, timeout;
	int ch, status, ret;
	const char *working_dir;
	char buf[512], fname[64], kname[64], statname[1024];
	bool fatal, rand_th, rand_time, verify_only;

	(void)testutil_set_progname(argv);

	compat = inmem = false;
	use_ts = true;
	nth = MIN_TH;
	rand_th = rand_time = true;
	timeout = MIN_TIME;
	verify_only = false;
	working_dir = "WT_TEST.timestamp-abort";

	while ((ch = __wt_getopt(progname, argc, argv, "Ch:mT:t:vz")) != EOF)
		switch (ch) {
		case 'C':
			compat = true;
			break;
		case 'h':
			working_dir = __wt_optarg;
			break;
		case 'm':
			inmem = true;
			break;
		case 'T':
			rand_th = false;
			nth = (uint32_t)atoi(__wt_optarg);
			break;
		case 't':
			rand_time = false;
			timeout = (uint32_t)atoi(__wt_optarg);
			break;
		case 'v':
			verify_only = true;
			break;
		case 'z':
			use_ts = false;
			break;
		default:
			usage();
		}
	argc -= __wt_optind;
	argv += __wt_optind;
	if (argc != 0)
		usage();

	testutil_work_dir_from_path(home, sizeof(home), working_dir);
	/*
	 * If the user wants to verify they need to tell us how many threads
	 * there were so we can find the old record files.
	 */
	if (verify_only && rand_th) {
		fprintf(stderr,
		    "Verify option requires specifying number of threads\n");
		exit (EXIT_FAILURE);
	}
	if (!verify_only) {
		testutil_make_work_dir(home);

		__wt_random_init_seed(NULL, &rnd);
		if (rand_time) {
			timeout = __wt_random(&rnd) % MAX_TIME;
			if (timeout < MIN_TIME)
				timeout = MIN_TIME;
		}
		if (rand_th) {
			nth = __wt_random(&rnd) % MAX_TH;
			if (nth < MIN_TH)
				nth = MIN_TH;
		}
		printf("Parent: compatibility: %s, "
		    "in-mem log sync: %s, timestamp in use: %s\n",
		    compat ? "true" : "false",
		    inmem ? "true" : "false",
		    use_ts ? "true" : "false");
		printf("Parent: Create %" PRIu32
		    " threads; sleep %" PRIu32 " seconds\n", nth, timeout);
		/*
		 * Fork a child to insert as many items.  We will then randomly
		 * kill the child, run recovery and make sure all items we wrote
		 * exist after recovery runs.
		 */
		testutil_checksys((pid = fork()) < 0);

		if (pid == 0) { /* child */
			run_workload(nth);
			return (EXIT_SUCCESS);
		}

		/* parent */
		/*
		 * Sleep for the configured amount of time before killing
		 * the child.  Start the timeout from the time we notice that
		 * the file has been created.  That allows the test to run
		 * correctly on really slow machines.  Verify the process ID
		 * still exists in case the child aborts for some reason we
		 * don't stay in this loop forever.
		 */
		testutil_check(__wt_snprintf(
		    statname, sizeof(statname), "%s/%s", home, ckpt_file));
		while (stat(statname, &sb) != 0 && kill(pid, 0) == 0)
			sleep(1);
		sleep(timeout);

		/*
		 * !!! It should be plenty long enough to make sure more than
		 * one log file exists.  If wanted, that check would be added
		 * here.
		 */
		printf("Kill child\n");
		testutil_checksys(kill(pid, SIGKILL) != 0);
		testutil_checksys(waitpid(pid, &status, 0) == -1);
	}
	/*
	 * !!! If we wanted to take a copy of the directory before recovery,
	 * this is the place to do it.
	 */
	if (chdir(home) != 0)
		testutil_die(errno, "parent chdir: %s", home);
	testutil_check(__wt_snprintf(buf, sizeof(buf),
	    "rm -rf ../%s.SAVE && mkdir ../%s.SAVE && "
	    "cp -p WiredTigerLog.* ../%s.SAVE",
	     home, home, home));
	(void)system(buf);
	printf("Open database, run recovery and verify content\n");

	/*
	 * Open the connection which forces recovery to be run.
	 */
	if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn)) != 0)
		testutil_die(ret, "wiredtiger_open");
	if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
		testutil_die(ret, "WT_CONNECTION:open_session");
	/*
	 * Open a cursor on all the tables.
	 */
	if ((ret = session->open_cursor(session,
	    uri_collection, NULL, NULL, &cur_coll)) != 0)
		testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_collection);
	if ((ret = session->open_cursor(session,
	    uri_local, NULL, NULL, &cur_local)) != 0)
		testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_local);
	if ((ret = session->open_cursor(session,
	    uri_oplog, NULL, NULL, &cur_oplog)) != 0)
		testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_oplog);
	if ((ret = session->open_cursor(session,
	    stable_store, NULL, NULL, &cur_stable)) != 0)
		testutil_die(ret, "WT_SESSION.open_cursor: %s", stable_store);

	/*
	 * Find the biggest stable timestamp value that was saved.
	 */
	stable_val = 0;
	memset(val, 0, sizeof(val));
	while (cur_stable->next(cur_stable) == 0) {
		cur_stable->get_key(cur_stable, &key);
		cur_stable->get_value(cur_stable, &val[key]);
		if (val[key] > stable_val)
			stable_val = val[key];

		if (use_ts)
			printf("Stable: key %" PRIu64 " value %" PRIu64 "\n",
			    key, val[key]);
	}
	if (use_ts)
		printf("Got stable_val %" PRIu64 "\n", stable_val);

	count = 0;
	absent_coll = absent_local = absent_oplog = 0;
	fatal = false;
	for (i = 1; i <= nth; ++i) {
		first_miss = middle_coll = middle_local = middle_oplog = 0;
		testutil_check(__wt_snprintf(
		    fname, sizeof(fname), RECORDS_FILE, i));
		if ((fp = fopen(fname, "r")) == NULL)
			testutil_die(errno, "fopen: %s", fname);

		/*
		 * For every key in the saved file, verify that the key exists
		 * in the table after recovery.  If we're doing in-memory
		 * log buffering we never expect a record missing in the middle,
		 * but records may be missing at the end.  If we did
		 * write-no-sync, we expect every key to have been recovered.
		 */
		for (last_key = UINT64_MAX;; ++count, last_key = key) {
			ret = fscanf(fp, "%" SCNu64 "%" SCNu64 "\n",
			    &stable_fp, &key);
			if (ret != EOF && ret != 2) {
				/*
				 * If we find a partial line, consider it
				 * like an EOF.
				 */
				if (ret == 1 || ret == 0)
					break;
				testutil_die(errno, "fscanf");
			}
			if (ret == EOF)
				break;
			/*
			 * If we're unlucky, the last line may be a partially
			 * written key at the end that can result in a false
			 * negative error for a missing record.  Detect it.
			 */
			if (last_key != UINT64_MAX && key != last_key + 1) {
				printf("%s: Ignore partial record %" PRIu64
				    " last valid key %" PRIu64 "\n",
				    fname, key, last_key);
				break;
			}
			testutil_check(__wt_snprintf(
			    kname, sizeof(kname), "%" PRIu64, key));
			cur_coll->set_key(cur_coll, kname);
			cur_local->set_key(cur_local, kname);
			cur_oplog->set_key(cur_oplog, kname);
			/*
			 * The collection table should always only have the
			 * data as of the checkpoint.
			 */
			if ((ret = cur_coll->search(cur_coll)) != 0) {
				if (ret != WT_NOTFOUND)
					testutil_die(ret, "search");
				/*
				 * If we don't find a record, the stable
				 * timestamp written to our file better be
				 * larger than the saved one.
				 */
				if (!inmem &&
				    stable_fp != 0 && stable_fp <= val[i]) {
					printf("%s: COLLECTION no record with "
					    "key %" PRIu64 " record ts %" PRIu64
					    " <= stable ts %" PRIu64 "\n",
					    fname, key, stable_fp, val[i]);
					absent_coll++;
				}
				if (middle_coll == 0)
					first_miss = key;
				middle_coll = key;
			} else if (middle_coll != 0) {
				/*
				 * We should never find an existing key after
				 * we have detected one missing.
				 */
				printf("%s: COLLECTION after absent records %"
				    PRIu64 "-%" PRIu64 " key %" PRIu64
				    " exists\n",
				    fname, first_miss, middle_coll, key);
				fatal = true;
			}
			/*
			 * The local table should always have all data.
			 */
			if ((ret = cur_local->search(cur_local)) != 0) {
				if (ret != WT_NOTFOUND)
					testutil_die(ret, "search");
				if (!inmem)
					printf("%s: LOCAL no record with key %"
					    PRIu64 "\n", fname, key);
				absent_local++;
				middle_local = key;
			} else if (middle_local != 0) {
				/*
				 * We should never find an existing key after
				 * we have detected one missing.
				 */
				printf("%s: LOCAL after absent record at %"
				    PRIu64 " key %" PRIu64 " exists\n",
				    fname, middle_local, key);
				fatal = true;
			}
			/*
			 * The oplog table should always have all data.
			 */
			if ((ret = cur_oplog->search(cur_oplog)) != 0) {
				if (ret != WT_NOTFOUND)
					testutil_die(ret, "search");
				if (!inmem)
					printf("%s: OPLOG no record with key %"
					    PRIu64 "\n", fname, key);
				absent_oplog++;
				middle_oplog = key;
			} else if (middle_oplog != 0) {
				/*
				 * We should never find an existing key after
				 * we have detected one missing.
				 */
				printf("%s: OPLOG after absent record at %"
				    PRIu64 " key %" PRIu64 " exists\n",
				    fname, middle_oplog, key);
				fatal = true;
			}
		}
		testutil_checksys(fclose(fp) != 0);
	}
	if ((ret = conn->close(conn, NULL)) != 0)
		testutil_die(ret, "WT_CONNECTION:close");
	if (fatal)
		return (EXIT_FAILURE);
	if (!inmem && absent_coll) {
		printf("COLLECTION: %" PRIu64
		    " record(s) absent from %" PRIu64 "\n",
		    absent_coll, count);
		fatal = true;
	}
	if (!inmem && absent_local) {
		printf("LOCAL: %" PRIu64 " record(s) absent from %" PRIu64 "\n",
		    absent_local, count);
		fatal = true;
	}
	if (!inmem && absent_oplog) {
		printf("OPLOG: %" PRIu64 " record(s) absent from %" PRIu64 "\n",
		    absent_oplog, count);
		fatal = true;
	}
	if (fatal)
		return (EXIT_FAILURE);
	printf("%" PRIu64 " records verified\n", count);
	return (EXIT_SUCCESS);
}
Exemple #20
0
/*
 * real_worker --
 *     A single worker thread that transactionally updates all tables with
 *     consistent values.
 */
static int
real_worker(void)
{
	WT_CURSOR **cursors;
	WT_RAND_STATE rnd;
	WT_SESSION *session;
	u_int i, keyno;
	int j, ret, t_ret;

	ret = t_ret = 0;

	__wt_random_init(&rnd);

	if ((cursors = calloc(
	    (size_t)(g.ntables), sizeof(WT_CURSOR *))) == NULL)
		return (log_print_err("malloc", ENOMEM, 1));

	if ((ret = g.conn->open_session(
	    g.conn, NULL, "isolation=snapshot", &session)) != 0) {
		(void)log_print_err("conn.open_session", ret, 1);
		goto err;
	}

	for (j = 0; j < g.ntables; j++)
		if ((ret = session->open_cursor(session,
		    g.cookies[j].uri, NULL, NULL, &cursors[j])) != 0) {
			(void)log_print_err("session.open_cursor", ret, 1);
			goto err;
		}

	for (i = 0; i < g.nops && g.running; ++i, __wt_yield()) {
		if ((ret = session->begin_transaction(session, NULL)) != 0) {
			(void)log_print_err(
			    "real_worker:begin_transaction", ret, 1);
			goto err;
		}
		keyno = __wt_random(&rnd) % g.nkeys + 1;
		for (j = 0; j < g.ntables; j++) {
			if ((ret = worker_op(cursors[j], keyno, i)) != 0)
				break;
		}
		if (ret == 0) {
			if ((ret = session->commit_transaction(
			    session, NULL)) != 0) {
				(void)log_print_err(
				    "real_worker:commit_transaction", ret, 1);
				goto err;
			}
		} else if (ret == WT_ROLLBACK) {
			if ((ret = session->rollback_transaction(
			    session, NULL)) != 0) {
				(void)log_print_err(
				    "real_worker:rollback_transaction", ret, 1);
				goto err;
			    }
		} else {
			(void)log_print_err("worker op failed", ret, 1);
			goto err;
		}
	}

err:	if ((t_ret = session->close(session, NULL)) != 0 && ret == 0) {
		ret = t_ret;
		(void)log_print_err("session.close", ret, 1);
	}
	free(cursors);

	return (ret);
}
Exemple #21
0
int
main(int argc, char *argv[])
{
	static const struct {
		u_int workers;
		u_int uris;
		bool  cache_cursors;
	} runs[] = {
		{  1,   1, false},
		{  1,   1, true},
		{  8,   1, false},
		{  8,   1, true},
		{ 16,   1, false},
		{ 16,   1, true},
		{ 16,   WT_ELEMENTS(uri_list), false},
		{ 16,   WT_ELEMENTS(uri_list), true},
		{200, 100, false},
		{200, 100, true},
		{200, WT_ELEMENTS(uri_list), false},
		{200, WT_ELEMENTS(uri_list), true},
		{300, 100, false},
		{300, 100, true},
		{600, WT_ELEMENTS(uri_list), false},
		{600, WT_ELEMENTS(uri_list), true},
	};
	WT_RAND_STATE rnd;
	u_int i, n;
	int ch;

	/*
	 * Bypass this test for valgrind. It has a fairly low thread limit.
	 */
	if (testutil_is_flag_set("TESTUTIL_BYPASS_VALGRIND"))
		return (EXIT_SUCCESS);

	(void)testutil_set_progname(argv);
	__wt_random_init_seed(NULL, &rnd);

	while ((ch = __wt_getopt(argv[0], argc, argv, "v")) != EOF) {
		switch (ch) {
		case 'v':
			verbose = true;
			break;
		default:
			fprintf(stderr, "usage: %s [-v]\n", argv[0]);
			return (EXIT_FAILURE);
		}
	}

	(void)signal(SIGALRM, on_alarm);

	/* Each test in the table runs for a minute, run 5 tests at random. */
	for (i = 0; i < 5; ++i) {
		n = __wt_random(&rnd) % WT_ELEMENTS(runs);
		workers = runs[n].workers;
		uris = runs[n].uris;
		run(runs[n].cache_cursors);
	}

	uri_teardown();

	return (EXIT_SUCCESS);
}
Exemple #22
0
int
main(int argc, char *argv[])
{
	time_t start;
	int ch, i, onerun, reps, ret;
	const char *config, *home;

	config = NULL;

	if ((g.progname = strrchr(argv[0], DIR_DELIM)) == NULL)
		g.progname = argv[0];
	else
		++g.progname;

#if 0
	/* Configure the GNU malloc for debugging. */
	(void)setenv("MALLOC_CHECK_", "2", 1);
#endif
#if 0
	/* Configure the FreeBSD malloc for debugging. */
	(void)setenv("MALLOC_OPTIONS", "AJ", 1);
#endif

	/* Track progress unless we're re-directing output to a file. */
	g.track = isatty(1) ? 1 : 0;

	/* Set values from the command line. */
	home = NULL;
	onerun = 0;
	while ((ch = __wt_getopt(
	    g.progname, argc, argv, "1C:c:H:h:Llqrt:")) != EOF)
		switch (ch) {
		case '1':			/* One run */
			onerun = 1;
			break;
		case 'C':			/* wiredtiger_open config */
			g.config_open = __wt_optarg;
			break;
		case 'c':			/* Configuration from a file */
			config = __wt_optarg;
			break;
		case 'H':
			g.helium_mount = __wt_optarg;
			break;
		case 'h':
			home = __wt_optarg;
			break;
		case 'L':			/* Re-direct output to a log */
			/*
			 * The -l option is a superset of -L, ignore -L if we
			 * have already configured logging for operations.
			 */
			if (g.logging == 0)
				g.logging = LOG_FILE;
			break;
		case 'l':			/* Turn on operation logging */
			g.logging = LOG_OPS;
			break;
		case 'q':			/* Quiet */
			g.track = 0;
			break;
		case 'r':			/* Replay a run */
			g.replay = 1;
			break;
		default:
			usage();
		}
	argc -= __wt_optind;
	argv += __wt_optind;

	/*
	 * Initialize the global RNG. Start with the standard seeds, and then
	 * use seconds since the Epoch modulo a prime to run the RNG for some
	 * number of steps, so we don't start with the same values every time.
	 */
	__wt_random_init(&g.rnd);
	for (i = (int)time(NULL) % 10007; i > 0; --i)
		(void)__wt_random(&g.rnd);

	/* Set up paths. */
	path_setup(home);

	/* If it's a replay, use the home directory's CONFIG file. */
	if (g.replay) {
		if (config != NULL)
			die(EINVAL, "-c incompatible with -r");
		if (access(g.home_config, R_OK) != 0)
			die(ENOENT, "%s", g.home_config);
		config = g.home_config;
	}

	/*
	 * If we weren't given a configuration file, set values from "CONFIG",
	 * if it exists.
	 *
	 * Small hack to ignore any CONFIG file named ".", that just makes it
	 * possible to ignore any local CONFIG file, used when running checks.
	 */
	if (config == NULL && access("CONFIG", R_OK) == 0)
		config = "CONFIG";
	if (config != NULL && strcmp(config, ".") != 0)
		config_file(config);

	/*
	 * The rest of the arguments are individual configurations that modify
	 * the base configuration.
	 */
	for (; *argv != NULL; ++argv)
		config_single(*argv, 1);

	/*
	 * Multithreaded runs can be replayed: it's useful and we'll get the
	 * configuration correct.  Obviously the order of operations changes,
	 * warn the user.
	 */
	if (g.replay && !SINGLETHREADED)
		printf("Warning: replaying a threaded run\n");

	/*
	 * Single-threaded runs historically exited after a single replay, which
	 * makes sense when you're debugging, leave that semantic in place.
	 */
	if (g.replay && SINGLETHREADED)
		g.c_runs = 1;

	/*
	 * Let the command line -1 flag override runs configured from other
	 * sources.
	 */
	if (onerun)
		g.c_runs = 1;

	/*
	 * Initialize locks to single-thread named checkpoints and backups, last
	 * last-record updates, and failures.
	 */
	if ((ret = pthread_rwlock_init(&g.append_lock, NULL)) != 0)
		die(ret, "pthread_rwlock_init: append lock");
	if ((ret = pthread_rwlock_init(&g.backup_lock, NULL)) != 0)
		die(ret, "pthread_rwlock_init: backup lock");
	if ((ret = pthread_rwlock_init(&g.death_lock, NULL)) != 0)
		die(ret, "pthread_rwlock_init: death lock");

	printf("%s: process %" PRIdMAX "\n", g.progname, (intmax_t)getpid());
	while (++g.run_cnt <= g.c_runs || g.c_runs == 0 ) {
		startup();			/* Start a run */

		config_setup();			/* Run configuration */
		config_print(0);		/* Dump run configuration */
		key_len_setup();		/* Setup keys */

		start = time(NULL);
		track("starting up", 0ULL, NULL);

#ifdef HAVE_BERKELEY_DB
		if (SINGLETHREADED)
			bdb_open();		/* Initial file config */
#endif
		wts_open(g.home, 1, &g.wts_conn);
		wts_create();

		wts_load();			/* Load initial records */
		wts_verify("post-bulk verify");	/* Verify */

		/*
		 * If we're not doing any operations, scan the bulk-load, copy
		 * the statistics and we're done. Otherwise, loop reading and
		 * operations, with a verify after each set.
		 */
		if (g.c_timer == 0 && g.c_ops == 0) {
			wts_read_scan();		/* Read scan */
			wts_stats();			/* Statistics */
		} else
			for (reps = 1; reps <= FORMAT_OPERATION_REPS; ++reps) {
				wts_read_scan();	/* Read scan */

							/* Operations */
				wts_ops(reps == FORMAT_OPERATION_REPS);

				/*
				 * Copy out the run's statistics after the last
				 * set of operations.
				 *
				 * XXX
				 * Verify closes the underlying handle and
				 * discards the statistics, read them first.
				 */
				if (reps == FORMAT_OPERATION_REPS)
					wts_stats();

							/* Verify */
				wts_verify("post-ops verify");
			}

		track("shutting down", 0ULL, NULL);
#ifdef HAVE_BERKELEY_DB
		if (SINGLETHREADED)
			bdb_close();
#endif
		wts_close();

		/*
		 * If single-threaded, we can dump and compare the WiredTiger
		 * and Berkeley DB data sets.
		 */
		if (SINGLETHREADED)
			wts_dump("standard", 1);

		/*
		 * Salvage testing.
		 */
		wts_salvage();

		/* Overwrite the progress line with a completion line. */
		if (g.track)
			printf("\r%78s\r", " ");
		printf("%4d: %s, %s (%.0f seconds)\n",
		    g.run_cnt, g.c_data_source,
		    g.c_file_type, difftime(time(NULL), start));
		fflush(stdout);
	}

	/* Flush/close any logging information. */
	fclose_and_clear(&g.logfp);
	fclose_and_clear(&g.randfp);

	config_print(0);

	if ((ret = pthread_rwlock_destroy(&g.append_lock)) != 0)
		die(ret, "pthread_rwlock_destroy: append lock");
	if ((ret = pthread_rwlock_destroy(&g.backup_lock)) != 0)
		die(ret, "pthread_rwlock_destroy: backup lock");

	config_clear();

	return (EXIT_SUCCESS);
}
Exemple #23
0
uint32_t
workgen_random(workgen_random_state volatile * rnd_state)
{
	return (__wt_random(&rnd_state->state));
}
Exemple #24
0
/*
 * thread_run --
 *	Runner function for the worker threads.
 */
static WT_THREAD_RET
thread_run(void *arg)
{
	FILE *fp;
	WT_CURSOR *cur_coll, *cur_local, *cur_oplog, *cur_stable;
	WT_ITEM data;
	WT_RAND_STATE rnd;
	WT_SESSION *session;
	WT_THREAD_DATA *td;
	uint64_t i, stable_ts;
	int ret;
	char cbuf[MAX_VAL], lbuf[MAX_VAL], obuf[MAX_VAL];
	char kname[64], tscfg[64];

	__wt_random_init(&rnd);
	memset(cbuf, 0, sizeof(cbuf));
	memset(lbuf, 0, sizeof(lbuf));
	memset(obuf, 0, sizeof(obuf));
	memset(kname, 0, sizeof(kname));

	td = (WT_THREAD_DATA *)arg;
	/*
	 * Set up the separate file for checking.
	 */
	testutil_check(__wt_snprintf(cbuf, sizeof(cbuf), RECORDS_FILE, td->id));
	(void)unlink(cbuf);
	testutil_checksys((fp = fopen(cbuf, "w")) == NULL);
	/*
	 * Set to line buffering.  But that is advisory only.  We've seen
	 * cases where the result files end up with partial lines.
	 */
	__wt_stream_set_line_buffer(fp);
	if ((ret = td->conn->open_session(td->conn, NULL, NULL, &session)) != 0)
		testutil_die(ret, "WT_CONNECTION:open_session");
	/*
	 * Open a cursor to each table.
	 */
	if ((ret = session->open_cursor(session,
	    uri_collection, NULL, NULL, &cur_coll)) != 0)
		testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_collection);
	if ((ret = session->open_cursor(session,
	    uri_local, NULL, NULL, &cur_local)) != 0)
		testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_local);
	if ((ret = session->open_cursor(session,
	    uri_oplog, NULL, NULL, &cur_oplog)) != 0)
		testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_oplog);

	if ((ret = session->open_cursor(
	    session, stable_store, NULL, NULL, &cur_stable)) != 0)
		testutil_die(ret, "WT_SESSION.open_cursor: %s", stable_store);

	/*
	 * Write our portion of the key space until we're killed.
	 */
	printf("Thread %" PRIu32 " starts at %" PRIu64 "\n", td->id, td->start);
	for (i = td->start; ; ++i) {
		if (use_ts)
			stable_ts = global_ts++;
		else
			stable_ts = 0;
		testutil_check(__wt_snprintf(
		    kname, sizeof(kname), "%" PRIu64, i));

		testutil_check(session->begin_transaction(session, NULL));
		cur_coll->set_key(cur_coll, kname);
		cur_local->set_key(cur_local, kname);
		cur_oplog->set_key(cur_oplog, kname);
		/*
		 * Put an informative string into the value so that it
		 * can be viewed well in a binary dump.
		 */
		testutil_check(__wt_snprintf(cbuf, sizeof(cbuf),
		    "COLL: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
		    td->id, stable_ts, i));
		testutil_check(__wt_snprintf(lbuf, sizeof(lbuf),
		    "LOCAL: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
		    td->id, stable_ts, i));
		testutil_check(__wt_snprintf(obuf, sizeof(obuf),
		    "OPLOG: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
		    td->id, stable_ts, i));
		data.size = __wt_random(&rnd) % MAX_VAL;
		data.data = cbuf;
		cur_coll->set_value(cur_coll, &data);
		if ((ret = cur_coll->insert(cur_coll)) != 0)
			testutil_die(ret, "WT_CURSOR.insert");
		data.size = __wt_random(&rnd) % MAX_VAL;
		data.data = obuf;
		cur_oplog->set_value(cur_oplog, &data);
		if ((ret = cur_oplog->insert(cur_oplog)) != 0)
			testutil_die(ret, "WT_CURSOR.insert");
		if (use_ts) {
			testutil_check(__wt_snprintf(tscfg, sizeof(tscfg),
			    "commit_timestamp=%" PRIx64, stable_ts));
			testutil_check(
			    session->commit_transaction(session, tscfg));
		} else
			testutil_check(
			    session->commit_transaction(session, NULL));
		/*
		 * Insert into the local table outside the timestamp txn.
		 */
		data.size = __wt_random(&rnd) % MAX_VAL;
		data.data = lbuf;
		cur_local->set_value(cur_local, &data);
		if ((ret = cur_local->insert(cur_local)) != 0)
			testutil_die(ret, "WT_CURSOR.insert");

		/*
		 * Every N records we will record our stable timestamp into the
		 * stable table.  That will define our threshold where we
		 * expect to find records after recovery.
		 */
		if (i % STABLE_PERIOD == 0) {
			if (use_ts) {
				/*
				 * Set both the oldest and stable timestamp
				 * so that we don't need to maintain read
				 * availability at older timestamps.
				 */
				testutil_check(__wt_snprintf(
				    tscfg, sizeof(tscfg),
				    "oldest_timestamp=%" PRIx64
				    ",stable_timestamp=%" PRIx64,
				    stable_ts, stable_ts));
				testutil_check(
				    td->conn->set_timestamp(td->conn, tscfg));
			}
			cur_stable->set_key(cur_stable, td->id);
			cur_stable->set_value(cur_stable, stable_ts);
			testutil_check(cur_stable->insert(cur_stable));
		}
		/*
		 * Save the timestamp and key separately for checking later.
		 */
		if (fprintf(fp,
		    "%" PRIu64 " %" PRIu64 "\n", stable_ts, i) < 0)
			testutil_die(EIO, "fprintf");
	}
	/* NOTREACHED */
}
Exemple #25
0
/*
 * thread_run --
 *	Runner function for the worker threads.
 */
static WT_THREAD_RET
thread_run(void *arg)
{
	FILE *fp;
	WT_CURSOR *cur_coll, *cur_local, *cur_oplog;
	WT_ITEM data;
	WT_RAND_STATE rnd;
	WT_SESSION *prepared_session, *session;
	THREAD_DATA *td;
	uint64_t i, active_ts;
	char cbuf[MAX_VAL], lbuf[MAX_VAL], obuf[MAX_VAL];
	char kname[64], tscfg[64], uri[128];
	bool use_prep;

	__wt_random_init(&rnd);
	memset(cbuf, 0, sizeof(cbuf));
	memset(lbuf, 0, sizeof(lbuf));
	memset(obuf, 0, sizeof(obuf));
	memset(kname, 0, sizeof(kname));

	prepared_session = NULL;
	td = (THREAD_DATA *)arg;
	/*
	 * Set up the separate file for checking.
	 */
	testutil_check(__wt_snprintf(
	    cbuf, sizeof(cbuf), RECORDS_FILE, td->info));
	(void)unlink(cbuf);
	testutil_checksys((fp = fopen(cbuf, "w")) == NULL);
	/*
	 * Set to line buffering.  But that is advisory only.  We've seen
	 * cases where the result files end up with partial lines.
	 */
	__wt_stream_set_line_buffer(fp);

	/*
	 * Have 10% of the threads use prepared transactions if timestamps
	 * are in use. Thread numbers start at 0 so we're always guaranteed
	 * that at least one thread is using prepared transactions.
	 */
	use_prep = (use_ts && td->info % PREPARE_PCT == 0) ? true : false;

	/*
	 * For the prepared case we have two sessions so that the oplog session
	 * can have its own transaction in parallel with the collection session
	 * We need this because prepared transactions cannot have any operations
	 * that modify a table that is logged. But we also want to test mixed
	 * logged and not-logged transactions.
	 */
	testutil_check(td->conn->open_session(td->conn, NULL, NULL, &session));
	if (use_prep)
		testutil_check(td->conn->open_session(
		    td->conn, NULL, NULL, &prepared_session));
	/*
	 * Open a cursor to each table.
	 */
	testutil_check(__wt_snprintf(
	    uri, sizeof(uri), "%s:%s", table_pfx, uri_collection));
	if (use_prep)
		testutil_check(prepared_session->open_cursor(prepared_session,
		    uri, NULL, NULL, &cur_coll));
	else
		testutil_check(session->open_cursor(session,
		    uri, NULL, NULL, &cur_coll));
	testutil_check(__wt_snprintf(
	    uri, sizeof(uri), "%s:%s", table_pfx, uri_local));
	if (use_prep)
		testutil_check(prepared_session->open_cursor(prepared_session,
		    uri, NULL, NULL, &cur_local));
	else
		testutil_check(session->open_cursor(session,
		    uri, NULL, NULL, &cur_local));
	testutil_check(__wt_snprintf(
	    uri, sizeof(uri), "%s:%s", table_pfx, uri_oplog));
	testutil_check(session->open_cursor(session,
	    uri, NULL, NULL, &cur_oplog));

	/*
	 * Write our portion of the key space until we're killed.
	 */
	printf("Thread %" PRIu32 " starts at %" PRIu64 "\n",
	    td->info, td->start);
	active_ts = 0;
	for (i = td->start;; ++i) {
		testutil_check(__wt_snprintf(
		    kname, sizeof(kname), "%" PRIu64, i));

		testutil_check(session->begin_transaction(session, NULL));
		if (use_prep)
			testutil_check(prepared_session->begin_transaction(
			    prepared_session, NULL));

		if (use_ts) {
			testutil_check(pthread_rwlock_rdlock(&ts_lock));
			active_ts = __wt_atomic_addv64(&global_ts, 1);
			testutil_check(__wt_snprintf(tscfg,
			    sizeof(tscfg), "commit_timestamp=%" PRIx64,
			    active_ts));
			/*
			 * Set the transaction's timestamp now before performing
			 * the operation. If we are using prepared transactions,
			 * set the timestamp for the session used for oplog. The
			 * collection session in that case would continue to use
			 * this timestamp.
			 */
			testutil_check(session->timestamp_transaction(
			    session, tscfg));
			testutil_check(pthread_rwlock_unlock(&ts_lock));
		}

		cur_coll->set_key(cur_coll, kname);
		cur_local->set_key(cur_local, kname);
		cur_oplog->set_key(cur_oplog, kname);
		/*
		 * Put an informative string into the value so that it
		 * can be viewed well in a binary dump.
		 */
		testutil_check(__wt_snprintf(cbuf, sizeof(cbuf),
		    "COLL: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
		    td->info, active_ts, i));
		testutil_check(__wt_snprintf(lbuf, sizeof(lbuf),
		    "LOCAL: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
		    td->info, active_ts, i));
		testutil_check(__wt_snprintf(obuf, sizeof(obuf),
		    "OPLOG: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
		    td->info, active_ts, i));
		data.size = __wt_random(&rnd) % MAX_VAL;
		data.data = cbuf;
		cur_coll->set_value(cur_coll, &data);
		testutil_check(cur_coll->insert(cur_coll));
		data.size = __wt_random(&rnd) % MAX_VAL;
		data.data = obuf;
		cur_oplog->set_value(cur_oplog, &data);
		testutil_check(cur_oplog->insert(cur_oplog));
		if (use_prep) {
			/*
			 * Run with prepare every once in a while. And also
			 * yield after prepare sometimes too. This is only done
			 * on the collection session.
			 */
			if (i % PREPARE_FREQ == 0) {
				testutil_check(__wt_snprintf(tscfg,
				    sizeof(tscfg), "prepare_timestamp=%"
				    PRIx64, active_ts));
				testutil_check(
				    prepared_session->prepare_transaction(
				    prepared_session, tscfg));
				if (i % PREPARE_YIELD == 0)
					__wt_yield();
				testutil_check(
				    __wt_snprintf(tscfg, sizeof(tscfg),
				    "commit_timestamp=%" PRIx64
				    ",durable_timestamp=%" PRIx64,
				    active_ts, active_ts));
			} else
				testutil_check(
				    __wt_snprintf(tscfg, sizeof(tscfg),
				    "commit_timestamp=%" PRIx64, active_ts));

			testutil_check(
			    prepared_session->commit_transaction(
			    prepared_session, tscfg));
		}
		testutil_check(
		    session->commit_transaction(session, NULL));
		/*
		 * Insert into the local table outside the timestamp txn.
		 */
		data.size = __wt_random(&rnd) % MAX_VAL;
		data.data = lbuf;
		cur_local->set_value(cur_local, &data);
		testutil_check(cur_local->insert(cur_local));

		/*
		 * Save the timestamp and key separately for checking later.
		 */
		if (fprintf(fp,
		    "%" PRIu64 " %" PRIu64 "\n", active_ts, i) < 0)
			testutil_die(EIO, "fprintf");
	}
	/* NOTREACHED */
}
Exemple #26
0
/*
 * __wt_log_slot_join --
 *	Join a consolidated logging slot. Callers should be prepared to deal
 *	with a ENOMEM return - which indicates no slots could accommodate
 *	the log record.
 */
int
__wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize,
    uint32_t flags, WT_MYSLOT *myslotp)
{
	WT_CONNECTION_IMPL *conn;
	WT_LOG *log;
	WT_LOGSLOT *slot;
	int64_t new_state, old_state;
	uint32_t allocated_slot, slot_attempts;

	conn = S2C(session);
	log = conn->log;
	slot_attempts = 0;

	if (mysize >= (uint64_t)log->slot_buf_size) {
		WT_STAT_FAST_CONN_INCR(session, log_slot_toobig);
		return (ENOMEM);
	}
find_slot:
#if WT_SLOT_ACTIVE == 1
	allocated_slot = 0;
#else
	allocated_slot = __wt_random(&session->rnd) % WT_SLOT_ACTIVE;
#endif
	/*
	 * Get the selected slot.  Use a barrier to prevent the compiler from
	 * caching this read.
	 */
	WT_BARRIER();
	slot = log->slot_array[allocated_slot];
join_slot:
	/*
	 * Read the current slot state.  Use a barrier to prevent the compiler
	 * from caching this read.
	 */
	WT_BARRIER();
	old_state = slot->slot_state;
	/*
	 * WT_LOG_SLOT_READY and higher means the slot is available for
	 * joining.  Any other state means it is in use and transitioning
	 * from the active array.
	 */
	if (old_state < WT_LOG_SLOT_READY) {
		WT_STAT_FAST_CONN_INCR(session, log_slot_transitions);
		goto find_slot;
	}
	/*
	 * Add in our size to the state and then atomically swap that
	 * into place if it is still the same value.
	 */
	new_state = old_state + (int64_t)mysize;
	if (new_state < old_state) {
		/* Our size doesn't fit here. */
		WT_STAT_FAST_CONN_INCR(session, log_slot_toobig);
		goto find_slot;
	}
	/*
	 * If the slot buffer isn't big enough to hold this update, try
	 * to find another slot.
	 */
	if (new_state > (int64_t)slot->slot_buf.memsize) {
		if (++slot_attempts > 5) {
			WT_STAT_FAST_CONN_INCR(session, log_slot_toosmall);
			return (ENOMEM);
		}
		goto find_slot;
	}
	/*
	 * We lost a race to add our size into this slot.  Check the state
	 * and try again.
	 */
	if (!WT_ATOMIC_CAS8(slot->slot_state, old_state, new_state)) {
		WT_STAT_FAST_CONN_INCR(session, log_slot_races);
		goto join_slot;
	}
	WT_ASSERT(session, myslotp != NULL);
	/*
	 * We joined this slot.  Fill in our information to return to
	 * the caller.
	 */
	WT_STAT_FAST_CONN_INCR(session, log_slot_joins);
	if (LF_ISSET(WT_LOG_DSYNC | WT_LOG_FSYNC))
		F_SET(slot, WT_SLOT_SYNC_DIR);
	if (LF_ISSET(WT_LOG_FSYNC))
		F_SET(slot, WT_SLOT_SYNC);
	myslotp->slot = slot;
	myslotp->offset = (wt_off_t)old_state - WT_LOG_SLOT_READY;
	return (0);
}
Exemple #27
0
int
main(void)
{
	WT_RAND_STATE rnd;
	size_t len;
	uint32_t hw, sw;
	u_int i, j;
	uint8_t *data;

	/* Allocate aligned memory for the data. */
	data = dcalloc(DATASIZE, sizeof(uint8_t));

	/* Initialize the RNG. */
	testutil_check(__wt_random_init_seed(NULL, &rnd));

	/* Initialize the WiredTiger library checksum functions. */
	__wt_cksum_init();

	/*
	 * Some simple known checksums.
	 */
	len = 1;
	hw = __wt_cksum(data, len);
	check(hw, (uint32_t)0x527d5351, len, "nul x1: hardware");
	sw = cksum_sw(data, len);
	check(sw, (uint32_t)0x527d5351, len, "nul x1: software");

	len = 2;
	hw = __wt_cksum(data, len);
	check(hw, (uint32_t)0xf16177d2, len, "nul x2: hardware");
	sw = cksum_sw(data, len);
	check(sw, (uint32_t)0xf16177d2, len, "nul x2: software");

	len = 3;
	hw = __wt_cksum(data, len);
	check(hw, (uint32_t)0x6064a37a, len, "nul x3: hardware");
	sw = cksum_sw(data, len);
	check(sw, (uint32_t)0x6064a37a, len, "nul x3: software");

	len = 4;
	hw = __wt_cksum(data, len);
	check(hw, (uint32_t)0x48674bc7, len, "nul x4: hardware");
	sw = cksum_sw(data, len);
	check(sw, (uint32_t)0x48674bc7, len, "nul x4: software");

	len = strlen("123456789");
	memcpy(data, "123456789", len);
	hw = __wt_cksum(data, len);
	check(hw, (uint32_t)0xe3069283, len, "known string #1: hardware");
	sw = cksum_sw(data, len);
	check(sw, (uint32_t)0xe3069283, len, "known string #1: software");

	len = strlen("The quick brown fox jumps over the lazy dog");
	memcpy(data, "The quick brown fox jumps over the lazy dog", len);
	hw = __wt_cksum(data, len);
	check(hw, (uint32_t)0x22620404, len, "known string #2: hardware");
	sw = cksum_sw(data, len);
	check(sw, (uint32_t)0x22620404, len, "known string #2: software");

	/*
	 * Checksums of power-of-two data chunks.
	 */
	for (i = 0, len = 512; i < 1000; ++i) {
		for (j = 0; j < len; ++j)
			data[j] = __wt_random(&rnd) & 0xff;
		hw = __wt_cksum(data, len);
		sw = cksum_sw(data, len);
		check(hw, sw, len, "random power-of-two");

		len *= 2;
		if (len > DATASIZE)
			len = 512;
	}

	/*
	 * Checksums of random data chunks.
	 */
	for (i = 0; i < 1000; ++i) {
		len = __wt_random(&rnd) % DATASIZE;
		for (j = 0; j < len; ++j)
			data[j] = __wt_random(&rnd) & 0xff;
		hw = __wt_cksum(data, len);
		sw = cksum_sw(data, len);
		check(hw, sw, len, "random");
	}

	free(data);
	return (EXIT_SUCCESS);
}
Exemple #28
0
/*
 * __wt_lsm_get_chunk_to_flush --
 *	Find and pin a chunk in the LSM tree that is likely to need flushing.
 */
int
__wt_lsm_get_chunk_to_flush(WT_SESSION_IMPL *session,
    WT_LSM_TREE *lsm_tree, bool force, WT_LSM_CHUNK **chunkp)
{
	WT_DECL_RET;
	WT_LSM_CHUNK *chunk, *evict_chunk, *flush_chunk;
	u_int i;

	*chunkp = NULL;
	chunk = evict_chunk = flush_chunk = NULL;

	WT_ASSERT(session, lsm_tree->queue_ref > 0);
	WT_RET(__wt_lsm_tree_readlock(session, lsm_tree));
	if (!F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE) || lsm_tree->nchunks == 0)
		return (__wt_lsm_tree_readunlock(session, lsm_tree));

	/* Search for a chunk to evict and/or a chunk to flush. */
	for (i = 0; i < lsm_tree->nchunks; i++) {
		chunk = lsm_tree->chunk[i];
		if (F_ISSET(chunk, WT_LSM_CHUNK_ONDISK)) {
			/*
			 * Normally we don't want to force out the last chunk.
			 * But if we're doing a forced flush on behalf of a
			 * compact, then we want to include the final chunk.
			 */
			if (evict_chunk == NULL &&
			    !chunk->evicted &&
			    !F_ISSET(chunk, WT_LSM_CHUNK_STABLE))
				evict_chunk = chunk;
		} else if (flush_chunk == NULL &&
		    chunk->switch_txn != 0 &&
		    (force || i < lsm_tree->nchunks - 1))
			flush_chunk = chunk;
	}

	/*
	 * Don't be overly zealous about pushing old chunks from cache.
	 * Attempting too many drops can interfere with checkpoints.
	 *
	 * If retrying a discard push an additional work unit so there are
	 * enough to trigger checkpoints.
	 */
	if (evict_chunk != NULL && flush_chunk != NULL) {
		chunk = (__wt_random(&session->rnd) & 1) ?
		    evict_chunk : flush_chunk;
		WT_ERR(__wt_lsm_manager_push_entry(
		    session, WT_LSM_WORK_FLUSH, 0, lsm_tree));
	} else
		chunk = (evict_chunk != NULL) ? evict_chunk : flush_chunk;

	if (chunk != NULL) {
		WT_ERR(__wt_verbose(session, WT_VERB_LSM,
		    "Flush%s: return chunk %u of %u: %s",
		    force ? " w/ force" : "",
		    i, lsm_tree->nchunks, chunk->uri));

		(void)__wt_atomic_add32(&chunk->refcnt, 1);
	}

err:	WT_RET(__wt_lsm_tree_readunlock(session, lsm_tree));

	*chunkp = chunk;
	return (ret);
}
Exemple #29
0
static void *
thread_insert(void *arg)
{
	TEST_OPTS *opts;
	THREAD_ARGS *threadargs;
	WT_CURSOR *maincur;
	WT_RAND_STATE rnd;
	WT_SESSION *session;
	double elapsed;
	time_t prevtime, curtime; /* 1 second resolution is okay */
	int bal, i, flag, key, post;
	const char *extra = S1024;

	threadargs = (THREAD_ARGS *)arg;
	opts = threadargs->testopts;
	testutil_check(__wt_random_init_seed(NULL, &rnd));
	(void)time(&prevtime);

	testutil_check(opts->conn->open_session(
	    opts->conn, NULL, NULL, &session));

	testutil_check(session->open_cursor(session, opts->uri, NULL, NULL,
	    &maincur));

	for (i = 0; i < N_INSERT; i++) {
		/*
		 * Insert threads may stomp on each other's records;
		 * that's okay.
		 */
		key = (int)(__wt_random(&rnd) % N_RECORDS);
		testutil_check(session->begin_transaction(session, NULL));
		maincur->set_key(maincur, key);
		if (__wt_random(&rnd) % 2 == 0)
			post = 54321;
		else
			post = i % 100000;
		if (__wt_random(&rnd) % 2 == 0) {
			bal = -100;
			flag = 1;
		} else {
			bal = 100 * (i + 1);
			flag = 0;
		}
		maincur->set_value(maincur, post, bal, extra, flag, key);
		testutil_check(maincur->insert(maincur));
		testutil_check(maincur->reset(maincur));
		testutil_check(session->commit_transaction(session, NULL));
		if (i % 1000 == 0 && i != 0) {
			if (i % 10000 == 0)
				fprintf(stderr, "*");
			else
				fprintf(stderr, ".");
			(void)time(&curtime);
			if ((elapsed = difftime(curtime, prevtime)) > 5.0) {
				fprintf(stderr, "\n"
				    "GAP: %.0f secs after %d inserts\n",
				    elapsed, i);
				threadargs->nfail++;
			}
			prevtime = curtime;
		}
	}
	testutil_check(maincur->close(maincur));
	testutil_check(session->close(session, NULL));
	return (NULL);
}