Example #1
0
static void *
ops(void *arg)
{
	TINFO *tinfo;
	WT_CONNECTION *conn;
	WT_CURSOR *cursor, *cursor_insert;
	WT_SESSION *session;
	WT_ITEM key, value;
	uint64_t cnt, keyno, sync_op, thread_ops;
	uint32_t op;
	uint8_t *keybuf, *valbuf;
	u_int np;
	int dir, insert, notfound, ret, sync_drop;
	char sync_name[64];

	conn = g.wts_conn;

	tinfo = arg;

	/* Set up the default key and value buffers. */
	memset(&key, 0, sizeof(key));
	key_gen_setup(&keybuf);
	memset(&value, 0, sizeof(value));
	val_gen_setup(&valbuf);

	/* Open a session. */
	if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
		die(ret, "connection.open_session");

	/*
	 * Open two cursors: one configured for overwriting and one configured
	 * for append if we're dealing with a column-store.
	 *
	 * The reason is when testing with existing records, we don't track if
	 * a record was deleted or not, which means we must use cursor->insert
	 * with overwriting configured.  But, in column-store files where we're
	 * testing with new, appended records, we don't want to have to specify
	 * the record number, which requires an append configuration.
	 */
	cursor = cursor_insert = NULL;
	if ((ret = session->open_cursor(session,
	    WT_TABLENAME, NULL, "overwrite", &cursor)) != 0)
		die(ret, "session.open_cursor");
	if ((g.c_file_type == FIX || g.c_file_type == VAR) &&
	    (ret = session->open_cursor(session,
	    WT_TABLENAME, NULL, "append", &cursor_insert)) != 0)
		die(ret, "session.open_cursor");

	/* Each thread does its share of the total operations. */
	thread_ops = g.c_ops / g.c_threads;

	/* Pick an operation where we'll do a sync and create the name. */
	sync_drop = 0;
	sync_op = MMRAND(1, thread_ops);
	snprintf(sync_name, sizeof(sync_name), "snapshot=thread-%d", tinfo->id);

	for (cnt = 0; cnt < thread_ops; ++cnt) {
		if (SINGLETHREADED && cnt % 100 == 0)
			track("read/write ops", 0ULL, tinfo);

		if (cnt == sync_op) {
			if (sync_drop && (int)MMRAND(1, 4) == 1) {
				if ((ret = session->drop(
				    session, WT_TABLENAME, sync_name)) != 0)
					die(ret, "session.drop: %s: %s",
					    WT_TABLENAME, sync_name);
				sync_drop = 0;
			} else {
				if ((ret = session->checkpoint(
				    session, sync_name)) != 0)
					die(ret, "session.checkpoint: %s",
					    sync_name);
				sync_drop = 1;
			}

			/*
			 * Pick the next sync operation, try for roughly five
			 * snapshot operations per thread run.
			 */
			sync_op += MMRAND(1, thread_ops) / 5;
		}

		insert = notfound = 0;

		keyno = MMRAND(1, g.rows);
		key.data = keybuf;
		value.data = valbuf;

		/*
		 * Perform some number of operations: the percentage of deletes,
		 * inserts and writes are specified, reads are the rest.  The
		 * percentages don't have to add up to 100, a high percentage
		 * of deletes will mean fewer inserts and writes.  A read
		 * operation always follows a modification to confirm it worked.
		 */
		op = (uint32_t)(wts_rand() % 100);
		if (op < g.c_delete_pct) {
			++tinfo->remove;
			switch (g.c_file_type) {
			case ROW:
				/*
				 * If deleting a non-existent record, the cursor
				 * won't be positioned, and so can't do a next.
				 */
				row_remove(cursor, &key, keyno, &notfound);
				break;
			case FIX:
			case VAR:
				col_remove(cursor, &key, keyno, &notfound);
				break;
			}
		} else if (op < g.c_delete_pct + g.c_insert_pct) {
			++tinfo->insert;
			switch (g.c_file_type) {
			case ROW:
				row_update(cursor, &key, &value, keyno, 1);
				break;
			case FIX:
			case VAR:
				/*
				 * Reset the standard cursor so it doesn't keep
				 * pages pinned.
				 */
				if ((ret = cursor->reset(cursor)) != 0)
					die(ret, "cursor.reset");
				col_insert(cursor_insert, &key, &value, &keyno);
				insert = 1;
				break;
			}
		} else if (
		    op < g.c_delete_pct + g.c_insert_pct + g.c_write_pct) {
			++tinfo->update;
			switch (g.c_file_type) {
			case ROW:
				row_update(cursor, &key, &value, keyno, 0);
				break;
			case FIX:
			case VAR:
				col_update(cursor, &key, &value, keyno);
				break;
			}
		} else {
			++tinfo->search;
			read_row(cursor, &key, keyno);
			continue;
		}

		/*
		 * If we did any operation, we've set the cursor, do a small
		 * number of next/prev cursor operations in a random direction.
		 */
		dir = (int)MMRAND(0, 1);
		for (np = 0; np < MMRAND(1, 8); ++np) {
			if (notfound)
				break;
			nextprev(
			    insert ? cursor_insert : cursor, dir, &notfound);
		}

		if (insert && (ret = cursor_insert->reset(cursor_insert)) != 0)
			die(ret, "cursor.reset");

		/* Read the value we modified to confirm the operation. */
		read_row(cursor, &key, keyno);
	}

	if ((ret = session->close(session, NULL)) != 0)
		die(ret, "session.close");

	free(keybuf);
	free(valbuf);

	tinfo->state = TINFO_COMPLETE;
	return (NULL);
}
Example #2
0
static void *
ops(void *arg)
{
	TINFO *tinfo;
	WT_CONNECTION *conn;
	WT_CURSOR *cursor, *cursor_insert;
	WT_SESSION *session;
	WT_ITEM key, value;
	uint64_t keyno, ckpt_op, session_op;
	uint32_t op;
	uint8_t *keybuf, *valbuf;
	u_int np;
	int ckpt_available, dir, insert, intxn, notfound, readonly, ret;
	char *ckpt_config, ckpt_name[64];

	tinfo = arg;

	/* Initialize the per-thread random number generator. */
	__wt_random_init(&tinfo->rnd);

	conn = g.wts_conn;
	keybuf = valbuf = NULL;
	readonly = 0;			/* -Wconditional-uninitialized */

	/* Set up the default key and value buffers. */
	key_gen_setup(&keybuf);
	val_gen_setup(&tinfo->rnd, &valbuf);

	/* Set the first operation where we'll create sessions and cursors. */
	session_op = 0;
	session = NULL;
	cursor = cursor_insert = NULL;

	/* Set the first operation where we'll perform checkpoint operations. */
	ckpt_op = g.c_checkpoints ? mmrand(&tinfo->rnd, 100, 10000) : 0;
	ckpt_available = 0;

	for (intxn = 0; !tinfo->quit; ++tinfo->ops) {
		/*
		 * We can't checkpoint or swap sessions/cursors while in a
		 * transaction, resolve any running transaction.
		 */
		if (intxn &&
		    (tinfo->ops == ckpt_op || tinfo->ops == session_op)) {
			if ((ret = session->commit_transaction(
			    session, NULL)) != 0)
				die(ret, "session.commit_transaction");
			++tinfo->commit;
			intxn = 0;
		}

		/* Open up a new session and cursors. */
		if (tinfo->ops == session_op ||
		    session == NULL || cursor == NULL) {
			if (session != NULL &&
			    (ret = session->close(session, NULL)) != 0)
				die(ret, "session.close");

			if ((ret = conn->open_session(conn, NULL,
			    ops_session_config(&tinfo->rnd), &session)) != 0)
				die(ret, "connection.open_session");

			/*
			 * 10% of the time, perform some read-only operations
			 * from a checkpoint.
			 *
			 * Skip that if we single-threaded and doing checks
			 * against a Berkeley DB database, because that won't
			 * work because the Berkeley DB database records won't
			 * match the checkpoint.  Also skip if we are using
			 * LSM, because it doesn't support reads from
			 * checkpoints.
			 */
			if (!SINGLETHREADED && !DATASOURCE("lsm") &&
			    ckpt_available && mmrand(&tinfo->rnd, 1, 10) == 1) {
				if ((ret = session->open_cursor(session,
				    g.uri, NULL, ckpt_name, &cursor)) != 0)
					die(ret, "session.open_cursor");

				/* Pick the next session/cursor close/open. */
				session_op += 250;

				/* Checkpoints are read-only. */
				readonly = 1;
			} else {
				/*
				 * Open two cursors: one for overwriting and one
				 * for append (if it's a column-store).
				 *
				 * The reason is when testing with existing
				 * records, we don't track if a record was
				 * deleted or not, which means we must use
				 * cursor->insert with overwriting configured.
				 * But, in column-store files where we're
				 * testing with new, appended records, we don't
				 * want to have to specify the record number,
				 * which requires an append configuration.
				 */
				if ((ret = session->open_cursor(session, g.uri,
				    NULL, "overwrite", &cursor)) != 0)
					die(ret, "session.open_cursor");
				if ((g.type == FIX || g.type == VAR) &&
				    (ret = session->open_cursor(session, g.uri,
				    NULL, "append", &cursor_insert)) != 0)
					die(ret, "session.open_cursor");

				/* Pick the next session/cursor close/open. */
				session_op += mmrand(&tinfo->rnd, 100, 5000);

				/* Updates supported. */
				readonly = 0;
			}
		}

		/* Checkpoint the database. */
		if (tinfo->ops == ckpt_op && g.c_checkpoints) {
			/*
			 * LSM and data-sources don't support named checkpoints,
			 * and we can't drop a named checkpoint while there's a
			 * cursor open on it, otherwise 20% of the time name the
			 * checkpoint.
			 */
			if (DATASOURCE("helium") || DATASOURCE("kvsbdb") ||
			    DATASOURCE("lsm") ||
			    readonly || mmrand(&tinfo->rnd, 1, 5) == 1)
				ckpt_config = NULL;
			else {
				(void)snprintf(ckpt_name, sizeof(ckpt_name),
				    "name=thread-%d", tinfo->id);
				ckpt_config = ckpt_name;
			}

			/* Named checkpoints lock out backups */
			if (ckpt_config != NULL &&
			    (ret = pthread_rwlock_wrlock(&g.backup_lock)) != 0)
				die(ret,
				    "pthread_rwlock_wrlock: backup lock");

			if ((ret =
			    session->checkpoint(session, ckpt_config)) != 0)
				die(ret, "session.checkpoint%s%s",
				    ckpt_config == NULL ? "" : ": ",
				    ckpt_config == NULL ? "" : ckpt_config);

			if (ckpt_config != NULL &&
			    (ret = pthread_rwlock_unlock(&g.backup_lock)) != 0)
				die(ret,
				    "pthread_rwlock_wrlock: backup lock");

			/* Rephrase the checkpoint name for cursor open. */
			if (ckpt_config == NULL)
				strcpy(ckpt_name,
				    "checkpoint=WiredTigerCheckpoint");
			else
				(void)snprintf(ckpt_name, sizeof(ckpt_name),
				    "checkpoint=thread-%d", tinfo->id);
			ckpt_available = 1;

			/* Pick the next checkpoint operation. */
			ckpt_op += mmrand(&tinfo->rnd, 5000, 20000);
		}

		/*
		 * If we're not single-threaded and we're not in a transaction,
		 * start a transaction 20% of the time.
		 */
		if (!SINGLETHREADED &&
		    !intxn && mmrand(&tinfo->rnd, 1, 10) >= 8) {
			if ((ret =
			    session->begin_transaction(session, NULL)) != 0)
				die(ret, "session.begin_transaction");
			intxn = 1;
		}

		insert = notfound = 0;

		keyno = mmrand(&tinfo->rnd, 1, (u_int)g.rows);
		key.data = keybuf;
		value.data = valbuf;

		/*
		 * Perform some number of operations: the percentage of deletes,
		 * inserts and writes are specified, reads are the rest.  The
		 * percentages don't have to add up to 100, a high percentage
		 * of deletes will mean fewer inserts and writes.  Modifications
		 * are always followed by a read to confirm it worked.
		 */
		op = readonly ? UINT32_MAX : mmrand(&tinfo->rnd, 1, 100);
		if (op < g.c_delete_pct) {
			++tinfo->remove;
			switch (g.type) {
			case ROW:
				/*
				 * If deleting a non-existent record, the cursor
				 * won't be positioned, and so can't do a next.
				 */
				if (row_remove(cursor, &key, keyno, &notfound))
					goto deadlock;
				break;
			case FIX:
			case VAR:
				if (col_remove(cursor, &key, keyno, &notfound))
					goto deadlock;
				break;
			}
		} else if (op < g.c_delete_pct + g.c_insert_pct) {
			++tinfo->insert;
			switch (g.type) {
			case ROW:
				if (row_insert(
				    tinfo, cursor, &key, &value, keyno))
					goto deadlock;
				insert = 1;
				break;
			case FIX:
			case VAR:
				/*
				 * We can only append so many new records, if
				 * we've reached that limit, update a record
				 * instead of doing an insert.
				 */
				if (g.append_cnt >= g.append_max)
					goto skip_insert;

				/* Insert, then reset the insert cursor. */
				if (col_insert(tinfo,
				    cursor_insert, &key, &value, &keyno))
					goto deadlock;
				if ((ret =
				    cursor_insert->reset(cursor_insert)) != 0)
					die(ret, "cursor.reset");

				insert = 1;
				break;
			}
		} else if (
		    op < g.c_delete_pct + g.c_insert_pct + g.c_write_pct) {
			++tinfo->update;
			switch (g.type) {
			case ROW:
				if (row_update(
				    tinfo, cursor, &key, &value, keyno))
					goto deadlock;
				break;
			case FIX:
			case VAR:
skip_insert:			if (col_update(tinfo,
				    cursor, &key, &value, keyno))
					goto deadlock;
				break;
			}
		} else {
			++tinfo->search;
			if (read_row(cursor, &key, keyno))
				if (intxn)
					goto deadlock;
			continue;
		}

		/*
		 * The cursor is positioned if we did any operation other than
		 * insert, do a small number of next/prev cursor operations in
		 * a random direction.
		 */
		if (!insert) {
			dir = (int)mmrand(&tinfo->rnd, 0, 1);
			for (np = 0; np < mmrand(&tinfo->rnd, 1, 8); ++np) {
				if (notfound)
					break;
				if (nextprev(cursor, dir, &notfound))
					goto deadlock;
			}
		}

		/* Read to confirm the operation. */
		++tinfo->search;
		if (read_row(cursor, &key, keyno))
			goto deadlock;

		/* Reset the cursor: there is no reason to keep pages pinned. */
		if ((ret = cursor->reset(cursor)) != 0)
			die(ret, "cursor.reset");

		/*
		 * If we're in the transaction, commit 40% of the time and
		 * rollback 10% of the time.
		 */
		if (intxn)
			switch (mmrand(&tinfo->rnd, 1, 10)) {
			case 1: case 2: case 3: case 4:		/* 40% */
				if ((ret = session->commit_transaction(
				    session, NULL)) != 0)
					die(ret, "session.commit_transaction");
				++tinfo->commit;
				intxn = 0;
				break;
			case 5:					/* 10% */
				if (0) {
deadlock:				++tinfo->deadlock;
				}
				if ((ret = session->rollback_transaction(
				    session, NULL)) != 0)
					die(ret,
					    "session.rollback_transaction");
				++tinfo->rollback;
				intxn = 0;
				break;
			default:
				break;
			}
	}

	if (session != NULL && (ret = session->close(session, NULL)) != 0)
		die(ret, "session.close");

	free(keybuf);
	free(valbuf);

	tinfo->state = TINFO_COMPLETE;
	return (NULL);
}
Example #3
0
static void *
ops(void *arg)
{
	TINFO *tinfo;
	WT_CONNECTION *conn;
	WT_CURSOR *cursor, *cursor_insert;
	WT_SESSION *session;
	WT_ITEM key, value;
	uint64_t cnt, keyno, ckpt_op, session_op, thread_ops;
	uint32_t op;
	uint8_t *keybuf, *valbuf;
	u_int np;
	int dir, insert, intxn, notfound, ret;
	char *ckpt_config, config[64];

	tinfo = arg;

	conn = g.wts_conn;
	keybuf = valbuf = NULL;

	/* Set up the default key and value buffers. */
	key_gen_setup(&keybuf);
	val_gen_setup(&valbuf);

	/*
	 * Each thread does its share of the total operations, and make sure
	 * that it's not 0 (testing runs: threads might be larger than ops).
	 */
	thread_ops = 100 + g.c_ops / g.c_threads;

	/*
	 * Select the first operation where we'll create sessions and cursors,
	 * perform checkpoint operations.
	 */
	ckpt_op = MMRAND(1, thread_ops);
	session_op = 0;

	session = NULL;
	cursor = cursor_insert = NULL;
	for (intxn = 0, cnt = 0; cnt < thread_ops; ++cnt) {
		if (SINGLETHREADED && cnt % 100 == 0)
			track("ops", 0ULL, tinfo);

		/*
		 * We can't checkpoint or swap sessions/cursors while in a
		 * transaction, resolve any running transaction.  Otherwise,
		 * reset the cursor: we may block waiting for a lock and there
		 * is no reason to keep pages pinned.
		 */
		if (cnt == ckpt_op || cnt == session_op) {
			if (intxn) {
				if ((ret = session->commit_transaction(
				    session, NULL)) != 0)
					die(ret, "session.commit_transaction");
				++tinfo->commit;
				intxn = 0;
			}
			else if (cursor != NULL &&
			    (ret = cursor->reset(cursor)) != 0)
				die(ret, "cursor.reset");
		}

		/* Open up a new session and cursors. */
		if (cnt == session_op || session == NULL || cursor == NULL) {
			if (session != NULL &&
			    (ret = session->close(session, NULL)) != 0)
				die(ret, "session.close");

			if ((ret = conn->open_session(
			    conn, NULL, NULL, &session)) != 0)
				die(ret, "connection.open_session");

			/*
			 * Open two cursors: one configured for overwriting and
			 * one configured for append if we're dealing with a
			 * column-store.
			 *
			 * The reason is when testing with existing records, we
			 * don't track if a record was deleted or not, which
			 * means we must use cursor->insert with overwriting
			 * configured.  But, in column-store files where we're
			 * testing with new, appended records, we don't want to
			 * have to specify the record number, which requires an
			 * append configuration.
			 */
			if ((ret = session->open_cursor(session,
			    g.uri, NULL, "overwrite", &cursor)) != 0)
				die(ret, "session.open_cursor");
			if ((g.type == FIX || g.type == VAR) &&
			    (ret = session->open_cursor(session,
			    g.uri, NULL, "append", &cursor_insert)) != 0)
				die(ret, "session.open_cursor");

			/* Pick the next session/cursor close/open. */
			session_op += SINGLETHREADED ?
			    MMRAND(1, thread_ops) : 100 * MMRAND(1, 50);
		}

		/* Checkpoint the database. */
		if (cnt == ckpt_op) {
			/*
			 * LSM and data-sources don't support named checkpoints,
			 * else 25% of the time we name the checkpoint.
			 */
			if (DATASOURCE("lsm") || DATASOURCE("kvsbdb") ||
			    DATASOURCE("memrata") || MMRAND(1, 4) == 1)
				ckpt_config = NULL;
			else {
				(void)snprintf(config, sizeof(config),
				    "name=thread-%d", tinfo->id);
				ckpt_config = config;
			}

			/* Named checkpoints lock out hot backups */
			if (ckpt_config != NULL &&
			    (ret = pthread_rwlock_wrlock(&g.backup_lock)) != 0)
				die(ret,
				    "pthread_rwlock_wrlock: hot-backup lock");

			if ((ret =
			    session->checkpoint(session, ckpt_config)) != 0)
				die(ret, "session.checkpoint%s%s",
				    ckpt_config == NULL ? "" : ": ",
				    ckpt_config == NULL ? "" : ckpt_config);

			if (ckpt_config != NULL &&
			    (ret = pthread_rwlock_unlock(&g.backup_lock)) != 0)
				die(ret,
				    "pthread_rwlock_wrlock: hot-backup lock");

			/*
			 * Pick the next checkpoint operation, try for roughly
			 * five checkpoint operations per thread run.
			 */
			ckpt_op += MMRAND(1, thread_ops) / 5;
		}

		/*
		 * If we're not single-threaded and we're not in a transaction,
		 * start a transaction 80% of the time.
		 */
		if (!SINGLETHREADED && !intxn && MMRAND(1, 10) >= 8) {
			if ((ret =
			    session->begin_transaction(session, NULL)) != 0)
				die(ret, "session.begin_transaction");
			intxn = 1;
		}

		insert = notfound = 0;

		keyno = MMRAND(1, g.rows);
		key.data = keybuf;
		value.data = valbuf;

		/*
		 * Perform some number of operations: the percentage of deletes,
		 * inserts and writes are specified, reads are the rest.  The
		 * percentages don't have to add up to 100, a high percentage
		 * of deletes will mean fewer inserts and writes.  Modifications
		 * are always followed by a read to confirm it worked.
		 */
		op = (uint32_t)(rng() % 100);
		if (op < g.c_delete_pct) {
			++tinfo->remove;
			switch (g.type) {
			case ROW:
				/*
				 * If deleting a non-existent record, the cursor
				 * won't be positioned, and so can't do a next.
				 */
				if (row_remove(cursor, &key, keyno, &notfound))
					goto deadlock;
				break;
			case FIX:
			case VAR:
				if (col_remove(cursor, &key, keyno, &notfound))
					goto deadlock;
				break;
			}
		} else if (op < g.c_delete_pct + g.c_insert_pct) {
			++tinfo->insert;
			switch (g.type) {
			case ROW:
				if (row_insert(cursor, &key, &value, keyno))
					goto deadlock;
				insert = 1;
				break;
			case FIX:
			case VAR:
				/*
				 * We can only append so many new records, if
				 * we've reached that limit, update a record
				 * instead of doing an insert.
				 */
				if (g.append_cnt >= g.append_max)
					goto skip_insert;

				/*
				 * Reset the standard cursor so it doesn't keep
				 * pages pinned.
				 */
				if ((ret = cursor->reset(cursor)) != 0)
					die(ret, "cursor.reset");

				/* Insert, then reset the insert cursor. */
				if (col_insert(
				    cursor_insert, &key, &value, &keyno))
					goto deadlock;
				if ((ret =
				    cursor_insert->reset(cursor_insert)) != 0)
					die(ret, "cursor.reset");

				insert = 1;
				break;
			}
		} else if (
		    op < g.c_delete_pct + g.c_insert_pct + g.c_write_pct) {
			++tinfo->update;
			switch (g.type) {
			case ROW:
				if (row_update(cursor, &key, &value, keyno))
					goto deadlock;
				break;
			case FIX:
			case VAR:
skip_insert:			if (col_update(cursor, &key, &value, keyno))
					goto deadlock;
				break;
			}
		} else {
			++tinfo->search;
			if (read_row(cursor, &key, keyno))
				goto deadlock;
			continue;
		}

		/*
		 * The cursor is positioned if we did any operation other than
		 * insert, do a small number of next/prev cursor operations in
		 * a random direction.
		 */
		if (!insert) {
			dir = (int)MMRAND(0, 1);
			for (np = 0; np < MMRAND(1, 8); ++np) {
				if (notfound)
					break;
				if (nextprev(cursor, dir, &notfound))
					goto deadlock;
			}
		}

		/* Read the value we modified to confirm the operation. */
		++tinfo->search;
		if (read_row(cursor, &key, keyno))
			goto deadlock;

		/*
		 * If we're in the transaction, commit 40% of the time and
		 * rollback 10% of the time.
		 */
		if (intxn)
			switch (MMRAND(1, 10)) {
			case 1: case 2: case 3: case 4:		/* 40% */
				if ((ret = session->commit_transaction(
				    session, NULL)) != 0)
					die(ret, "session.commit_transaction");
				++tinfo->commit;
				intxn = 0;
				break;
			case 5:					/* 10% */
				if (0) {
deadlock:				++tinfo->deadlock;
				}
				if ((ret = session->rollback_transaction(
				    session, NULL)) != 0)
					die(ret, "session.commit_transaction");
				++tinfo->rollback;
				intxn = 0;
				break;
			default:
				break;
			}
	}

	if (session != NULL && (ret = session->close(session, NULL)) != 0)
		die(ret, "session.close");

	free(keybuf);
	free(valbuf);

	tinfo->state = TINFO_COMPLETE;
	return (NULL);
}