/*
* Each thread inserts a set of keys into the record store database. The keys
* are generated in such a way that there are large gaps in the key range.
*/
static void *
thread_func(void *arg)
{
TEST_OPTS *opts;
WT_CURSOR *cursor, *idx_cursor;
WT_SESSION *session;
uint64_t i, ins_rotor, ins_thr_idx, thr_idx, ts;
uint64_t *obj_data;
opts = (TEST_OPTS *)arg;
thr_idx = __wt_atomic_fetch_addv64(&opts->next_threadid, 1);
ts = g_ts;
obj_data = dcalloc(
(NR_OBJECTS/NR_THREADS + 1) * NR_FIELDS, sizeof(*obj_data));
testutil_check(opts->conn->open_session(
opts->conn, NULL, NULL, &session));
testutil_check(session->open_cursor(
session, opts->uri, NULL, NULL, &cursor));
testutil_check(session->open_cursor(
session, "table:index", NULL, NULL, &idx_cursor));
for (ins_rotor = 1; ins_rotor < 10; ++ins_rotor) {
for (ins_thr_idx = thr_idx, i = 0; ins_thr_idx < NR_OBJECTS;
ins_thr_idx += NR_THREADS, i += NR_FIELDS) {
testutil_check(
session->begin_transaction(session, "sync=false"));
cursor->set_key(cursor, ins_thr_idx << 40 | ins_rotor);
cursor->set_value(cursor, ts,
obj_data[i+0], obj_data[i+1], obj_data[i+2],
obj_data[i+3], obj_data[i+4], obj_data[i+5],
obj_data[i+6], obj_data[i+7]);
testutil_check(cursor->insert(cursor));
idx_cursor->set_key(
idx_cursor, ins_thr_idx << 40 | ts);
idx_cursor->set_value(idx_cursor, ins_rotor);
testutil_check(idx_cursor->insert(idx_cursor));
testutil_check(
session->commit_transaction(session, NULL));
/* change object fields */
++obj_data[i + ((ins_thr_idx + ins_rotor) % NR_FIELDS)];
++obj_data[i +
((ins_thr_idx + ins_rotor + 1) % NR_FIELDS)];
++g_ts;
/* 5K updates/sec */
(void)usleep(1000000ULL * NR_THREADS / 5000);
}
}
testutil_check(session->close(session, NULL));
free(obj_data);
return (NULL);
}
void WiredTigerRecoveryUnit::_txnClose(bool commit) {
invariant(_isActive(), toString(_state));
WT_SESSION* s = _session->getSession();
if (_timer) {
const int transactionTime = _timer->millis();
// `serverGlobalParams.slowMs` can be set to values <= 0. In those cases, give logging a
// break.
if (transactionTime >= std::max(1, serverGlobalParams.slowMS)) {
LOG(kSlowTransactionSeverity) << "Slow WT transaction. Lifetime of SnapshotId "
<< _mySnapshotId << " was " << transactionTime << "ms";
}
}
int wtRet;
if (commit) {
if (!_commitTimestamp.isNull()) {
const std::string conf = "commit_timestamp=" + integerToHex(_commitTimestamp.asULL());
invariantWTOK(s->timestamp_transaction(s, conf.c_str()));
_isTimestamped = true;
}
wtRet = s->commit_transaction(s, nullptr);
LOG(3) << "WT commit_transaction for snapshot id " << _mySnapshotId;
} else {
wtRet = s->rollback_transaction(s, nullptr);
invariant(!wtRet);
LOG(3) << "WT rollback_transaction for snapshot id " << _mySnapshotId;
}
if (_isTimestamped) {
if (!_orderedCommit) {
// We only need to update oplog visibility where commits can be out-of-order with
// respect to their assigned optime and such commits might otherwise be visible.
// This should happen only on primary nodes.
_oplogManager->triggerJournalFlush();
}
_isTimestamped = false;
}
invariantWTOK(wtRet);
invariant(!_lastTimestampSet || _commitTimestamp.isNull(),
str::stream() << "Cannot have both a _lastTimestampSet and a "
"_commitTimestamp. _lastTimestampSet: "
<< _lastTimestampSet->toString()
<< ". _commitTimestamp: "
<< _commitTimestamp.toString());
// We reset the _lastTimestampSet between transactions. Since it is legal for one
// transaction on a RecoveryUnit to call setTimestamp() and another to call
// setCommitTimestamp().
_lastTimestampSet = boost::none;
_prepareTimestamp = Timestamp();
_mySnapshotId = nextSnapshotId.fetchAndAdd(1);
_isOplogReader = false;
_orderedCommit = true; // Default value is true; we assume all writes are ordered.
}
void WiredTigerRecoveryUnit::_txnClose(bool commit) {
invariant(_active);
WT_SESSION* s = _session->getSession();
if (commit) {
invariantWTOK(s->commit_transaction(s, NULL));
LOG(3) << "WT commit_transaction for snapshot id " << _mySnapshotId;
} else {
invariantWTOK(s->rollback_transaction(s, NULL));
LOG(3) << "WT rollback_transaction for snapshot id " << _mySnapshotId;
}
_active = false;
_mySnapshotId = nextSnapshotId.fetchAndAdd(1);
}
void WiredTigerRecoveryUnit::_txnClose(bool commit) {
invariant(_active);
WT_SESSION* s = _session->getSession();
if (commit) {
invariantWTOK(s->commit_transaction(s, NULL));
LOG(2) << "WT commit_transaction";
} else {
invariantWTOK(s->rollback_transaction(s, NULL));
LOG(2) << "WT rollback_transaction";
}
_active = false;
_myTransactionCount++;
_ticket.reset(NULL);
}
bool do_commit() override {
resetCursors();
#if TERARK_WT_USE_TXN
WT_SESSION* ses = m_session.ses;
int err = ses->commit_transaction(ses, NULL);
if (err) {
m_strError = "wiredtiger commit_transaction: ";
m_strError += ses->strerror(ses, err);
assert(!"wiredtiger commit_transaction failed");
return false;
}
#endif
m_wrtStore->estimateIncDataSize(m_sizeDiff);
return true;
}
void WiredTigerRecoveryUnit::_txnClose( bool commit ) {
invariant( _active );
WT_SESSION *s = _session->getSession();
if ( commit ) {
invariantWTOK( s->commit_transaction(s, NULL) );
LOG(2) << "WT commit_transaction";
if ( _syncing )
awaitCommitData.syncHappend();
}
else {
invariantWTOK( s->rollback_transaction(s, NULL) );
LOG(2) << "WT rollback_transaction";
}
_active = false;
}
void WiredTigerKVEngine::syncSizeInfo() const {
if ( !_sizeStorer )
return;
try {
WiredTigerSession session( _conn, -1 );
WT_SESSION* s = session.getSession();
invariantWTOK( s->begin_transaction( s, "sync=true" ) );
_sizeStorer->storeInto( &session, _sizeStorerUri );
invariantWTOK( s->commit_transaction( s, NULL ) );
}
catch ( const WriteConflictException& de ) {
// ignore, it means someone else is doing it
}
}
/*
* Append to a table in a "racy" fashion - that is attempt to insert the
* same record another thread is likely to also be inserting.
*/
void *
thread_insert_race(void *arg)
{
TEST_OPTS *opts;
WT_CONNECTION *conn;
WT_CURSOR *cursor;
WT_SESSION *session;
uint64_t i, value;
int ret;
opts = (TEST_OPTS *)arg;
conn = opts->conn;
testutil_check(conn->open_session(conn, NULL, NULL, &session));
testutil_check(session->open_cursor(
session, opts->uri, NULL, NULL, &cursor));
printf("Running insert thread\n");
for (i = 0; i < opts->nrecords; ++i) {
testutil_check(
session->begin_transaction(session, "isolation=snapshot"));
cursor->set_key(cursor, 1);
testutil_check(cursor->search(cursor));
testutil_check(cursor->get_value(cursor, &value));
cursor->set_key(cursor, 1);
cursor->set_value(cursor, value + 1);
if ((ret = cursor->update(cursor)) != 0) {
if (ret == WT_ROLLBACK) {
testutil_check(session->rollback_transaction(
session, NULL));
i--;
continue;
}
printf("Error in update: %d\n", ret);
}
testutil_check(session->commit_transaction(session, NULL));
if (i % 10000 == 0) {
printf("insert: %" PRIu64 "\r", i);
fflush(stdout);
}
}
if (i > 10000)
printf("\n");
opts->running = false;
return (NULL);
}
int main(void)
{
int ret;
WT_CONNECTION *conn;
WT_SESSION *session;
WT_CURSOR *cursor;
const char *key, *value;
/*! [configure cache size] */
if ((ret = wiredtiger_open(home, NULL,
"create,cache_size=500M", &conn)) != 0)
fprintf(stderr, "Error connecting to %s: %s\n",
home, wiredtiger_strerror(ret));
/*! [configure cache size] */
/*! [create a table] */
ret = conn->open_session(conn, NULL, NULL, &session);
ret = session->create(session,
"table:access", "key_format=S,value_format=S");
/*! [create a table] */
/*! [transaction] */
ret = session->begin_transaction(session, "priority=100,name=mytxn");
ret = session->open_cursor(session, "config:", NULL, NULL, &cursor);
while ((ret = cursor->next(cursor)) == 0) {
cursor->get_key(cursor, &key);
cursor->get_value(cursor, &value);
printf("configuration value: %s = %s\n", key, value);
}
ret = session->commit_transaction(session, NULL);
/*! [transaction] */
ret = conn->close(conn, NULL);
return (ret);
}
void WiredTigerRecoveryUnit::_txnClose(bool commit) {
invariant(_active);
WT_SESSION* s = _session->getSession();
if (_timer) {
const int transactionTime = _timer->millis();
if (transactionTime >= serverGlobalParams.slowMS) {
LOG(kSlowTransactionSeverity) << "Slow WT transaction. Lifetime of SnapshotId "
<< _mySnapshotId << " was " << transactionTime << "ms";
}
}
if (commit) {
invariantWTOK(s->commit_transaction(s, NULL));
LOG(3) << "WT commit_transaction for snapshot id " << _mySnapshotId;
} else {
invariantWTOK(s->rollback_transaction(s, NULL));
LOG(3) << "WT rollback_transaction for snapshot id " << _mySnapshotId;
}
_active = false;
_mySnapshotId = nextSnapshotId.fetchAndAdd(1);
_oplogReadTill = RecordId();
}
void
populate(TEST_OPTS *opts)
{
WT_CURSOR *maincur;
WT_RAND_STATE rnd;
WT_SESSION *session;
uint32_t key;
int balance, i, flag, post;
__wt_random_init_seed(NULL, &rnd);
testutil_check(opts->conn->open_session(
opts->conn, NULL, NULL, &session));
testutil_check(session->open_cursor(session, opts->uri, NULL, NULL,
&maincur));
for (i = 0; i < N_INSERT; i++) {
testutil_check(session->begin_transaction(session, NULL));
key = (__wt_random(&rnd) % (N_RECORDS));
maincur->set_key(maincur, key);
if (__wt_random(&rnd) % 11 == 0)
post = 54321;
else
post = i % 100000;
if (__wt_random(&rnd) % 4 == 0) {
balance = -100;
flag = 1;
} else {
balance = 100 * (i + 1);
flag = 0;
}
maincur->set_value(maincur, post, balance, flag, key);
testutil_check(maincur->insert(maincur));
testutil_check(session->commit_transaction(session, NULL));
}
testutil_check(maincur->close(maincur));
testutil_check(session->close(session, NULL));
}
static void *
ops(void *arg)
{
TINFO *tinfo;
WT_CONNECTION *conn;
WT_CURSOR *cursor, *cursor_insert;
WT_SESSION *session;
WT_ITEM key, value;
uint64_t cnt, keyno, ckpt_op, session_op, thread_ops;
uint32_t op;
uint8_t *keybuf, *valbuf;
u_int np;
int dir, insert, intxn, notfound, ret;
char *ckpt_config, config[64];
tinfo = arg;
conn = g.wts_conn;
keybuf = valbuf = NULL;
/* Set up the default key and value buffers. */
key_gen_setup(&keybuf);
val_gen_setup(&valbuf);
/*
* Each thread does its share of the total operations, and make sure
* that it's not 0 (testing runs: threads might be larger than ops).
*/
thread_ops = 100 + g.c_ops / g.c_threads;
/*
* Select the first operation where we'll create sessions and cursors,
* perform checkpoint operations.
*/
ckpt_op = MMRAND(1, thread_ops);
session_op = 0;
session = NULL;
cursor = cursor_insert = NULL;
for (intxn = 0, cnt = 0; cnt < thread_ops; ++cnt) {
if (SINGLETHREADED && cnt % 100 == 0)
track("ops", 0ULL, tinfo);
/*
* We can't checkpoint or swap sessions/cursors while in a
* transaction, resolve any running transaction. Otherwise,
* reset the cursor: we may block waiting for a lock and there
* is no reason to keep pages pinned.
*/
if (cnt == ckpt_op || cnt == session_op) {
if (intxn) {
if ((ret = session->commit_transaction(
session, NULL)) != 0)
die(ret, "session.commit_transaction");
++tinfo->commit;
intxn = 0;
}
else if (cursor != NULL &&
(ret = cursor->reset(cursor)) != 0)
die(ret, "cursor.reset");
}
/* Open up a new session and cursors. */
if (cnt == session_op || session == NULL || cursor == NULL) {
if (session != NULL &&
(ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
if ((ret = conn->open_session(
conn, NULL, NULL, &session)) != 0)
die(ret, "connection.open_session");
/*
* Open two cursors: one configured for overwriting and
* one configured for append if we're dealing with a
* column-store.
*
* The reason is when testing with existing records, we
* don't track if a record was deleted or not, which
* means we must use cursor->insert with overwriting
* configured. But, in column-store files where we're
* testing with new, appended records, we don't want to
* have to specify the record number, which requires an
* append configuration.
*/
if ((ret = session->open_cursor(session,
g.uri, NULL, "overwrite", &cursor)) != 0)
die(ret, "session.open_cursor");
if ((g.type == FIX || g.type == VAR) &&
(ret = session->open_cursor(session,
g.uri, NULL, "append", &cursor_insert)) != 0)
die(ret, "session.open_cursor");
/* Pick the next session/cursor close/open. */
session_op += SINGLETHREADED ?
MMRAND(1, thread_ops) : 100 * MMRAND(1, 50);
}
/* Checkpoint the database. */
if (cnt == ckpt_op) {
/*
* LSM and data-sources don't support named checkpoints,
* else 25% of the time we name the checkpoint.
*/
if (DATASOURCE("lsm") || DATASOURCE("kvsbdb") ||
DATASOURCE("memrata") || MMRAND(1, 4) == 1)
ckpt_config = NULL;
else {
(void)snprintf(config, sizeof(config),
"name=thread-%d", tinfo->id);
ckpt_config = config;
}
/* Named checkpoints lock out hot backups */
if (ckpt_config != NULL &&
(ret = pthread_rwlock_wrlock(&g.backup_lock)) != 0)
die(ret,
"pthread_rwlock_wrlock: hot-backup lock");
if ((ret =
session->checkpoint(session, ckpt_config)) != 0)
die(ret, "session.checkpoint%s%s",
ckpt_config == NULL ? "" : ": ",
ckpt_config == NULL ? "" : ckpt_config);
if (ckpt_config != NULL &&
(ret = pthread_rwlock_unlock(&g.backup_lock)) != 0)
die(ret,
"pthread_rwlock_wrlock: hot-backup lock");
/*
* Pick the next checkpoint operation, try for roughly
* five checkpoint operations per thread run.
*/
ckpt_op += MMRAND(1, thread_ops) / 5;
}
/*
* If we're not single-threaded and we're not in a transaction,
* start a transaction 80% of the time.
*/
if (!SINGLETHREADED && !intxn && MMRAND(1, 10) >= 8) {
if ((ret =
session->begin_transaction(session, NULL)) != 0)
die(ret, "session.begin_transaction");
intxn = 1;
}
insert = notfound = 0;
keyno = MMRAND(1, g.rows);
key.data = keybuf;
value.data = valbuf;
/*
* Perform some number of operations: the percentage of deletes,
* inserts and writes are specified, reads are the rest. The
* percentages don't have to add up to 100, a high percentage
* of deletes will mean fewer inserts and writes. Modifications
* are always followed by a read to confirm it worked.
*/
op = (uint32_t)(rng() % 100);
if (op < g.c_delete_pct) {
++tinfo->remove;
switch (g.type) {
case ROW:
/*
* If deleting a non-existent record, the cursor
* won't be positioned, and so can't do a next.
*/
if (row_remove(cursor, &key, keyno, ¬found))
goto deadlock;
break;
case FIX:
case VAR:
if (col_remove(cursor, &key, keyno, ¬found))
goto deadlock;
break;
}
} else if (op < g.c_delete_pct + g.c_insert_pct) {
++tinfo->insert;
switch (g.type) {
case ROW:
if (row_insert(cursor, &key, &value, keyno))
goto deadlock;
insert = 1;
break;
case FIX:
case VAR:
/*
* We can only append so many new records, if
* we've reached that limit, update a record
* instead of doing an insert.
*/
if (g.append_cnt >= g.append_max)
goto skip_insert;
/*
* Reset the standard cursor so it doesn't keep
* pages pinned.
*/
if ((ret = cursor->reset(cursor)) != 0)
die(ret, "cursor.reset");
/* Insert, then reset the insert cursor. */
if (col_insert(
cursor_insert, &key, &value, &keyno))
goto deadlock;
if ((ret =
cursor_insert->reset(cursor_insert)) != 0)
die(ret, "cursor.reset");
insert = 1;
break;
}
} else if (
op < g.c_delete_pct + g.c_insert_pct + g.c_write_pct) {
++tinfo->update;
switch (g.type) {
case ROW:
if (row_update(cursor, &key, &value, keyno))
goto deadlock;
break;
case FIX:
case VAR:
skip_insert: if (col_update(cursor, &key, &value, keyno))
goto deadlock;
break;
}
} else {
++tinfo->search;
if (read_row(cursor, &key, keyno))
goto deadlock;
continue;
}
/*
* The cursor is positioned if we did any operation other than
* insert, do a small number of next/prev cursor operations in
* a random direction.
*/
if (!insert) {
dir = (int)MMRAND(0, 1);
for (np = 0; np < MMRAND(1, 8); ++np) {
if (notfound)
break;
if (nextprev(cursor, dir, ¬found))
goto deadlock;
}
}
/* Read the value we modified to confirm the operation. */
++tinfo->search;
if (read_row(cursor, &key, keyno))
goto deadlock;
/*
* If we're in the transaction, commit 40% of the time and
* rollback 10% of the time.
*/
if (intxn)
switch (MMRAND(1, 10)) {
case 1: case 2: case 3: case 4: /* 40% */
if ((ret = session->commit_transaction(
session, NULL)) != 0)
die(ret, "session.commit_transaction");
++tinfo->commit;
intxn = 0;
break;
case 5: /* 10% */
if (0) {
deadlock: ++tinfo->deadlock;
}
if ((ret = session->rollback_transaction(
session, NULL)) != 0)
die(ret, "session.commit_transaction");
++tinfo->rollback;
intxn = 0;
break;
default:
break;
}
}
if (session != NULL && (ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
free(keybuf);
free(valbuf);
tinfo->state = TINFO_COMPLETE;
return (NULL);
}
/*
* thread_run --
* Runner function for the worker threads.
*/
static WT_THREAD_RET
thread_run(void *arg)
{
FILE *fp;
WT_CURSOR *cur_coll, *cur_local, *cur_oplog, *cur_stable;
WT_ITEM data;
WT_RAND_STATE rnd;
WT_SESSION *session;
WT_THREAD_DATA *td;
uint64_t i, stable_ts;
int ret;
char cbuf[MAX_VAL], lbuf[MAX_VAL], obuf[MAX_VAL];
char kname[64], tscfg[64];
__wt_random_init(&rnd);
memset(cbuf, 0, sizeof(cbuf));
memset(lbuf, 0, sizeof(lbuf));
memset(obuf, 0, sizeof(obuf));
memset(kname, 0, sizeof(kname));
td = (WT_THREAD_DATA *)arg;
/*
* Set up the separate file for checking.
*/
testutil_check(__wt_snprintf(cbuf, sizeof(cbuf), RECORDS_FILE, td->id));
(void)unlink(cbuf);
testutil_checksys((fp = fopen(cbuf, "w")) == NULL);
/*
* Set to line buffering. But that is advisory only. We've seen
* cases where the result files end up with partial lines.
*/
__wt_stream_set_line_buffer(fp);
if ((ret = td->conn->open_session(td->conn, NULL, NULL, &session)) != 0)
testutil_die(ret, "WT_CONNECTION:open_session");
/*
* Open a cursor to each table.
*/
if ((ret = session->open_cursor(session,
uri_collection, NULL, NULL, &cur_coll)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_collection);
if ((ret = session->open_cursor(session,
uri_local, NULL, NULL, &cur_local)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_local);
if ((ret = session->open_cursor(session,
uri_oplog, NULL, NULL, &cur_oplog)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_oplog);
if ((ret = session->open_cursor(
session, stable_store, NULL, NULL, &cur_stable)) != 0)
testutil_die(ret, "WT_SESSION.open_cursor: %s", stable_store);
/*
* Write our portion of the key space until we're killed.
*/
printf("Thread %" PRIu32 " starts at %" PRIu64 "\n", td->id, td->start);
for (i = td->start; ; ++i) {
if (use_ts)
stable_ts = global_ts++;
else
stable_ts = 0;
testutil_check(__wt_snprintf(
kname, sizeof(kname), "%" PRIu64, i));
testutil_check(session->begin_transaction(session, NULL));
cur_coll->set_key(cur_coll, kname);
cur_local->set_key(cur_local, kname);
cur_oplog->set_key(cur_oplog, kname);
/*
* Put an informative string into the value so that it
* can be viewed well in a binary dump.
*/
testutil_check(__wt_snprintf(cbuf, sizeof(cbuf),
"COLL: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
td->id, stable_ts, i));
testutil_check(__wt_snprintf(lbuf, sizeof(lbuf),
"LOCAL: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
td->id, stable_ts, i));
testutil_check(__wt_snprintf(obuf, sizeof(obuf),
"OPLOG: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64,
td->id, stable_ts, i));
data.size = __wt_random(&rnd) % MAX_VAL;
data.data = cbuf;
cur_coll->set_value(cur_coll, &data);
if ((ret = cur_coll->insert(cur_coll)) != 0)
testutil_die(ret, "WT_CURSOR.insert");
data.size = __wt_random(&rnd) % MAX_VAL;
data.data = obuf;
cur_oplog->set_value(cur_oplog, &data);
if ((ret = cur_oplog->insert(cur_oplog)) != 0)
testutil_die(ret, "WT_CURSOR.insert");
if (use_ts) {
testutil_check(__wt_snprintf(tscfg, sizeof(tscfg),
"commit_timestamp=%" PRIx64, stable_ts));
testutil_check(
session->commit_transaction(session, tscfg));
} else
testutil_check(
session->commit_transaction(session, NULL));
/*
* Insert into the local table outside the timestamp txn.
*/
data.size = __wt_random(&rnd) % MAX_VAL;
data.data = lbuf;
cur_local->set_value(cur_local, &data);
if ((ret = cur_local->insert(cur_local)) != 0)
testutil_die(ret, "WT_CURSOR.insert");
/*
* Every N records we will record our stable timestamp into the
* stable table. That will define our threshold where we
* expect to find records after recovery.
*/
if (i % STABLE_PERIOD == 0) {
if (use_ts) {
/*
* Set both the oldest and stable timestamp
* so that we don't need to maintain read
* availability at older timestamps.
*/
testutil_check(__wt_snprintf(
tscfg, sizeof(tscfg),
"oldest_timestamp=%" PRIx64
",stable_timestamp=%" PRIx64,
stable_ts, stable_ts));
testutil_check(
td->conn->set_timestamp(td->conn, tscfg));
}
cur_stable->set_key(cur_stable, td->id);
cur_stable->set_value(cur_stable, stable_ts);
testutil_check(cur_stable->insert(cur_stable));
}
/*
* Save the timestamp and key separately for checking later.
*/
if (fprintf(fp,
"%" PRIu64 " %" PRIu64 "\n", stable_ts, i) < 0)
testutil_die(EIO, "fprintf");
}
/* NOTREACHED */
}
void WiredTigerSizeStorer::syncCache(bool syncToDisk) {
stdx::lock_guard<stdx::mutex> cursorLock(_cursorMutex);
_checkMagic();
Map myMap;
{
stdx::lock_guard<stdx::mutex> lk(_entriesMutex);
for (Map::iterator it = _entries.begin(); it != _entries.end(); ++it) {
std::string uriKey = it->first;
Entry& entry = it->second;
if (entry.rs) {
if (entry.dataSize != entry.rs->dataSize(NULL)) {
entry.dataSize = entry.rs->dataSize(NULL);
entry.dirty = true;
}
if (entry.numRecords != entry.rs->numRecords(NULL)) {
entry.numRecords = entry.rs->numRecords(NULL);
entry.dirty = true;
}
}
if (!entry.dirty)
continue;
myMap[uriKey] = entry;
}
}
if (myMap.empty())
return; // Nothing to do.
WT_SESSION* session = _session.getSession();
invariantWTOK(session->begin_transaction(session, syncToDisk ? "sync=true" : ""));
ScopeGuard rollbacker = MakeGuard(session->rollback_transaction, session, "");
for (Map::iterator it = myMap.begin(); it != myMap.end(); ++it) {
string uriKey = it->first;
Entry& entry = it->second;
BSONObj data;
{
BSONObjBuilder b;
b.append("numRecords", entry.numRecords);
b.append("dataSize", entry.dataSize);
data = b.obj();
}
LOG(2) << "WiredTigerSizeStorer::storeInto " << uriKey << " -> " << redact(data);
WiredTigerItem key(uriKey.c_str(), uriKey.size());
WiredTigerItem value(data.objdata(), data.objsize());
_cursor->set_key(_cursor, key.Get());
_cursor->set_value(_cursor, value.Get());
invariantWTOK(_cursor->insert(_cursor));
}
invariantWTOK(_cursor->reset(_cursor));
rollbacker.Dismiss();
invariantWTOK(session->commit_transaction(session, NULL));
{
stdx::lock_guard<stdx::mutex> lk(_entriesMutex);
for (Map::iterator it = _entries.begin(); it != _entries.end(); ++it) {
it->second.dirty = false;
}
}
}
static void
transaction_ops(WT_SESSION *session_arg)
{
WT_CONNECTION *conn;
WT_CURSOR *cursor;
WT_SESSION *session;
session = session_arg;
conn = session->connection;
/*! [transaction commit/rollback] */
/*
* Cursors may be opened before or after the transaction begins, and in
* either case, subsequent operations are included in the transaction.
* Opening cursors before the transaction begins allows applications to
* cache cursors and use them for multiple operations.
*/
error_check(session->open_cursor(
session, "table:mytable", NULL, NULL, &cursor));
error_check(session->begin_transaction(session, NULL));
cursor->set_key(cursor, "key");
cursor->set_value(cursor, "value");
switch (cursor->update(cursor)) {
case 0: /* Update success */
error_check(session->commit_transaction(session, NULL));
/*
* If commit_transaction succeeds, cursors remain positioned; if
* commit_transaction fails, the transaction was rolled-back and
* and all cursors are reset.
*/
break;
case WT_ROLLBACK: /* Update conflict */
default: /* Other error */
error_check(session->rollback_transaction(session, NULL));
/* The rollback_transaction call resets all cursors. */
break;
}
/*
* Cursors remain open and may be used for multiple transactions.
*/
/*! [transaction commit/rollback] */
error_check(cursor->close(cursor));
/*! [transaction isolation] */
/* A single transaction configured for snapshot isolation. */
error_check(session->open_cursor(
session, "table:mytable", NULL, NULL, &cursor));
error_check(session->begin_transaction(session, "isolation=snapshot"));
cursor->set_key(cursor, "some-key");
cursor->set_value(cursor, "some-value");
error_check(cursor->update(cursor));
error_check(session->commit_transaction(session, NULL));
/*! [transaction isolation] */
{
/*! [transaction prepare] */
/*
* Prepare a transaction which guarantees a subsequent commit will
* succeed. Only commit and rollback are allowed on a transaction after
* it has been prepared.
*/
error_check(session->open_cursor(
session, "table:mytable", NULL, NULL, &cursor));
error_check(session->begin_transaction(session, NULL));
cursor->set_key(cursor, "key");
cursor->set_value(cursor, "value");
error_check(session->prepare_transaction(
session, "prepare_timestamp=2a"));
error_check(session->commit_transaction(
session, "commit_timestamp=2b"));
/*! [transaction prepare] */
}
/*! [session isolation configuration] */
/* Open a session configured for read-uncommitted isolation. */
error_check(conn->open_session(
conn, NULL, "isolation=read-uncommitted", &session));
/*! [session isolation configuration] */
/*! [session isolation re-configuration] */
/* Re-configure a session for snapshot isolation. */
error_check(session->reconfigure(session, "isolation=snapshot"));
/*! [session isolation re-configuration] */
error_check(session->close(session, NULL));
session = session_arg;
{
/*! [transaction pinned range] */
/* Check the transaction ID range pinned by the session handle. */
uint64_t range;
error_check(session->transaction_pinned_range(session, &range));
/*! [transaction pinned range] */
}
error_check(session->begin_transaction(session, NULL));
{
/*! [query timestamp] */
char timestamp_buf[2 * sizeof(uint64_t) + 1];
/*! [transaction timestamp] */
error_check(
session->timestamp_transaction(session, "commit_timestamp=2a"));
/*! [transaction timestamp] */
error_check(session->commit_transaction(session, NULL));
error_check(conn->query_timestamp(
conn, timestamp_buf, "get=all_committed"));
/*! [query timestamp] */
}
/*! [set commit timestamp] */
error_check(conn->set_timestamp(conn, "commit_timestamp=2a"));
/*! [set commit timestamp] */
/*! [set oldest timestamp] */
error_check(conn->set_timestamp(conn, "oldest_timestamp=2a"));
/*! [set oldest timestamp] */
/*! [set stable timestamp] */
error_check(conn->set_timestamp(conn, "stable_timestamp=2a"));
/*! [set stable timestamp] */
/*! [rollback to stable] */
error_check(conn->rollback_to_stable(conn, NULL));
/*! [rollback to stable] */
}
/*
* subtest_populate --
* Populate the tables.
*/
static void
subtest_populate(TEST_OPTS *opts, bool close_test)
{
WT_CURSOR *maincur, *maincur2;
WT_RAND_STATE rnd;
WT_SESSION *session;
uint64_t i, nrecords;
uint32_t rndint;
int key, v0, v1, v2;
char *big, *bigref;
bool failed;
failed = false;
__wt_random_init_seed(NULL, &rnd);
CHECK(create_big_string(&bigref), false);
nrecords = opts->nrecords;
CHECK(opts->conn->open_session(
opts->conn, NULL, NULL, &session), false);
CHECK(session->open_cursor(session, "table:subtest", NULL,
NULL, &maincur), false);
CHECK(session->open_cursor(session, "table:subtest2", NULL,
NULL, &maincur2), false);
for (i = 0; i < nrecords && !failed; i++) {
rndint = __wt_random(&rnd);
generate_key(i, &key);
generate_value(rndint, i, bigref, &v0, &v1, &v2, &big);
CHECK(session->begin_transaction(session, NULL), false);
maincur->set_key(maincur, key);
maincur->set_value(maincur, v0, v1, v2, big);
CHECK(maincur->insert(maincur), false);
maincur2->set_key(maincur2, key);
maincur2->set_value(maincur2, rndint);
CHECK(maincur2->insert(maincur2), false);
CHECK(session->commit_transaction(session, NULL), false);
if (i == 0)
/*
* Force an initial checkpoint, that helps to
* distinguish a clear failure from just not running
* long enough.
*/
CHECK(session->checkpoint(session, NULL), false);
if ((i + 1) % VERBOSE_PRINT == 0 && opts->verbose)
printf(" %" PRIu64 "/%" PRIu64 "\n",
(i + 1), nrecords);
/* Attempt to isolate the failures to checkpointing. */
if (i == (nrecords/100)) {
enable_failures(opts->nops, 1000000);
/* CHECK should expect failures. */
CHECK(session->checkpoint(session, NULL), true);
disable_failures();
if (failed && opts->verbose)
printf("checkpoint failed (expected).\n");
}
}
/*
* Closing handles after an extreme fail is likely to cause
* cascading failures (or crashes), so recommended practice is
* to immediately exit. We're interested in testing both with
* and without the recommended practice.
*/
if (failed) {
if (!close_test) {
fprintf(stderr, "exit early.\n");
exit(0);
} else
fprintf(stderr, "closing after failure.\n");
}
free(bigref);
CHECK(maincur->close(maincur), false);
CHECK(maincur2->close(maincur2), false);
CHECK(session->close(session, NULL), false);
}
/*
* real_worker --
* A single worker thread that transactionally updates all tables with
* consistent values.
*/
static int
real_worker(void)
{
WT_CURSOR **cursors;
WT_RAND_STATE rnd;
WT_SESSION *session;
u_int i, keyno;
int j, ret, t_ret;
ret = t_ret = 0;
__wt_random_init(&rnd);
if ((cursors = calloc(
(size_t)(g.ntables), sizeof(WT_CURSOR *))) == NULL)
return (log_print_err("malloc", ENOMEM, 1));
if ((ret = g.conn->open_session(
g.conn, NULL, "isolation=snapshot", &session)) != 0) {
(void)log_print_err("conn.open_session", ret, 1);
goto err;
}
for (j = 0; j < g.ntables; j++)
if ((ret = session->open_cursor(session,
g.cookies[j].uri, NULL, NULL, &cursors[j])) != 0) {
(void)log_print_err("session.open_cursor", ret, 1);
goto err;
}
for (i = 0; i < g.nops && g.running; ++i, __wt_yield()) {
if ((ret = session->begin_transaction(session, NULL)) != 0) {
(void)log_print_err(
"real_worker:begin_transaction", ret, 1);
goto err;
}
keyno = __wt_random(&rnd) % g.nkeys + 1;
for (j = 0; j < g.ntables; j++) {
if ((ret = worker_op(cursors[j], keyno, i)) != 0)
break;
}
if (ret == 0) {
if ((ret = session->commit_transaction(
session, NULL)) != 0) {
(void)log_print_err(
"real_worker:commit_transaction", ret, 1);
goto err;
}
} else if (ret == WT_ROLLBACK) {
if ((ret = session->rollback_transaction(
session, NULL)) != 0) {
(void)log_print_err(
"real_worker:rollback_transaction", ret, 1);
goto err;
}
} else {
(void)log_print_err("worker op failed", ret, 1);
goto err;
}
}
err: if ((t_ret = session->close(session, NULL)) != 0 && ret == 0) {
ret = t_ret;
(void)log_print_err("session.close", ret, 1);
}
free(cursors);
return (ret);
}
int
main(void)
{
WT_CONNECTION *wt_conn;
WT_CURSOR *cursor;
WT_SESSION *session;
int i, record_count, ret;
char cmd_buf[256], k[16], v[16];
snprintf(cmd_buf, sizeof(cmd_buf), "rm -rf %s %s && mkdir %s %s",
home1, home2, home1, home2);
if ((ret = system(cmd_buf)) != 0) {
fprintf(stderr, "%s: failed ret %d\n", cmd_buf, ret);
return (ret);
}
if ((ret = wiredtiger_open(home1, NULL, CONN_CONFIG, &wt_conn)) != 0) {
fprintf(stderr, "Error connecting to %s: %s\n",
home1, wiredtiger_strerror(ret));
return (ret);
}
ret = wt_conn->open_session(wt_conn, NULL, NULL, &session);
ret = session->create(session, uri, "key_format=S,value_format=S");
ret = session->open_cursor(session, uri, NULL, NULL, &cursor);
/*
* Perform some operations with individual auto-commit transactions.
*/
for (record_count = 0, i = 0; i < MAX_KEYS; i++, record_count++) {
snprintf(k, sizeof(k), "key%d", i);
snprintf(v, sizeof(v), "value%d", i);
cursor->set_key(cursor, k);
cursor->set_value(cursor, v);
ret = cursor->insert(cursor);
}
ret = session->begin_transaction(session, NULL);
/*
* Perform some operations within a single transaction.
*/
for (i = MAX_KEYS; i < MAX_KEYS+5; i++, record_count++) {
snprintf(k, sizeof(k), "key%d", i);
snprintf(v, sizeof(v), "value%d", i);
cursor->set_key(cursor, k);
cursor->set_value(cursor, v);
ret = cursor->insert(cursor);
}
ret = session->commit_transaction(session, NULL);
ret = cursor->close(cursor);
/*! [log cursor printf] */
ret = session->log_printf(session, "Wrote %d records", record_count);
/*! [log cursor printf] */
/*
* Close and reopen the connection so that the log ends up with
* a variety of records such as file sync and checkpoint. We
* have archiving turned off.
*/
ret = wt_conn->close(wt_conn, NULL);
if ((ret = wiredtiger_open(home1, NULL, CONN_CONFIG, &wt_conn)) != 0) {
fprintf(stderr, "Error connecting to %s: %s\n",
home1, wiredtiger_strerror(ret));
return (ret);
}
ret = wt_conn->open_session(wt_conn, NULL, NULL, &session);
ret = simple_walk_log(session);
ret = walk_log(session);
ret = wt_conn->close(wt_conn, NULL);
return (ret);
}
static void *
ops(void *arg)
{
TINFO *tinfo;
WT_CONNECTION *conn;
WT_CURSOR *cursor, *cursor_insert;
WT_SESSION *session;
WT_ITEM key, value;
uint64_t keyno, ckpt_op, session_op;
uint32_t op;
uint8_t *keybuf, *valbuf;
u_int np;
int ckpt_available, dir, insert, intxn, notfound, readonly, ret;
char *ckpt_config, ckpt_name[64];
tinfo = arg;
/* Initialize the per-thread random number generator. */
__wt_random_init(&tinfo->rnd);
conn = g.wts_conn;
keybuf = valbuf = NULL;
readonly = 0; /* -Wconditional-uninitialized */
/* Set up the default key and value buffers. */
key_gen_setup(&keybuf);
val_gen_setup(&tinfo->rnd, &valbuf);
/* Set the first operation where we'll create sessions and cursors. */
session_op = 0;
session = NULL;
cursor = cursor_insert = NULL;
/* Set the first operation where we'll perform checkpoint operations. */
ckpt_op = g.c_checkpoints ? mmrand(&tinfo->rnd, 100, 10000) : 0;
ckpt_available = 0;
for (intxn = 0; !tinfo->quit; ++tinfo->ops) {
/*
* We can't checkpoint or swap sessions/cursors while in a
* transaction, resolve any running transaction.
*/
if (intxn &&
(tinfo->ops == ckpt_op || tinfo->ops == session_op)) {
if ((ret = session->commit_transaction(
session, NULL)) != 0)
die(ret, "session.commit_transaction");
++tinfo->commit;
intxn = 0;
}
/* Open up a new session and cursors. */
if (tinfo->ops == session_op ||
session == NULL || cursor == NULL) {
if (session != NULL &&
(ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
if ((ret = conn->open_session(conn, NULL,
ops_session_config(&tinfo->rnd), &session)) != 0)
die(ret, "connection.open_session");
/*
* 10% of the time, perform some read-only operations
* from a checkpoint.
*
* Skip that if we single-threaded and doing checks
* against a Berkeley DB database, because that won't
* work because the Berkeley DB database records won't
* match the checkpoint. Also skip if we are using
* LSM, because it doesn't support reads from
* checkpoints.
*/
if (!SINGLETHREADED && !DATASOURCE("lsm") &&
ckpt_available && mmrand(&tinfo->rnd, 1, 10) == 1) {
if ((ret = session->open_cursor(session,
g.uri, NULL, ckpt_name, &cursor)) != 0)
die(ret, "session.open_cursor");
/* Pick the next session/cursor close/open. */
session_op += 250;
/* Checkpoints are read-only. */
readonly = 1;
} else {
/*
* Open two cursors: one for overwriting and one
* for append (if it's a column-store).
*
* The reason is when testing with existing
* records, we don't track if a record was
* deleted or not, which means we must use
* cursor->insert with overwriting configured.
* But, in column-store files where we're
* testing with new, appended records, we don't
* want to have to specify the record number,
* which requires an append configuration.
*/
if ((ret = session->open_cursor(session, g.uri,
NULL, "overwrite", &cursor)) != 0)
die(ret, "session.open_cursor");
if ((g.type == FIX || g.type == VAR) &&
(ret = session->open_cursor(session, g.uri,
NULL, "append", &cursor_insert)) != 0)
die(ret, "session.open_cursor");
/* Pick the next session/cursor close/open. */
session_op += mmrand(&tinfo->rnd, 100, 5000);
/* Updates supported. */
readonly = 0;
}
}
/* Checkpoint the database. */
if (tinfo->ops == ckpt_op && g.c_checkpoints) {
/*
* LSM and data-sources don't support named checkpoints,
* and we can't drop a named checkpoint while there's a
* cursor open on it, otherwise 20% of the time name the
* checkpoint.
*/
if (DATASOURCE("helium") || DATASOURCE("kvsbdb") ||
DATASOURCE("lsm") ||
readonly || mmrand(&tinfo->rnd, 1, 5) == 1)
ckpt_config = NULL;
else {
(void)snprintf(ckpt_name, sizeof(ckpt_name),
"name=thread-%d", tinfo->id);
ckpt_config = ckpt_name;
}
/* Named checkpoints lock out backups */
if (ckpt_config != NULL &&
(ret = pthread_rwlock_wrlock(&g.backup_lock)) != 0)
die(ret,
"pthread_rwlock_wrlock: backup lock");
if ((ret =
session->checkpoint(session, ckpt_config)) != 0)
die(ret, "session.checkpoint%s%s",
ckpt_config == NULL ? "" : ": ",
ckpt_config == NULL ? "" : ckpt_config);
if (ckpt_config != NULL &&
(ret = pthread_rwlock_unlock(&g.backup_lock)) != 0)
die(ret,
"pthread_rwlock_wrlock: backup lock");
/* Rephrase the checkpoint name for cursor open. */
if (ckpt_config == NULL)
strcpy(ckpt_name,
"checkpoint=WiredTigerCheckpoint");
else
(void)snprintf(ckpt_name, sizeof(ckpt_name),
"checkpoint=thread-%d", tinfo->id);
ckpt_available = 1;
/* Pick the next checkpoint operation. */
ckpt_op += mmrand(&tinfo->rnd, 5000, 20000);
}
/*
* If we're not single-threaded and we're not in a transaction,
* start a transaction 20% of the time.
*/
if (!SINGLETHREADED &&
!intxn && mmrand(&tinfo->rnd, 1, 10) >= 8) {
if ((ret =
session->begin_transaction(session, NULL)) != 0)
die(ret, "session.begin_transaction");
intxn = 1;
}
insert = notfound = 0;
keyno = mmrand(&tinfo->rnd, 1, (u_int)g.rows);
key.data = keybuf;
value.data = valbuf;
/*
* Perform some number of operations: the percentage of deletes,
* inserts and writes are specified, reads are the rest. The
* percentages don't have to add up to 100, a high percentage
* of deletes will mean fewer inserts and writes. Modifications
* are always followed by a read to confirm it worked.
*/
op = readonly ? UINT32_MAX : mmrand(&tinfo->rnd, 1, 100);
if (op < g.c_delete_pct) {
++tinfo->remove;
switch (g.type) {
case ROW:
/*
* If deleting a non-existent record, the cursor
* won't be positioned, and so can't do a next.
*/
if (row_remove(cursor, &key, keyno, ¬found))
goto deadlock;
break;
case FIX:
case VAR:
if (col_remove(cursor, &key, keyno, ¬found))
goto deadlock;
break;
}
} else if (op < g.c_delete_pct + g.c_insert_pct) {
++tinfo->insert;
switch (g.type) {
case ROW:
if (row_insert(
tinfo, cursor, &key, &value, keyno))
goto deadlock;
insert = 1;
break;
case FIX:
case VAR:
/*
* We can only append so many new records, if
* we've reached that limit, update a record
* instead of doing an insert.
*/
if (g.append_cnt >= g.append_max)
goto skip_insert;
/* Insert, then reset the insert cursor. */
if (col_insert(tinfo,
cursor_insert, &key, &value, &keyno))
goto deadlock;
if ((ret =
cursor_insert->reset(cursor_insert)) != 0)
die(ret, "cursor.reset");
insert = 1;
break;
}
} else if (
op < g.c_delete_pct + g.c_insert_pct + g.c_write_pct) {
++tinfo->update;
switch (g.type) {
case ROW:
if (row_update(
tinfo, cursor, &key, &value, keyno))
goto deadlock;
break;
case FIX:
case VAR:
skip_insert: if (col_update(tinfo,
cursor, &key, &value, keyno))
goto deadlock;
break;
}
} else {
++tinfo->search;
if (read_row(cursor, &key, keyno))
if (intxn)
goto deadlock;
continue;
}
/*
* The cursor is positioned if we did any operation other than
* insert, do a small number of next/prev cursor operations in
* a random direction.
*/
if (!insert) {
dir = (int)mmrand(&tinfo->rnd, 0, 1);
for (np = 0; np < mmrand(&tinfo->rnd, 1, 8); ++np) {
if (notfound)
break;
if (nextprev(cursor, dir, ¬found))
goto deadlock;
}
}
/* Read to confirm the operation. */
++tinfo->search;
if (read_row(cursor, &key, keyno))
goto deadlock;
/* Reset the cursor: there is no reason to keep pages pinned. */
if ((ret = cursor->reset(cursor)) != 0)
die(ret, "cursor.reset");
/*
* If we're in the transaction, commit 40% of the time and
* rollback 10% of the time.
*/
if (intxn)
switch (mmrand(&tinfo->rnd, 1, 10)) {
case 1: case 2: case 3: case 4: /* 40% */
if ((ret = session->commit_transaction(
session, NULL)) != 0)
die(ret, "session.commit_transaction");
++tinfo->commit;
intxn = 0;
break;
case 5: /* 10% */
if (0) {
deadlock: ++tinfo->deadlock;
}
if ((ret = session->rollback_transaction(
session, NULL)) != 0)
die(ret,
"session.rollback_transaction");
++tinfo->rollback;
intxn = 0;
break;
default:
break;
}
}
if (session != NULL && (ret = session->close(session, NULL)) != 0)
die(ret, "session.close");
free(keybuf);
free(valbuf);
tinfo->state = TINFO_COMPLETE;
return (NULL);
}
static void *
thread_insert(void *arg)
{
TEST_OPTS *opts;
THREAD_ARGS *threadargs;
WT_CURSOR *maincur;
WT_RAND_STATE rnd;
WT_SESSION *session;
double elapsed;
time_t prevtime, curtime; /* 1 second resolution is okay */
int bal, i, flag, key, post;
const char *extra = S1024;
threadargs = (THREAD_ARGS *)arg;
opts = threadargs->testopts;
testutil_check(__wt_random_init_seed(NULL, &rnd));
(void)time(&prevtime);
testutil_check(opts->conn->open_session(
opts->conn, NULL, NULL, &session));
testutil_check(session->open_cursor(session, opts->uri, NULL, NULL,
&maincur));
for (i = 0; i < N_INSERT; i++) {
/*
* Insert threads may stomp on each other's records;
* that's okay.
*/
key = (int)(__wt_random(&rnd) % N_RECORDS);
testutil_check(session->begin_transaction(session, NULL));
maincur->set_key(maincur, key);
if (__wt_random(&rnd) % 2 == 0)
post = 54321;
else
post = i % 100000;
if (__wt_random(&rnd) % 2 == 0) {
bal = -100;
flag = 1;
} else {
bal = 100 * (i + 1);
flag = 0;
}
maincur->set_value(maincur, post, bal, extra, flag, key);
testutil_check(maincur->insert(maincur));
testutil_check(maincur->reset(maincur));
testutil_check(session->commit_transaction(session, NULL));
if (i % 1000 == 0 && i != 0) {
if (i % 10000 == 0)
fprintf(stderr, "*");
else
fprintf(stderr, ".");
(void)time(&curtime);
if ((elapsed = difftime(curtime, prevtime)) > 5.0) {
fprintf(stderr, "\n"
"GAP: %.0f secs after %d inserts\n",
elapsed, i);
threadargs->nfail++;
}
prevtime = curtime;
}
}
testutil_check(maincur->close(maincur));
testutil_check(session->close(session, NULL));
return (NULL);
}