/* * Each thread inserts a set of keys into the record store database. The keys * are generated in such a way that there are large gaps in the key range. */ static void * thread_func(void *arg) { TEST_OPTS *opts; WT_CURSOR *cursor, *idx_cursor; WT_SESSION *session; uint64_t i, ins_rotor, ins_thr_idx, thr_idx, ts; uint64_t *obj_data; opts = (TEST_OPTS *)arg; thr_idx = __wt_atomic_fetch_addv64(&opts->next_threadid, 1); ts = g_ts; obj_data = dcalloc( (NR_OBJECTS/NR_THREADS + 1) * NR_FIELDS, sizeof(*obj_data)); testutil_check(opts->conn->open_session( opts->conn, NULL, NULL, &session)); testutil_check(session->open_cursor( session, opts->uri, NULL, NULL, &cursor)); testutil_check(session->open_cursor( session, "table:index", NULL, NULL, &idx_cursor)); for (ins_rotor = 1; ins_rotor < 10; ++ins_rotor) { for (ins_thr_idx = thr_idx, i = 0; ins_thr_idx < NR_OBJECTS; ins_thr_idx += NR_THREADS, i += NR_FIELDS) { testutil_check( session->begin_transaction(session, "sync=false")); cursor->set_key(cursor, ins_thr_idx << 40 | ins_rotor); cursor->set_value(cursor, ts, obj_data[i+0], obj_data[i+1], obj_data[i+2], obj_data[i+3], obj_data[i+4], obj_data[i+5], obj_data[i+6], obj_data[i+7]); testutil_check(cursor->insert(cursor)); idx_cursor->set_key( idx_cursor, ins_thr_idx << 40 | ts); idx_cursor->set_value(idx_cursor, ins_rotor); testutil_check(idx_cursor->insert(idx_cursor)); testutil_check( session->commit_transaction(session, NULL)); /* change object fields */ ++obj_data[i + ((ins_thr_idx + ins_rotor) % NR_FIELDS)]; ++obj_data[i + ((ins_thr_idx + ins_rotor + 1) % NR_FIELDS)]; ++g_ts; /* 5K updates/sec */ (void)usleep(1000000ULL * NR_THREADS / 5000); } } testutil_check(session->close(session, NULL)); free(obj_data); return (NULL); }
void WiredTigerRecoveryUnit::_txnOpen() { invariant( !_active ); WT_SESSION *s = _session->getSession(); _syncing = _syncing || awaitCommitData.numWaitingForSync.load() > 0; invariantWTOK( s->begin_transaction(s, _syncing ? "sync=true" : NULL) ); LOG(2) << "WT begin_transaction"; _timer.reset(); _active = true; }
void WiredTigerRecoveryUnit::_txnOpen(OperationContext* opCtx) { invariant( !_active ); _getTicket(opCtx); WT_SESSION *s = _session->getSession(); _syncing = _syncing || waitUntilDurableData.numWaitingForSync.load() > 0; invariantWTOK( s->begin_transaction(s, _syncing ? "sync=true" : NULL) ); LOG(2) << "WT begin_transaction"; _timer.reset(); _active = true; }
void WiredTigerKVEngine::syncSizeInfo() const { if ( !_sizeStorer ) return; try { WiredTigerSession session( _conn, -1 ); WT_SESSION* s = session.getSession(); invariantWTOK( s->begin_transaction( s, "sync=true" ) ); _sizeStorer->storeInto( &session, _sizeStorerUri ); invariantWTOK( s->commit_transaction( s, NULL ) ); } catch ( const WriteConflictException& de ) { // ignore, it means someone else is doing it } }
/* * Append to a table in a "racy" fashion - that is attempt to insert the * same record another thread is likely to also be inserting. */ void * thread_insert_race(void *arg) { TEST_OPTS *opts; WT_CONNECTION *conn; WT_CURSOR *cursor; WT_SESSION *session; uint64_t i, value; int ret; opts = (TEST_OPTS *)arg; conn = opts->conn; testutil_check(conn->open_session(conn, NULL, NULL, &session)); testutil_check(session->open_cursor( session, opts->uri, NULL, NULL, &cursor)); printf("Running insert thread\n"); for (i = 0; i < opts->nrecords; ++i) { testutil_check( session->begin_transaction(session, "isolation=snapshot")); cursor->set_key(cursor, 1); testutil_check(cursor->search(cursor)); testutil_check(cursor->get_value(cursor, &value)); cursor->set_key(cursor, 1); cursor->set_value(cursor, value + 1); if ((ret = cursor->update(cursor)) != 0) { if (ret == WT_ROLLBACK) { testutil_check(session->rollback_transaction( session, NULL)); i--; continue; } printf("Error in update: %d\n", ret); } testutil_check(session->commit_transaction(session, NULL)); if (i % 10000 == 0) { printf("insert: %" PRIu64 "\r", i); fflush(stdout); } } if (i > 10000) printf("\n"); opts->running = false; return (NULL); }
void WiredTigerRecoveryUnit::_txnOpen(OperationContext* opCtx) { invariant(!_active); _ensureSession(); WT_SESSION* s = _session->getSession(); if (_readFromMajorityCommittedSnapshot) { _majorityCommittedSnapshot = _sessionCache->snapshotManager().beginTransactionOnCommittedSnapshot(s); } else { invariantWTOK(s->begin_transaction(s, NULL)); } LOG(3) << "WT begin_transaction for snapshot id " << _mySnapshotId; _timer.reset(); _active = true; }
void WiredTigerRecoveryUnit::_txnOpen(OperationContext* opCtx) { invariant(!_active); _getTicket(opCtx); WT_SESSION* s = _session->getSession(); _syncing = _syncing || waitUntilDurableData.numWaitingForSync.load() > 0; if (_readFromMajorityCommittedSnapshot) { _sessionCache->snapshotManager().beginTransactionOnCommittedSnapshot(s, _syncing); } else { invariantWTOK(s->begin_transaction(s, _syncing ? "sync=true" : NULL)); } LOG(2) << "WT begin_transaction"; _timer.reset(); _active = true; }
void do_startTransaction() override { #if TERARK_WT_USE_TXN WT_SESSION* ses = m_session.ses; const char* txnConfig = getenv("TerarkDB_WiredtigerTransactionConfig"); if (NULL == txnConfig) { // wiredtiger 2.8.0 is not binary compatible with 2.7.0 txnConfig = "isolation=read-committed,sync=false"; } // fprintf(stderr, "INFO: %s: txnConfig=%s\n", BOOST_CURRENT_FUNCTION, txnConfig); int err = ses->begin_transaction(ses, txnConfig); if (err) { THROW_STD(invalid_argument , "ERROR: wiredtiger begin_transaction: %s" , ses->strerror(ses, err)); } #endif m_sizeDiff = 0; }
void WiredTigerRecoveryUnit::_txnOpen(OperationContext* opCtx) { invariant(!_active); _ensureSession(); // Only start a timer for transaction's lifetime if we're going to log it. if (shouldLog(kSlowTransactionSeverity)) { _timer.reset(new Timer()); } WT_SESSION* s = _session->getSession(); if (_readFromMajorityCommittedSnapshot) { _majorityCommittedSnapshot = _sessionCache->snapshotManager().beginTransactionOnCommittedSnapshot(s); } else { invariantWTOK(s->begin_transaction(s, NULL)); } LOG(3) << "WT begin_transaction for snapshot id " << _mySnapshotId; _active = true; }
int main(void) { int ret; WT_CONNECTION *conn; WT_SESSION *session; WT_CURSOR *cursor; const char *key, *value; /*! [configure cache size] */ if ((ret = wiredtiger_open(home, NULL, "create,cache_size=500M", &conn)) != 0) fprintf(stderr, "Error connecting to %s: %s\n", home, wiredtiger_strerror(ret)); /*! [configure cache size] */ /*! [create a table] */ ret = conn->open_session(conn, NULL, NULL, &session); ret = session->create(session, "table:access", "key_format=S,value_format=S"); /*! [create a table] */ /*! [transaction] */ ret = session->begin_transaction(session, "priority=100,name=mytxn"); ret = session->open_cursor(session, "config:", NULL, NULL, &cursor); while ((ret = cursor->next(cursor)) == 0) { cursor->get_key(cursor, &key); cursor->get_value(cursor, &value); printf("configuration value: %s = %s\n", key, value); } ret = session->commit_transaction(session, NULL); /*! [transaction] */ ret = conn->close(conn, NULL); return (ret); }
void populate(TEST_OPTS *opts) { WT_CURSOR *maincur; WT_RAND_STATE rnd; WT_SESSION *session; uint32_t key; int balance, i, flag, post; __wt_random_init_seed(NULL, &rnd); testutil_check(opts->conn->open_session( opts->conn, NULL, NULL, &session)); testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &maincur)); for (i = 0; i < N_INSERT; i++) { testutil_check(session->begin_transaction(session, NULL)); key = (__wt_random(&rnd) % (N_RECORDS)); maincur->set_key(maincur, key); if (__wt_random(&rnd) % 11 == 0) post = 54321; else post = i % 100000; if (__wt_random(&rnd) % 4 == 0) { balance = -100; flag = 1; } else { balance = 100 * (i + 1); flag = 0; } maincur->set_value(maincur, post, balance, flag, key); testutil_check(maincur->insert(maincur)); testutil_check(session->commit_transaction(session, NULL)); } testutil_check(maincur->close(maincur)); testutil_check(session->close(session, NULL)); }
/* * subtest_populate -- * Populate the tables. */ static void subtest_populate(TEST_OPTS *opts, bool close_test) { WT_CURSOR *maincur, *maincur2; WT_RAND_STATE rnd; WT_SESSION *session; uint64_t i, nrecords; uint32_t rndint; int key, v0, v1, v2; char *big, *bigref; bool failed; failed = false; __wt_random_init_seed(NULL, &rnd); CHECK(create_big_string(&bigref), false); nrecords = opts->nrecords; CHECK(opts->conn->open_session( opts->conn, NULL, NULL, &session), false); CHECK(session->open_cursor(session, "table:subtest", NULL, NULL, &maincur), false); CHECK(session->open_cursor(session, "table:subtest2", NULL, NULL, &maincur2), false); for (i = 0; i < nrecords && !failed; i++) { rndint = __wt_random(&rnd); generate_key(i, &key); generate_value(rndint, i, bigref, &v0, &v1, &v2, &big); CHECK(session->begin_transaction(session, NULL), false); maincur->set_key(maincur, key); maincur->set_value(maincur, v0, v1, v2, big); CHECK(maincur->insert(maincur), false); maincur2->set_key(maincur2, key); maincur2->set_value(maincur2, rndint); CHECK(maincur2->insert(maincur2), false); CHECK(session->commit_transaction(session, NULL), false); if (i == 0) /* * Force an initial checkpoint, that helps to * distinguish a clear failure from just not running * long enough. */ CHECK(session->checkpoint(session, NULL), false); if ((i + 1) % VERBOSE_PRINT == 0 && opts->verbose) printf(" %" PRIu64 "/%" PRIu64 "\n", (i + 1), nrecords); /* Attempt to isolate the failures to checkpointing. */ if (i == (nrecords/100)) { enable_failures(opts->nops, 1000000); /* CHECK should expect failures. */ CHECK(session->checkpoint(session, NULL), true); disable_failures(); if (failed && opts->verbose) printf("checkpoint failed (expected).\n"); } } /* * Closing handles after an extreme fail is likely to cause * cascading failures (or crashes), so recommended practice is * to immediately exit. We're interested in testing both with * and without the recommended practice. */ if (failed) { if (!close_test) { fprintf(stderr, "exit early.\n"); exit(0); } else fprintf(stderr, "closing after failure.\n"); } free(bigref); CHECK(maincur->close(maincur), false); CHECK(maincur2->close(maincur2), false); CHECK(session->close(session, NULL), false); }
static void transaction_ops(WT_SESSION *session_arg) { WT_CONNECTION *conn; WT_CURSOR *cursor; WT_SESSION *session; session = session_arg; conn = session->connection; /*! [transaction commit/rollback] */ /* * Cursors may be opened before or after the transaction begins, and in * either case, subsequent operations are included in the transaction. * Opening cursors before the transaction begins allows applications to * cache cursors and use them for multiple operations. */ error_check(session->open_cursor( session, "table:mytable", NULL, NULL, &cursor)); error_check(session->begin_transaction(session, NULL)); cursor->set_key(cursor, "key"); cursor->set_value(cursor, "value"); switch (cursor->update(cursor)) { case 0: /* Update success */ error_check(session->commit_transaction(session, NULL)); /* * If commit_transaction succeeds, cursors remain positioned; if * commit_transaction fails, the transaction was rolled-back and * and all cursors are reset. */ break; case WT_ROLLBACK: /* Update conflict */ default: /* Other error */ error_check(session->rollback_transaction(session, NULL)); /* The rollback_transaction call resets all cursors. */ break; } /* * Cursors remain open and may be used for multiple transactions. */ /*! [transaction commit/rollback] */ error_check(cursor->close(cursor)); /*! [transaction isolation] */ /* A single transaction configured for snapshot isolation. */ error_check(session->open_cursor( session, "table:mytable", NULL, NULL, &cursor)); error_check(session->begin_transaction(session, "isolation=snapshot")); cursor->set_key(cursor, "some-key"); cursor->set_value(cursor, "some-value"); error_check(cursor->update(cursor)); error_check(session->commit_transaction(session, NULL)); /*! [transaction isolation] */ { /*! [transaction prepare] */ /* * Prepare a transaction which guarantees a subsequent commit will * succeed. Only commit and rollback are allowed on a transaction after * it has been prepared. */ error_check(session->open_cursor( session, "table:mytable", NULL, NULL, &cursor)); error_check(session->begin_transaction(session, NULL)); cursor->set_key(cursor, "key"); cursor->set_value(cursor, "value"); error_check(session->prepare_transaction( session, "prepare_timestamp=2a")); error_check(session->commit_transaction( session, "commit_timestamp=2b")); /*! [transaction prepare] */ } /*! [session isolation configuration] */ /* Open a session configured for read-uncommitted isolation. */ error_check(conn->open_session( conn, NULL, "isolation=read-uncommitted", &session)); /*! [session isolation configuration] */ /*! [session isolation re-configuration] */ /* Re-configure a session for snapshot isolation. */ error_check(session->reconfigure(session, "isolation=snapshot")); /*! [session isolation re-configuration] */ error_check(session->close(session, NULL)); session = session_arg; { /*! [transaction pinned range] */ /* Check the transaction ID range pinned by the session handle. */ uint64_t range; error_check(session->transaction_pinned_range(session, &range)); /*! [transaction pinned range] */ } error_check(session->begin_transaction(session, NULL)); { /*! [query timestamp] */ char timestamp_buf[2 * sizeof(uint64_t) + 1]; /*! [transaction timestamp] */ error_check( session->timestamp_transaction(session, "commit_timestamp=2a")); /*! [transaction timestamp] */ error_check(session->commit_transaction(session, NULL)); error_check(conn->query_timestamp( conn, timestamp_buf, "get=all_committed")); /*! [query timestamp] */ } /*! [set commit timestamp] */ error_check(conn->set_timestamp(conn, "commit_timestamp=2a")); /*! [set commit timestamp] */ /*! [set oldest timestamp] */ error_check(conn->set_timestamp(conn, "oldest_timestamp=2a")); /*! [set oldest timestamp] */ /*! [set stable timestamp] */ error_check(conn->set_timestamp(conn, "stable_timestamp=2a")); /*! [set stable timestamp] */ /*! [rollback to stable] */ error_check(conn->rollback_to_stable(conn, NULL)); /*! [rollback to stable] */ }
static void * thread_get(void *arg) { SHARED_OPTS *sharedopts; TEST_OPTS *opts; THREAD_ARGS *threadargs; WT_CURSOR *maincur, *postcur; WT_SESSION *session; double elapsed; time_t prevtime, curtime; /* 1 second resolution is okay */ int bal, flag, key, key2, post, bal2, flag2, post2; char *extra; threadargs = (THREAD_ARGS *)arg; opts = threadargs->testopts; sharedopts = threadargs->sharedopts; (void)time(&prevtime); testutil_check(opts->conn->open_session( opts->conn, NULL, NULL, &session)); testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &maincur)); testutil_check(session->open_cursor( session, sharedopts->posturi, NULL, NULL, &postcur)); for (threadargs->njoins = 0; threadargs->done == 0; threadargs->njoins++) { testutil_check(session->begin_transaction(session, NULL)); postcur->set_key(postcur, 54321); testutil_check(postcur->search(postcur)); while (postcur->next(postcur) == 0) { testutil_check(postcur->get_key(postcur, &post)); testutil_check(postcur->get_value(postcur, &post2, &bal, &extra, &flag, &key)); testutil_assert(post == post2); if (post != 54321) break; maincur->set_key(maincur, key); testutil_check(maincur->search(maincur)); testutil_check(maincur->get_value(maincur, &post2, &bal2, &extra, &flag2, &key2)); testutil_check(maincur->reset(maincur)); testutil_assert(key == key2); testutil_assert(post == post2); testutil_assert(bal == bal2); testutil_assert(flag == flag2); testutil_assert((flag2 > 0 && bal2 < 0) || (flag2 == 0 && bal2 >= 0)); } /* * Reset the cursors, potentially allowing the insert * threads to proceed. */ testutil_check(postcur->reset(postcur)); if (threadargs->njoins % 100 == 0) fprintf(stderr, "G"); testutil_check(session->rollback_transaction(session, NULL)); (void)time(&curtime); if ((elapsed = difftime(curtime, prevtime)) > 5.0) { fprintf(stderr, "\n" "GAP: %.0f secs after %d gets\n", elapsed, threadargs->njoins); threadargs->nfail++; } prevtime = curtime; } testutil_check(postcur->close(postcur)); testutil_check(maincur->close(maincur)); testutil_check(session->close(session, NULL)); return (NULL); }
void WiredTigerSizeStorer::syncCache(bool syncToDisk) { stdx::lock_guard<stdx::mutex> cursorLock(_cursorMutex); _checkMagic(); Map myMap; { stdx::lock_guard<stdx::mutex> lk(_entriesMutex); for (Map::iterator it = _entries.begin(); it != _entries.end(); ++it) { std::string uriKey = it->first; Entry& entry = it->second; if (entry.rs) { if (entry.dataSize != entry.rs->dataSize(NULL)) { entry.dataSize = entry.rs->dataSize(NULL); entry.dirty = true; } if (entry.numRecords != entry.rs->numRecords(NULL)) { entry.numRecords = entry.rs->numRecords(NULL); entry.dirty = true; } } if (!entry.dirty) continue; myMap[uriKey] = entry; } } if (myMap.empty()) return; // Nothing to do. WT_SESSION* session = _session.getSession(); invariantWTOK(session->begin_transaction(session, syncToDisk ? "sync=true" : "")); ScopeGuard rollbacker = MakeGuard(session->rollback_transaction, session, ""); for (Map::iterator it = myMap.begin(); it != myMap.end(); ++it) { string uriKey = it->first; Entry& entry = it->second; BSONObj data; { BSONObjBuilder b; b.append("numRecords", entry.numRecords); b.append("dataSize", entry.dataSize); data = b.obj(); } LOG(2) << "WiredTigerSizeStorer::storeInto " << uriKey << " -> " << redact(data); WiredTigerItem key(uriKey.c_str(), uriKey.size()); WiredTigerItem value(data.objdata(), data.objsize()); _cursor->set_key(_cursor, key.Get()); _cursor->set_value(_cursor, value.Get()); invariantWTOK(_cursor->insert(_cursor)); } invariantWTOK(_cursor->reset(_cursor)); rollbacker.Dismiss(); invariantWTOK(session->commit_transaction(session, NULL)); { stdx::lock_guard<stdx::mutex> lk(_entriesMutex); for (Map::iterator it = _entries.begin(); it != _entries.end(); ++it) { it->second.dirty = false; } } }
int main(void) { WT_CONNECTION *wt_conn; WT_CURSOR *cursor; WT_SESSION *session; int i, record_count, ret; char cmd_buf[256], k[16], v[16]; snprintf(cmd_buf, sizeof(cmd_buf), "rm -rf %s %s && mkdir %s %s", home1, home2, home1, home2); if ((ret = system(cmd_buf)) != 0) { fprintf(stderr, "%s: failed ret %d\n", cmd_buf, ret); return (ret); } if ((ret = wiredtiger_open(home1, NULL, CONN_CONFIG, &wt_conn)) != 0) { fprintf(stderr, "Error connecting to %s: %s\n", home1, wiredtiger_strerror(ret)); return (ret); } ret = wt_conn->open_session(wt_conn, NULL, NULL, &session); ret = session->create(session, uri, "key_format=S,value_format=S"); ret = session->open_cursor(session, uri, NULL, NULL, &cursor); /* * Perform some operations with individual auto-commit transactions. */ for (record_count = 0, i = 0; i < MAX_KEYS; i++, record_count++) { snprintf(k, sizeof(k), "key%d", i); snprintf(v, sizeof(v), "value%d", i); cursor->set_key(cursor, k); cursor->set_value(cursor, v); ret = cursor->insert(cursor); } ret = session->begin_transaction(session, NULL); /* * Perform some operations within a single transaction. */ for (i = MAX_KEYS; i < MAX_KEYS+5; i++, record_count++) { snprintf(k, sizeof(k), "key%d", i); snprintf(v, sizeof(v), "value%d", i); cursor->set_key(cursor, k); cursor->set_value(cursor, v); ret = cursor->insert(cursor); } ret = session->commit_transaction(session, NULL); ret = cursor->close(cursor); /*! [log cursor printf] */ ret = session->log_printf(session, "Wrote %d records", record_count); /*! [log cursor printf] */ /* * Close and reopen the connection so that the log ends up with * a variety of records such as file sync and checkpoint. We * have archiving turned off. */ ret = wt_conn->close(wt_conn, NULL); if ((ret = wiredtiger_open(home1, NULL, CONN_CONFIG, &wt_conn)) != 0) { fprintf(stderr, "Error connecting to %s: %s\n", home1, wiredtiger_strerror(ret)); return (ret); } ret = wt_conn->open_session(wt_conn, NULL, NULL, &session); ret = simple_walk_log(session); ret = walk_log(session); ret = wt_conn->close(wt_conn, NULL); return (ret); }
/* * real_worker -- * A single worker thread that transactionally updates all tables with * consistent values. */ static int real_worker(void) { WT_CURSOR **cursors; WT_RAND_STATE rnd; WT_SESSION *session; u_int i, keyno; int j, ret, t_ret; ret = t_ret = 0; __wt_random_init(&rnd); if ((cursors = calloc( (size_t)(g.ntables), sizeof(WT_CURSOR *))) == NULL) return (log_print_err("malloc", ENOMEM, 1)); if ((ret = g.conn->open_session( g.conn, NULL, "isolation=snapshot", &session)) != 0) { (void)log_print_err("conn.open_session", ret, 1); goto err; } for (j = 0; j < g.ntables; j++) if ((ret = session->open_cursor(session, g.cookies[j].uri, NULL, NULL, &cursors[j])) != 0) { (void)log_print_err("session.open_cursor", ret, 1); goto err; } for (i = 0; i < g.nops && g.running; ++i, __wt_yield()) { if ((ret = session->begin_transaction(session, NULL)) != 0) { (void)log_print_err( "real_worker:begin_transaction", ret, 1); goto err; } keyno = __wt_random(&rnd) % g.nkeys + 1; for (j = 0; j < g.ntables; j++) { if ((ret = worker_op(cursors[j], keyno, i)) != 0) break; } if (ret == 0) { if ((ret = session->commit_transaction( session, NULL)) != 0) { (void)log_print_err( "real_worker:commit_transaction", ret, 1); goto err; } } else if (ret == WT_ROLLBACK) { if ((ret = session->rollback_transaction( session, NULL)) != 0) { (void)log_print_err( "real_worker:rollback_transaction", ret, 1); goto err; } } else { (void)log_print_err("worker op failed", ret, 1); goto err; } } err: if ((t_ret = session->close(session, NULL)) != 0 && ret == 0) { ret = t_ret; (void)log_print_err("session.close", ret, 1); } free(cursors); return (ret); }
/* * thread_run -- * Runner function for the worker threads. */ static WT_THREAD_RET thread_run(void *arg) { FILE *fp; WT_CURSOR *cur_coll, *cur_local, *cur_oplog, *cur_stable; WT_ITEM data; WT_RAND_STATE rnd; WT_SESSION *session; WT_THREAD_DATA *td; uint64_t i, stable_ts; int ret; char cbuf[MAX_VAL], lbuf[MAX_VAL], obuf[MAX_VAL]; char kname[64], tscfg[64]; __wt_random_init(&rnd); memset(cbuf, 0, sizeof(cbuf)); memset(lbuf, 0, sizeof(lbuf)); memset(obuf, 0, sizeof(obuf)); memset(kname, 0, sizeof(kname)); td = (WT_THREAD_DATA *)arg; /* * Set up the separate file for checking. */ testutil_check(__wt_snprintf(cbuf, sizeof(cbuf), RECORDS_FILE, td->id)); (void)unlink(cbuf); testutil_checksys((fp = fopen(cbuf, "w")) == NULL); /* * Set to line buffering. But that is advisory only. We've seen * cases where the result files end up with partial lines. */ __wt_stream_set_line_buffer(fp); if ((ret = td->conn->open_session(td->conn, NULL, NULL, &session)) != 0) testutil_die(ret, "WT_CONNECTION:open_session"); /* * Open a cursor to each table. */ if ((ret = session->open_cursor(session, uri_collection, NULL, NULL, &cur_coll)) != 0) testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_collection); if ((ret = session->open_cursor(session, uri_local, NULL, NULL, &cur_local)) != 0) testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_local); if ((ret = session->open_cursor(session, uri_oplog, NULL, NULL, &cur_oplog)) != 0) testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_oplog); if ((ret = session->open_cursor( session, stable_store, NULL, NULL, &cur_stable)) != 0) testutil_die(ret, "WT_SESSION.open_cursor: %s", stable_store); /* * Write our portion of the key space until we're killed. */ printf("Thread %" PRIu32 " starts at %" PRIu64 "\n", td->id, td->start); for (i = td->start; ; ++i) { if (use_ts) stable_ts = global_ts++; else stable_ts = 0; testutil_check(__wt_snprintf( kname, sizeof(kname), "%" PRIu64, i)); testutil_check(session->begin_transaction(session, NULL)); cur_coll->set_key(cur_coll, kname); cur_local->set_key(cur_local, kname); cur_oplog->set_key(cur_oplog, kname); /* * Put an informative string into the value so that it * can be viewed well in a binary dump. */ testutil_check(__wt_snprintf(cbuf, sizeof(cbuf), "COLL: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64, td->id, stable_ts, i)); testutil_check(__wt_snprintf(lbuf, sizeof(lbuf), "LOCAL: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64, td->id, stable_ts, i)); testutil_check(__wt_snprintf(obuf, sizeof(obuf), "OPLOG: thread:%" PRIu64 " ts:%" PRIu64 " key: %" PRIu64, td->id, stable_ts, i)); data.size = __wt_random(&rnd) % MAX_VAL; data.data = cbuf; cur_coll->set_value(cur_coll, &data); if ((ret = cur_coll->insert(cur_coll)) != 0) testutil_die(ret, "WT_CURSOR.insert"); data.size = __wt_random(&rnd) % MAX_VAL; data.data = obuf; cur_oplog->set_value(cur_oplog, &data); if ((ret = cur_oplog->insert(cur_oplog)) != 0) testutil_die(ret, "WT_CURSOR.insert"); if (use_ts) { testutil_check(__wt_snprintf(tscfg, sizeof(tscfg), "commit_timestamp=%" PRIx64, stable_ts)); testutil_check( session->commit_transaction(session, tscfg)); } else testutil_check( session->commit_transaction(session, NULL)); /* * Insert into the local table outside the timestamp txn. */ data.size = __wt_random(&rnd) % MAX_VAL; data.data = lbuf; cur_local->set_value(cur_local, &data); if ((ret = cur_local->insert(cur_local)) != 0) testutil_die(ret, "WT_CURSOR.insert"); /* * Every N records we will record our stable timestamp into the * stable table. That will define our threshold where we * expect to find records after recovery. */ if (i % STABLE_PERIOD == 0) { if (use_ts) { /* * Set both the oldest and stable timestamp * so that we don't need to maintain read * availability at older timestamps. */ testutil_check(__wt_snprintf( tscfg, sizeof(tscfg), "oldest_timestamp=%" PRIx64 ",stable_timestamp=%" PRIx64, stable_ts, stable_ts)); testutil_check( td->conn->set_timestamp(td->conn, tscfg)); } cur_stable->set_key(cur_stable, td->id); cur_stable->set_value(cur_stable, stable_ts); testutil_check(cur_stable->insert(cur_stable)); } /* * Save the timestamp and key separately for checking later. */ if (fprintf(fp, "%" PRIu64 " %" PRIu64 "\n", stable_ts, i) < 0) testutil_die(EIO, "fprintf"); } /* NOTREACHED */ }
static void * ops(void *arg) { TINFO *tinfo; WT_CONNECTION *conn; WT_CURSOR *cursor, *cursor_insert; WT_SESSION *session; WT_ITEM key, value; uint64_t cnt, keyno, ckpt_op, session_op, thread_ops; uint32_t op; uint8_t *keybuf, *valbuf; u_int np; int dir, insert, intxn, notfound, ret; char *ckpt_config, config[64]; tinfo = arg; conn = g.wts_conn; keybuf = valbuf = NULL; /* Set up the default key and value buffers. */ key_gen_setup(&keybuf); val_gen_setup(&valbuf); /* * Each thread does its share of the total operations, and make sure * that it's not 0 (testing runs: threads might be larger than ops). */ thread_ops = 100 + g.c_ops / g.c_threads; /* * Select the first operation where we'll create sessions and cursors, * perform checkpoint operations. */ ckpt_op = MMRAND(1, thread_ops); session_op = 0; session = NULL; cursor = cursor_insert = NULL; for (intxn = 0, cnt = 0; cnt < thread_ops; ++cnt) { if (SINGLETHREADED && cnt % 100 == 0) track("ops", 0ULL, tinfo); /* * We can't checkpoint or swap sessions/cursors while in a * transaction, resolve any running transaction. Otherwise, * reset the cursor: we may block waiting for a lock and there * is no reason to keep pages pinned. */ if (cnt == ckpt_op || cnt == session_op) { if (intxn) { if ((ret = session->commit_transaction( session, NULL)) != 0) die(ret, "session.commit_transaction"); ++tinfo->commit; intxn = 0; } else if (cursor != NULL && (ret = cursor->reset(cursor)) != 0) die(ret, "cursor.reset"); } /* Open up a new session and cursors. */ if (cnt == session_op || session == NULL || cursor == NULL) { if (session != NULL && (ret = session->close(session, NULL)) != 0) die(ret, "session.close"); if ((ret = conn->open_session( conn, NULL, NULL, &session)) != 0) die(ret, "connection.open_session"); /* * Open two cursors: one configured for overwriting and * one configured for append if we're dealing with a * column-store. * * The reason is when testing with existing records, we * don't track if a record was deleted or not, which * means we must use cursor->insert with overwriting * configured. But, in column-store files where we're * testing with new, appended records, we don't want to * have to specify the record number, which requires an * append configuration. */ if ((ret = session->open_cursor(session, g.uri, NULL, "overwrite", &cursor)) != 0) die(ret, "session.open_cursor"); if ((g.type == FIX || g.type == VAR) && (ret = session->open_cursor(session, g.uri, NULL, "append", &cursor_insert)) != 0) die(ret, "session.open_cursor"); /* Pick the next session/cursor close/open. */ session_op += SINGLETHREADED ? MMRAND(1, thread_ops) : 100 * MMRAND(1, 50); } /* Checkpoint the database. */ if (cnt == ckpt_op) { /* * LSM and data-sources don't support named checkpoints, * else 25% of the time we name the checkpoint. */ if (DATASOURCE("lsm") || DATASOURCE("kvsbdb") || DATASOURCE("memrata") || MMRAND(1, 4) == 1) ckpt_config = NULL; else { (void)snprintf(config, sizeof(config), "name=thread-%d", tinfo->id); ckpt_config = config; } /* Named checkpoints lock out hot backups */ if (ckpt_config != NULL && (ret = pthread_rwlock_wrlock(&g.backup_lock)) != 0) die(ret, "pthread_rwlock_wrlock: hot-backup lock"); if ((ret = session->checkpoint(session, ckpt_config)) != 0) die(ret, "session.checkpoint%s%s", ckpt_config == NULL ? "" : ": ", ckpt_config == NULL ? "" : ckpt_config); if (ckpt_config != NULL && (ret = pthread_rwlock_unlock(&g.backup_lock)) != 0) die(ret, "pthread_rwlock_wrlock: hot-backup lock"); /* * Pick the next checkpoint operation, try for roughly * five checkpoint operations per thread run. */ ckpt_op += MMRAND(1, thread_ops) / 5; } /* * If we're not single-threaded and we're not in a transaction, * start a transaction 80% of the time. */ if (!SINGLETHREADED && !intxn && MMRAND(1, 10) >= 8) { if ((ret = session->begin_transaction(session, NULL)) != 0) die(ret, "session.begin_transaction"); intxn = 1; } insert = notfound = 0; keyno = MMRAND(1, g.rows); key.data = keybuf; value.data = valbuf; /* * Perform some number of operations: the percentage of deletes, * inserts and writes are specified, reads are the rest. The * percentages don't have to add up to 100, a high percentage * of deletes will mean fewer inserts and writes. Modifications * are always followed by a read to confirm it worked. */ op = (uint32_t)(rng() % 100); if (op < g.c_delete_pct) { ++tinfo->remove; switch (g.type) { case ROW: /* * If deleting a non-existent record, the cursor * won't be positioned, and so can't do a next. */ if (row_remove(cursor, &key, keyno, ¬found)) goto deadlock; break; case FIX: case VAR: if (col_remove(cursor, &key, keyno, ¬found)) goto deadlock; break; } } else if (op < g.c_delete_pct + g.c_insert_pct) { ++tinfo->insert; switch (g.type) { case ROW: if (row_insert(cursor, &key, &value, keyno)) goto deadlock; insert = 1; break; case FIX: case VAR: /* * We can only append so many new records, if * we've reached that limit, update a record * instead of doing an insert. */ if (g.append_cnt >= g.append_max) goto skip_insert; /* * Reset the standard cursor so it doesn't keep * pages pinned. */ if ((ret = cursor->reset(cursor)) != 0) die(ret, "cursor.reset"); /* Insert, then reset the insert cursor. */ if (col_insert( cursor_insert, &key, &value, &keyno)) goto deadlock; if ((ret = cursor_insert->reset(cursor_insert)) != 0) die(ret, "cursor.reset"); insert = 1; break; } } else if ( op < g.c_delete_pct + g.c_insert_pct + g.c_write_pct) { ++tinfo->update; switch (g.type) { case ROW: if (row_update(cursor, &key, &value, keyno)) goto deadlock; break; case FIX: case VAR: skip_insert: if (col_update(cursor, &key, &value, keyno)) goto deadlock; break; } } else { ++tinfo->search; if (read_row(cursor, &key, keyno)) goto deadlock; continue; } /* * The cursor is positioned if we did any operation other than * insert, do a small number of next/prev cursor operations in * a random direction. */ if (!insert) { dir = (int)MMRAND(0, 1); for (np = 0; np < MMRAND(1, 8); ++np) { if (notfound) break; if (nextprev(cursor, dir, ¬found)) goto deadlock; } } /* Read the value we modified to confirm the operation. */ ++tinfo->search; if (read_row(cursor, &key, keyno)) goto deadlock; /* * If we're in the transaction, commit 40% of the time and * rollback 10% of the time. */ if (intxn) switch (MMRAND(1, 10)) { case 1: case 2: case 3: case 4: /* 40% */ if ((ret = session->commit_transaction( session, NULL)) != 0) die(ret, "session.commit_transaction"); ++tinfo->commit; intxn = 0; break; case 5: /* 10% */ if (0) { deadlock: ++tinfo->deadlock; } if ((ret = session->rollback_transaction( session, NULL)) != 0) die(ret, "session.commit_transaction"); ++tinfo->rollback; intxn = 0; break; default: break; } } if (session != NULL && (ret = session->close(session, NULL)) != 0) die(ret, "session.close"); free(keybuf); free(valbuf); tinfo->state = TINFO_COMPLETE; return (NULL); }
static void * ops(void *arg) { TINFO *tinfo; WT_CONNECTION *conn; WT_CURSOR *cursor, *cursor_insert; WT_SESSION *session; WT_ITEM key, value; uint64_t keyno, ckpt_op, session_op; uint32_t op; uint8_t *keybuf, *valbuf; u_int np; int ckpt_available, dir, insert, intxn, notfound, readonly, ret; char *ckpt_config, ckpt_name[64]; tinfo = arg; /* Initialize the per-thread random number generator. */ __wt_random_init(&tinfo->rnd); conn = g.wts_conn; keybuf = valbuf = NULL; readonly = 0; /* -Wconditional-uninitialized */ /* Set up the default key and value buffers. */ key_gen_setup(&keybuf); val_gen_setup(&tinfo->rnd, &valbuf); /* Set the first operation where we'll create sessions and cursors. */ session_op = 0; session = NULL; cursor = cursor_insert = NULL; /* Set the first operation where we'll perform checkpoint operations. */ ckpt_op = g.c_checkpoints ? mmrand(&tinfo->rnd, 100, 10000) : 0; ckpt_available = 0; for (intxn = 0; !tinfo->quit; ++tinfo->ops) { /* * We can't checkpoint or swap sessions/cursors while in a * transaction, resolve any running transaction. */ if (intxn && (tinfo->ops == ckpt_op || tinfo->ops == session_op)) { if ((ret = session->commit_transaction( session, NULL)) != 0) die(ret, "session.commit_transaction"); ++tinfo->commit; intxn = 0; } /* Open up a new session and cursors. */ if (tinfo->ops == session_op || session == NULL || cursor == NULL) { if (session != NULL && (ret = session->close(session, NULL)) != 0) die(ret, "session.close"); if ((ret = conn->open_session(conn, NULL, ops_session_config(&tinfo->rnd), &session)) != 0) die(ret, "connection.open_session"); /* * 10% of the time, perform some read-only operations * from a checkpoint. * * Skip that if we single-threaded and doing checks * against a Berkeley DB database, because that won't * work because the Berkeley DB database records won't * match the checkpoint. Also skip if we are using * LSM, because it doesn't support reads from * checkpoints. */ if (!SINGLETHREADED && !DATASOURCE("lsm") && ckpt_available && mmrand(&tinfo->rnd, 1, 10) == 1) { if ((ret = session->open_cursor(session, g.uri, NULL, ckpt_name, &cursor)) != 0) die(ret, "session.open_cursor"); /* Pick the next session/cursor close/open. */ session_op += 250; /* Checkpoints are read-only. */ readonly = 1; } else { /* * Open two cursors: one for overwriting and one * for append (if it's a column-store). * * The reason is when testing with existing * records, we don't track if a record was * deleted or not, which means we must use * cursor->insert with overwriting configured. * But, in column-store files where we're * testing with new, appended records, we don't * want to have to specify the record number, * which requires an append configuration. */ if ((ret = session->open_cursor(session, g.uri, NULL, "overwrite", &cursor)) != 0) die(ret, "session.open_cursor"); if ((g.type == FIX || g.type == VAR) && (ret = session->open_cursor(session, g.uri, NULL, "append", &cursor_insert)) != 0) die(ret, "session.open_cursor"); /* Pick the next session/cursor close/open. */ session_op += mmrand(&tinfo->rnd, 100, 5000); /* Updates supported. */ readonly = 0; } } /* Checkpoint the database. */ if (tinfo->ops == ckpt_op && g.c_checkpoints) { /* * LSM and data-sources don't support named checkpoints, * and we can't drop a named checkpoint while there's a * cursor open on it, otherwise 20% of the time name the * checkpoint. */ if (DATASOURCE("helium") || DATASOURCE("kvsbdb") || DATASOURCE("lsm") || readonly || mmrand(&tinfo->rnd, 1, 5) == 1) ckpt_config = NULL; else { (void)snprintf(ckpt_name, sizeof(ckpt_name), "name=thread-%d", tinfo->id); ckpt_config = ckpt_name; } /* Named checkpoints lock out backups */ if (ckpt_config != NULL && (ret = pthread_rwlock_wrlock(&g.backup_lock)) != 0) die(ret, "pthread_rwlock_wrlock: backup lock"); if ((ret = session->checkpoint(session, ckpt_config)) != 0) die(ret, "session.checkpoint%s%s", ckpt_config == NULL ? "" : ": ", ckpt_config == NULL ? "" : ckpt_config); if (ckpt_config != NULL && (ret = pthread_rwlock_unlock(&g.backup_lock)) != 0) die(ret, "pthread_rwlock_wrlock: backup lock"); /* Rephrase the checkpoint name for cursor open. */ if (ckpt_config == NULL) strcpy(ckpt_name, "checkpoint=WiredTigerCheckpoint"); else (void)snprintf(ckpt_name, sizeof(ckpt_name), "checkpoint=thread-%d", tinfo->id); ckpt_available = 1; /* Pick the next checkpoint operation. */ ckpt_op += mmrand(&tinfo->rnd, 5000, 20000); } /* * If we're not single-threaded and we're not in a transaction, * start a transaction 20% of the time. */ if (!SINGLETHREADED && !intxn && mmrand(&tinfo->rnd, 1, 10) >= 8) { if ((ret = session->begin_transaction(session, NULL)) != 0) die(ret, "session.begin_transaction"); intxn = 1; } insert = notfound = 0; keyno = mmrand(&tinfo->rnd, 1, (u_int)g.rows); key.data = keybuf; value.data = valbuf; /* * Perform some number of operations: the percentage of deletes, * inserts and writes are specified, reads are the rest. The * percentages don't have to add up to 100, a high percentage * of deletes will mean fewer inserts and writes. Modifications * are always followed by a read to confirm it worked. */ op = readonly ? UINT32_MAX : mmrand(&tinfo->rnd, 1, 100); if (op < g.c_delete_pct) { ++tinfo->remove; switch (g.type) { case ROW: /* * If deleting a non-existent record, the cursor * won't be positioned, and so can't do a next. */ if (row_remove(cursor, &key, keyno, ¬found)) goto deadlock; break; case FIX: case VAR: if (col_remove(cursor, &key, keyno, ¬found)) goto deadlock; break; } } else if (op < g.c_delete_pct + g.c_insert_pct) { ++tinfo->insert; switch (g.type) { case ROW: if (row_insert( tinfo, cursor, &key, &value, keyno)) goto deadlock; insert = 1; break; case FIX: case VAR: /* * We can only append so many new records, if * we've reached that limit, update a record * instead of doing an insert. */ if (g.append_cnt >= g.append_max) goto skip_insert; /* Insert, then reset the insert cursor. */ if (col_insert(tinfo, cursor_insert, &key, &value, &keyno)) goto deadlock; if ((ret = cursor_insert->reset(cursor_insert)) != 0) die(ret, "cursor.reset"); insert = 1; break; } } else if ( op < g.c_delete_pct + g.c_insert_pct + g.c_write_pct) { ++tinfo->update; switch (g.type) { case ROW: if (row_update( tinfo, cursor, &key, &value, keyno)) goto deadlock; break; case FIX: case VAR: skip_insert: if (col_update(tinfo, cursor, &key, &value, keyno)) goto deadlock; break; } } else { ++tinfo->search; if (read_row(cursor, &key, keyno)) if (intxn) goto deadlock; continue; } /* * The cursor is positioned if we did any operation other than * insert, do a small number of next/prev cursor operations in * a random direction. */ if (!insert) { dir = (int)mmrand(&tinfo->rnd, 0, 1); for (np = 0; np < mmrand(&tinfo->rnd, 1, 8); ++np) { if (notfound) break; if (nextprev(cursor, dir, ¬found)) goto deadlock; } } /* Read to confirm the operation. */ ++tinfo->search; if (read_row(cursor, &key, keyno)) goto deadlock; /* Reset the cursor: there is no reason to keep pages pinned. */ if ((ret = cursor->reset(cursor)) != 0) die(ret, "cursor.reset"); /* * If we're in the transaction, commit 40% of the time and * rollback 10% of the time. */ if (intxn) switch (mmrand(&tinfo->rnd, 1, 10)) { case 1: case 2: case 3: case 4: /* 40% */ if ((ret = session->commit_transaction( session, NULL)) != 0) die(ret, "session.commit_transaction"); ++tinfo->commit; intxn = 0; break; case 5: /* 10% */ if (0) { deadlock: ++tinfo->deadlock; } if ((ret = session->rollback_transaction( session, NULL)) != 0) die(ret, "session.rollback_transaction"); ++tinfo->rollback; intxn = 0; break; default: break; } } if (session != NULL && (ret = session->close(session, NULL)) != 0) die(ret, "session.close"); free(keybuf); free(valbuf); tinfo->state = TINFO_COMPLETE; return (NULL); }
static void * thread_insert(void *arg) { TEST_OPTS *opts; THREAD_ARGS *threadargs; WT_CURSOR *maincur; WT_RAND_STATE rnd; WT_SESSION *session; double elapsed; time_t prevtime, curtime; /* 1 second resolution is okay */ int bal, i, flag, key, post; const char *extra = S1024; threadargs = (THREAD_ARGS *)arg; opts = threadargs->testopts; testutil_check(__wt_random_init_seed(NULL, &rnd)); (void)time(&prevtime); testutil_check(opts->conn->open_session( opts->conn, NULL, NULL, &session)); testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &maincur)); for (i = 0; i < N_INSERT; i++) { /* * Insert threads may stomp on each other's records; * that's okay. */ key = (int)(__wt_random(&rnd) % N_RECORDS); testutil_check(session->begin_transaction(session, NULL)); maincur->set_key(maincur, key); if (__wt_random(&rnd) % 2 == 0) post = 54321; else post = i % 100000; if (__wt_random(&rnd) % 2 == 0) { bal = -100; flag = 1; } else { bal = 100 * (i + 1); flag = 0; } maincur->set_value(maincur, post, bal, extra, flag, key); testutil_check(maincur->insert(maincur)); testutil_check(maincur->reset(maincur)); testutil_check(session->commit_transaction(session, NULL)); if (i % 1000 == 0 && i != 0) { if (i % 10000 == 0) fprintf(stderr, "*"); else fprintf(stderr, "."); (void)time(&curtime); if ((elapsed = difftime(curtime, prevtime)) > 5.0) { fprintf(stderr, "\n" "GAP: %.0f secs after %d inserts\n", elapsed, i); threadargs->nfail++; } prevtime = curtime; } } testutil_check(maincur->close(maincur)); testutil_check(session->close(session, NULL)); return (NULL); }