/* * Create a guaranteed unique table and open and close a bulk cursor on it. */ void op_bulk_unique(void *arg) { TEST_OPTS *opts; TEST_PER_THREAD_OPTS *args; WT_CURSOR *c; WT_RAND_STATE rnd; WT_SESSION *session; int ret; char new_uri[64]; args = (TEST_PER_THREAD_OPTS *)arg; opts = args->testopts; __wt_random_init_seed(NULL, &rnd); testutil_check( opts->conn->open_session(opts->conn, NULL, NULL, &session)); /* Generate a unique object name. */ testutil_check(__wt_snprintf( new_uri, sizeof(new_uri), "%s.%" PRIu64, opts->uri, __wt_atomic_add64(&opts->unique_id, 1))); testutil_check(session->create(session, new_uri, DEFAULT_TABLE_SCHEMA)); __wt_yield(); /* * Opening a bulk cursor may have raced with a forced checkpoint * which created a checkpoint of the empty file, and triggers an EINVAL. */ if ((ret = session->open_cursor( session, new_uri, NULL, "bulk,checkpoint_wait=false", &c)) == 0) { testutil_check(c->close(c)); } else if (ret != EINVAL && ret != EBUSY) testutil_die(ret, "session.open_cursor bulk unique: %s", new_uri); while ((ret = session->drop(session, new_uri, __wt_random(&rnd) & 1 ? "force,checkpoint_wait=false" : "checkpoint_wait=false")) != 0) if (ret != EBUSY) testutil_die(ret, "session.drop: %s", new_uri); else /* * The EBUSY is expected when we run with * checkpoint_wait set to false, so we increment the * counter while in this loop to avoid false positives. */ args->thread_counter++; testutil_check(session->close(session, NULL)); args->thread_counter++; }
int workgen_random_alloc(WT_SESSION *session, workgen_random_state **rnd_state) { workgen_random_state *state; state = malloc(sizeof(workgen_random_state)); if (state == NULL) { *rnd_state = NULL; return (ENOMEM); } __wt_random_init_seed((WT_SESSION_IMPL *)session, &state->state); *rnd_state = state; return (0); }
/* * Create and drop a unique guaranteed table. */ void op_create_unique(void *arg) { TEST_OPTS *opts; TEST_PER_THREAD_OPTS *args; WT_RAND_STATE rnd; WT_SESSION *session; int ret; char new_uri[64]; args = (TEST_PER_THREAD_OPTS *)arg; opts = args->testopts; __wt_random_init_seed(NULL, &rnd); testutil_check( opts->conn->open_session(opts->conn, NULL, NULL, &session)); /* Generate a unique object name. */ testutil_check(__wt_snprintf( new_uri, sizeof(new_uri), "%s.%" PRIu64, opts->uri, __wt_atomic_add64(&opts->unique_id, 1))); testutil_check(session->create(session, new_uri, DEFAULT_TABLE_SCHEMA)); __wt_yield(); while ((ret = session->drop(session, new_uri, __wt_random(&rnd) & 1 ? "force,checkpoint_wait=false" : "checkpoint_wait=false")) != 0) if (ret != EBUSY) testutil_die(ret, "session.drop: %s", new_uri); else /* * The EBUSY is expected when we run with * checkpoint_wait set to false, so we increment the * counter while in this loop to avoid false positives. */ args->thread_counter++; testutil_check(session->close(session, NULL)); args->thread_counter++; }
void populate(TEST_OPTS *opts) { WT_CURSOR *maincur; WT_RAND_STATE rnd; WT_SESSION *session; uint32_t key; int balance, i, flag, post; __wt_random_init_seed(NULL, &rnd); testutil_check(opts->conn->open_session( opts->conn, NULL, NULL, &session)); testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &maincur)); for (i = 0; i < N_INSERT; i++) { testutil_check(session->begin_transaction(session, NULL)); key = (__wt_random(&rnd) % (N_RECORDS)); maincur->set_key(maincur, key); if (__wt_random(&rnd) % 11 == 0) post = 54321; else post = i % 100000; if (__wt_random(&rnd) % 4 == 0) { balance = -100; flag = 1; } else { balance = 100 * (i + 1); flag = 0; } maincur->set_value(maincur, post, balance, flag, key); testutil_check(maincur->insert(maincur)); testutil_check(session->commit_transaction(session, NULL)); } testutil_check(maincur->close(maincur)); testutil_check(session->close(session, NULL)); }
static void * wthread(void *arg) { WT_CURSOR *cursor_list[10]; WT_RAND_STATE rnd; WT_SESSION *session; u_int next; (void)arg; memset(cursor_list, 0, sizeof(cursor_list)); testutil_check(conn->open_session(conn, NULL, NULL, &session)); __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); for (next = 0; !done;) { if (++next == WT_ELEMENTS(cursor_list)) next = 0; op(session, &rnd, &cursor_list[next]); } return (NULL); }
static void * vthread(void *arg) { WT_CURSOR *cursor_list[10]; WT_DECL_RET; WT_RAND_STATE rnd; WT_SESSION *session; u_int i, next; (void)arg; memset(cursor_list, 0, sizeof(cursor_list)); testutil_check(conn->open_session(conn, NULL, NULL, &session)); __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd); for (next = 0; !done;) { if (++next == WT_ELEMENTS(cursor_list)) next = 0; op(session, &rnd, &cursor_list[next]); while (!done) { i = __wt_random(&rnd) % uris; ret = session->verify(session, uri_list[i], NULL); if (ret == EBUSY) { (void)__wt_atomic_add64(&verify_busy, 1); continue; } testutil_check(ret); (void)__wt_atomic_add64(&verify, 1); break; } } return (NULL); }
/* * Drop a table. */ void op_drop(void *arg) { TEST_OPTS *opts; TEST_PER_THREAD_OPTS *args; WT_RAND_STATE rnd; WT_SESSION *session; int ret; args = (TEST_PER_THREAD_OPTS *)arg; opts = args->testopts; __wt_random_init_seed(NULL, &rnd); testutil_check( opts->conn->open_session(opts->conn, NULL, NULL, &session)); if ((ret = session->drop(session, opts->uri, __wt_random(&rnd) & 1 ? "force,checkpoint_wait=false" : "checkpoint_wait=false")) != 0) if (ret != ENOENT && ret != EBUSY) testutil_die(ret, "session.drop"); testutil_check(session->close(session, NULL)); args->thread_counter++; }
/* * Worker thread. Executes random operations from the set of 6. */ static WT_THREAD_RET do_ops(void *args) { WT_RAND_STATE rnd; time_t now, start; __wt_random_init_seed(NULL, &rnd); (void)time(&start); (void)time(&now); while (difftime(now, start) < RUNTIME) { switch (__wt_random(&rnd) % 6) { case 0: op_bulk(args); break; case 1: op_create(args); break; case 2: op_cursor(args); break; case 3: op_drop(args); break; case 4: op_bulk_unique(args); break; case 5: op_create_unique(args); break; } (void)time(&now); } return (WT_THREAD_RET_VALUE); }
static void * thread_insert(void *arg) { TEST_OPTS *opts; THREAD_ARGS *threadargs; WT_CURSOR *maincur; WT_RAND_STATE rnd; WT_SESSION *session; double elapsed; time_t prevtime, curtime; /* 1 second resolution is okay */ int bal, i, flag, key, post; const char *extra = S1024; threadargs = (THREAD_ARGS *)arg; opts = threadargs->testopts; testutil_check(__wt_random_init_seed(NULL, &rnd)); (void)time(&prevtime); testutil_check(opts->conn->open_session( opts->conn, NULL, NULL, &session)); testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, &maincur)); for (i = 0; i < N_INSERT; i++) { /* * Insert threads may stomp on each other's records; * that's okay. */ key = (int)(__wt_random(&rnd) % N_RECORDS); testutil_check(session->begin_transaction(session, NULL)); maincur->set_key(maincur, key); if (__wt_random(&rnd) % 2 == 0) post = 54321; else post = i % 100000; if (__wt_random(&rnd) % 2 == 0) { bal = -100; flag = 1; } else { bal = 100 * (i + 1); flag = 0; } maincur->set_value(maincur, post, bal, extra, flag, key); testutil_check(maincur->insert(maincur)); testutil_check(maincur->reset(maincur)); testutil_check(session->commit_transaction(session, NULL)); if (i % 1000 == 0 && i != 0) { if (i % 10000 == 0) fprintf(stderr, "*"); else fprintf(stderr, "."); (void)time(&curtime); if ((elapsed = difftime(curtime, prevtime)) > 5.0) { fprintf(stderr, "\n" "GAP: %.0f secs after %d inserts\n", elapsed, i); threadargs->nfail++; } prevtime = curtime; } } testutil_check(maincur->close(maincur)); testutil_check(session->close(session, NULL)); return (NULL); }
int main(int argc, char *argv[]) { time_t start; int ch, onerun, reps; const char *config, *home; config = NULL; #ifdef _WIN32 g.progname = "t_format.exe"; #else if ((g.progname = strrchr(argv[0], DIR_DELIM)) == NULL) g.progname = argv[0]; else ++g.progname; #endif #if 0 /* Configure the GNU malloc for debugging. */ (void)setenv("MALLOC_CHECK_", "2", 1); #endif #if 0 /* Configure the FreeBSD malloc for debugging. */ (void)setenv("MALLOC_OPTIONS", "AJ", 1); #endif /* Track progress unless we're re-directing output to a file. */ g.c_quiet = isatty(1) ? 0 : 1; /* Set values from the command line. */ home = NULL; onerun = 0; while ((ch = __wt_getopt( g.progname, argc, argv, "1C:c:H:h:Llqrt:")) != EOF) switch (ch) { case '1': /* One run */ onerun = 1; break; case 'C': /* wiredtiger_open config */ g.config_open = __wt_optarg; break; case 'c': /* Configuration from a file */ config = __wt_optarg; break; case 'H': g.helium_mount = __wt_optarg; break; case 'h': home = __wt_optarg; break; case 'L': /* Re-direct output to a log */ /* * The -l option is a superset of -L, ignore -L if we * have already configured logging for operations. */ if (g.logging == 0) g.logging = LOG_FILE; break; case 'l': /* Turn on operation logging */ g.logging = LOG_OPS; break; case 'q': /* Quiet */ g.c_quiet = 1; break; case 'r': /* Replay a run */ g.replay = 1; break; default: usage(); } argc -= __wt_optind; argv += __wt_optind; /* Initialize the global RNG. */ testutil_check(__wt_random_init_seed(NULL, &g.rnd)); /* Set up paths. */ path_setup(home); /* If it's a replay, use the home directory's CONFIG file. */ if (g.replay) { if (config != NULL) testutil_die(EINVAL, "-c incompatible with -r"); if (access(g.home_config, R_OK) != 0) testutil_die(ENOENT, "%s", g.home_config); config = g.home_config; } /* * If we weren't given a configuration file, set values from "CONFIG", * if it exists. * * Small hack to ignore any CONFIG file named ".", that just makes it * possible to ignore any local CONFIG file, used when running checks. */ if (config == NULL && access("CONFIG", R_OK) == 0) config = "CONFIG"; if (config != NULL && strcmp(config, ".") != 0) config_file(config); /* * The rest of the arguments are individual configurations that modify * the base configuration. */ for (; *argv != NULL; ++argv) config_single(*argv, 1); /* * Multithreaded runs can be replayed: it's useful and we'll get the * configuration correct. Obviously the order of operations changes, * warn the user. */ if (g.replay && !SINGLETHREADED) printf("Warning: replaying a threaded run\n"); /* * Single-threaded runs historically exited after a single replay, which * makes sense when you're debugging, leave that semantic in place. */ if (g.replay && SINGLETHREADED) g.c_runs = 1; /* * Let the command line -1 flag override runs configured from other * sources. */ if (onerun) g.c_runs = 1; /* * Initialize locks to single-thread named checkpoints and backups, last * last-record updates, and failures. */ testutil_check(pthread_rwlock_init(&g.append_lock, NULL)); testutil_check(pthread_rwlock_init(&g.backup_lock, NULL)); testutil_check(pthread_rwlock_init(&g.checkpoint_lock, NULL)); testutil_check(pthread_rwlock_init(&g.death_lock, NULL)); printf("%s: process %" PRIdMAX "\n", g.progname, (intmax_t)getpid()); while (++g.run_cnt <= g.c_runs || g.c_runs == 0 ) { startup(); /* Start a run */ config_setup(); /* Run configuration */ config_print(0); /* Dump run configuration */ key_len_setup(); /* Setup keys */ start = time(NULL); track("starting up", 0ULL, NULL); #ifdef HAVE_BERKELEY_DB if (SINGLETHREADED) bdb_open(); /* Initial file config */ #endif wts_open(g.home, true, &g.wts_conn); wts_init(); wts_load(); /* Load initial records */ wts_verify("post-bulk verify"); /* Verify */ /* * If we're not doing any operations, scan the bulk-load, copy * the statistics and we're done. Otherwise, loop reading and * operations, with a verify after each set. */ if (g.c_timer == 0 && g.c_ops == 0) { wts_read_scan(); /* Read scan */ wts_stats(); /* Statistics */ } else for (reps = 1; reps <= FORMAT_OPERATION_REPS; ++reps) { wts_read_scan(); /* Read scan */ /* Operations */ wts_ops(reps == FORMAT_OPERATION_REPS); /* * Copy out the run's statistics after the last * set of operations. * * XXX * Verify closes the underlying handle and * discards the statistics, read them first. */ if (reps == FORMAT_OPERATION_REPS) wts_stats(); /* Verify */ wts_verify("post-ops verify"); } track("shutting down", 0ULL, NULL); #ifdef HAVE_BERKELEY_DB if (SINGLETHREADED) bdb_close(); #endif wts_close(); /* * Rebalance testing. */ wts_rebalance(); /* * If single-threaded, we can dump and compare the WiredTiger * and Berkeley DB data sets. */ if (SINGLETHREADED) wts_dump("standard", 1); /* * Salvage testing. */ wts_salvage(); /* Overwrite the progress line with a completion line. */ if (!g.c_quiet) printf("\r%78s\r", " "); printf("%4d: %s, %s (%.0f seconds)\n", g.run_cnt, g.c_data_source, g.c_file_type, difftime(time(NULL), start)); fflush(stdout); } /* Flush/close any logging information. */ fclose_and_clear(&g.logfp); fclose_and_clear(&g.randfp); config_print(0); testutil_check(pthread_rwlock_destroy(&g.append_lock)); testutil_check(pthread_rwlock_destroy(&g.backup_lock)); testutil_check(pthread_rwlock_destroy(&g.checkpoint_lock)); testutil_check(pthread_rwlock_destroy(&g.death_lock)); config_clear(); return (EXIT_SUCCESS); }
int main(int argc, char *argv[]) { struct stat sb; FILE *fp; WT_CONNECTION *conn; WT_CURSOR *cur_coll, *cur_local, *cur_oplog, *cur_stable; WT_RAND_STATE rnd; WT_SESSION *session; pid_t pid; uint64_t absent_coll, absent_local, absent_oplog, count, key, last_key; uint64_t first_miss, middle_coll, middle_local, middle_oplog; uint64_t stable_fp, stable_val, val[MAX_TH+1]; uint32_t i, nth, timeout; int ch, status, ret; const char *working_dir; char buf[512], fname[64], kname[64], statname[1024]; bool fatal, rand_th, rand_time, verify_only; (void)testutil_set_progname(argv); compat = inmem = false; use_ts = true; nth = MIN_TH; rand_th = rand_time = true; timeout = MIN_TIME; verify_only = false; working_dir = "WT_TEST.timestamp-abort"; while ((ch = __wt_getopt(progname, argc, argv, "Ch:mT:t:vz")) != EOF) switch (ch) { case 'C': compat = true; break; case 'h': working_dir = __wt_optarg; break; case 'm': inmem = true; break; case 'T': rand_th = false; nth = (uint32_t)atoi(__wt_optarg); break; case 't': rand_time = false; timeout = (uint32_t)atoi(__wt_optarg); break; case 'v': verify_only = true; break; case 'z': use_ts = false; break; default: usage(); } argc -= __wt_optind; argv += __wt_optind; if (argc != 0) usage(); testutil_work_dir_from_path(home, sizeof(home), working_dir); /* * If the user wants to verify they need to tell us how many threads * there were so we can find the old record files. */ if (verify_only && rand_th) { fprintf(stderr, "Verify option requires specifying number of threads\n"); exit (EXIT_FAILURE); } if (!verify_only) { testutil_make_work_dir(home); __wt_random_init_seed(NULL, &rnd); if (rand_time) { timeout = __wt_random(&rnd) % MAX_TIME; if (timeout < MIN_TIME) timeout = MIN_TIME; } if (rand_th) { nth = __wt_random(&rnd) % MAX_TH; if (nth < MIN_TH) nth = MIN_TH; } printf("Parent: compatibility: %s, " "in-mem log sync: %s, timestamp in use: %s\n", compat ? "true" : "false", inmem ? "true" : "false", use_ts ? "true" : "false"); printf("Parent: Create %" PRIu32 " threads; sleep %" PRIu32 " seconds\n", nth, timeout); /* * Fork a child to insert as many items. We will then randomly * kill the child, run recovery and make sure all items we wrote * exist after recovery runs. */ testutil_checksys((pid = fork()) < 0); if (pid == 0) { /* child */ run_workload(nth); return (EXIT_SUCCESS); } /* parent */ /* * Sleep for the configured amount of time before killing * the child. Start the timeout from the time we notice that * the file has been created. That allows the test to run * correctly on really slow machines. Verify the process ID * still exists in case the child aborts for some reason we * don't stay in this loop forever. */ testutil_check(__wt_snprintf( statname, sizeof(statname), "%s/%s", home, ckpt_file)); while (stat(statname, &sb) != 0 && kill(pid, 0) == 0) sleep(1); sleep(timeout); /* * !!! It should be plenty long enough to make sure more than * one log file exists. If wanted, that check would be added * here. */ printf("Kill child\n"); testutil_checksys(kill(pid, SIGKILL) != 0); testutil_checksys(waitpid(pid, &status, 0) == -1); } /* * !!! If we wanted to take a copy of the directory before recovery, * this is the place to do it. */ if (chdir(home) != 0) testutil_die(errno, "parent chdir: %s", home); testutil_check(__wt_snprintf(buf, sizeof(buf), "rm -rf ../%s.SAVE && mkdir ../%s.SAVE && " "cp -p WiredTigerLog.* ../%s.SAVE", home, home, home)); (void)system(buf); printf("Open database, run recovery and verify content\n"); /* * Open the connection which forces recovery to be run. */ if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn)) != 0) testutil_die(ret, "wiredtiger_open"); if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) testutil_die(ret, "WT_CONNECTION:open_session"); /* * Open a cursor on all the tables. */ if ((ret = session->open_cursor(session, uri_collection, NULL, NULL, &cur_coll)) != 0) testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_collection); if ((ret = session->open_cursor(session, uri_local, NULL, NULL, &cur_local)) != 0) testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_local); if ((ret = session->open_cursor(session, uri_oplog, NULL, NULL, &cur_oplog)) != 0) testutil_die(ret, "WT_SESSION.open_cursor: %s", uri_oplog); if ((ret = session->open_cursor(session, stable_store, NULL, NULL, &cur_stable)) != 0) testutil_die(ret, "WT_SESSION.open_cursor: %s", stable_store); /* * Find the biggest stable timestamp value that was saved. */ stable_val = 0; memset(val, 0, sizeof(val)); while (cur_stable->next(cur_stable) == 0) { cur_stable->get_key(cur_stable, &key); cur_stable->get_value(cur_stable, &val[key]); if (val[key] > stable_val) stable_val = val[key]; if (use_ts) printf("Stable: key %" PRIu64 " value %" PRIu64 "\n", key, val[key]); } if (use_ts) printf("Got stable_val %" PRIu64 "\n", stable_val); count = 0; absent_coll = absent_local = absent_oplog = 0; fatal = false; for (i = 1; i <= nth; ++i) { first_miss = middle_coll = middle_local = middle_oplog = 0; testutil_check(__wt_snprintf( fname, sizeof(fname), RECORDS_FILE, i)); if ((fp = fopen(fname, "r")) == NULL) testutil_die(errno, "fopen: %s", fname); /* * For every key in the saved file, verify that the key exists * in the table after recovery. If we're doing in-memory * log buffering we never expect a record missing in the middle, * but records may be missing at the end. If we did * write-no-sync, we expect every key to have been recovered. */ for (last_key = UINT64_MAX;; ++count, last_key = key) { ret = fscanf(fp, "%" SCNu64 "%" SCNu64 "\n", &stable_fp, &key); if (ret != EOF && ret != 2) { /* * If we find a partial line, consider it * like an EOF. */ if (ret == 1 || ret == 0) break; testutil_die(errno, "fscanf"); } if (ret == EOF) break; /* * If we're unlucky, the last line may be a partially * written key at the end that can result in a false * negative error for a missing record. Detect it. */ if (last_key != UINT64_MAX && key != last_key + 1) { printf("%s: Ignore partial record %" PRIu64 " last valid key %" PRIu64 "\n", fname, key, last_key); break; } testutil_check(__wt_snprintf( kname, sizeof(kname), "%" PRIu64, key)); cur_coll->set_key(cur_coll, kname); cur_local->set_key(cur_local, kname); cur_oplog->set_key(cur_oplog, kname); /* * The collection table should always only have the * data as of the checkpoint. */ if ((ret = cur_coll->search(cur_coll)) != 0) { if (ret != WT_NOTFOUND) testutil_die(ret, "search"); /* * If we don't find a record, the stable * timestamp written to our file better be * larger than the saved one. */ if (!inmem && stable_fp != 0 && stable_fp <= val[i]) { printf("%s: COLLECTION no record with " "key %" PRIu64 " record ts %" PRIu64 " <= stable ts %" PRIu64 "\n", fname, key, stable_fp, val[i]); absent_coll++; } if (middle_coll == 0) first_miss = key; middle_coll = key; } else if (middle_coll != 0) { /* * We should never find an existing key after * we have detected one missing. */ printf("%s: COLLECTION after absent records %" PRIu64 "-%" PRIu64 " key %" PRIu64 " exists\n", fname, first_miss, middle_coll, key); fatal = true; } /* * The local table should always have all data. */ if ((ret = cur_local->search(cur_local)) != 0) { if (ret != WT_NOTFOUND) testutil_die(ret, "search"); if (!inmem) printf("%s: LOCAL no record with key %" PRIu64 "\n", fname, key); absent_local++; middle_local = key; } else if (middle_local != 0) { /* * We should never find an existing key after * we have detected one missing. */ printf("%s: LOCAL after absent record at %" PRIu64 " key %" PRIu64 " exists\n", fname, middle_local, key); fatal = true; } /* * The oplog table should always have all data. */ if ((ret = cur_oplog->search(cur_oplog)) != 0) { if (ret != WT_NOTFOUND) testutil_die(ret, "search"); if (!inmem) printf("%s: OPLOG no record with key %" PRIu64 "\n", fname, key); absent_oplog++; middle_oplog = key; } else if (middle_oplog != 0) { /* * We should never find an existing key after * we have detected one missing. */ printf("%s: OPLOG after absent record at %" PRIu64 " key %" PRIu64 " exists\n", fname, middle_oplog, key); fatal = true; } } testutil_checksys(fclose(fp) != 0); } if ((ret = conn->close(conn, NULL)) != 0) testutil_die(ret, "WT_CONNECTION:close"); if (fatal) return (EXIT_FAILURE); if (!inmem && absent_coll) { printf("COLLECTION: %" PRIu64 " record(s) absent from %" PRIu64 "\n", absent_coll, count); fatal = true; } if (!inmem && absent_local) { printf("LOCAL: %" PRIu64 " record(s) absent from %" PRIu64 "\n", absent_local, count); fatal = true; } if (!inmem && absent_oplog) { printf("OPLOG: %" PRIu64 " record(s) absent from %" PRIu64 "\n", absent_oplog, count); fatal = true; } if (fatal) return (EXIT_FAILURE); printf("%" PRIu64 " records verified\n", count); return (EXIT_SUCCESS); }
int main(int argc, char *argv[]) { enum { CACHE_SHARED, CACHE_SET, CACHE_NONE } cache; TEST_OPTS *opts, _opts; WT_RAND_STATE rnd; WT_SESSION *session; size_t len; u_int i, j; const char *p; char *config; opts = &_opts; memset(opts, 0, sizeof(*opts)); opts->table_type = TABLE_ROW; testutil_check(testutil_parse_opts(argc, argv, opts)); testutil_make_work_dir(opts->home); testutil_check( wiredtiger_open(opts->home, &event_handler, "create", &opts->conn)); /* Open an LSM file so the LSM reconfiguration options make sense. */ testutil_check( opts->conn->open_session(opts->conn, NULL, NULL, &session)); testutil_check(session->create( session, opts->uri, "type=lsm,key_format=S,value_format=S")); /* Initialize the RNG. */ __wt_random_init_seed(NULL, &rnd); /* Allocate memory for the config. */ len = WT_ELEMENTS(list) * 64; config = dmalloc(len); /* Set an alarm so we can debug hangs. */ (void)signal(SIGALRM, on_alarm); /* A linear pass through the list. */ for (i = 0; i < WT_ELEMENTS(list); ++i) reconfig(opts, session, list[i]); /* * A linear pass through the list, adding random elements. * * WiredTiger configurations are usually "the last one set wins", but * "shared_cache" and "cache_set" options aren't allowed in the same * configuration string. */ for (i = 0; i < WT_ELEMENTS(list); ++i) { p = list[i]; cache = CACHE_NONE; if (WT_PREFIX_MATCH(p, ",shared_cache")) cache = CACHE_SHARED; else if (WT_PREFIX_MATCH(p, ",cache_size")) cache = CACHE_SET; strcpy(config, p); for (j = (__wt_random(&rnd) % WT_ELEMENTS(list)) + 1; j > 0; --j) { p = list[__wt_random(&rnd) % WT_ELEMENTS(list)]; if (WT_PREFIX_MATCH(p, ",shared_cache")) { if (cache == CACHE_SET) continue; cache = CACHE_SHARED; } else if (WT_PREFIX_MATCH(p, ",cache_size")) { if (cache == CACHE_SHARED) continue; cache = CACHE_SET; } strcat(config, p); } reconfig(opts, session, config); } /* * Turn on-close statistics off, if on-close is on and statistics were * randomly turned off during the run, close would fail. */ testutil_check(opts->conn->reconfigure( opts->conn, "statistics_log=(on_close=0)")); free(config); testutil_cleanup(opts); return (EXIT_SUCCESS); }
/* * subtest_populate -- * Populate the tables. */ static void subtest_populate(TEST_OPTS *opts, bool close_test) { WT_CURSOR *maincur, *maincur2; WT_RAND_STATE rnd; WT_SESSION *session; uint64_t i, nrecords; uint32_t rndint; int key, v0, v1, v2; char *big, *bigref; bool failed; failed = false; __wt_random_init_seed(NULL, &rnd); CHECK(create_big_string(&bigref), false); nrecords = opts->nrecords; CHECK(opts->conn->open_session( opts->conn, NULL, NULL, &session), false); CHECK(session->open_cursor(session, "table:subtest", NULL, NULL, &maincur), false); CHECK(session->open_cursor(session, "table:subtest2", NULL, NULL, &maincur2), false); for (i = 0; i < nrecords && !failed; i++) { rndint = __wt_random(&rnd); generate_key(i, &key); generate_value(rndint, i, bigref, &v0, &v1, &v2, &big); CHECK(session->begin_transaction(session, NULL), false); maincur->set_key(maincur, key); maincur->set_value(maincur, v0, v1, v2, big); CHECK(maincur->insert(maincur), false); maincur2->set_key(maincur2, key); maincur2->set_value(maincur2, rndint); CHECK(maincur2->insert(maincur2), false); CHECK(session->commit_transaction(session, NULL), false); if (i == 0) /* * Force an initial checkpoint, that helps to * distinguish a clear failure from just not running * long enough. */ CHECK(session->checkpoint(session, NULL), false); if ((i + 1) % VERBOSE_PRINT == 0 && opts->verbose) printf(" %" PRIu64 "/%" PRIu64 "\n", (i + 1), nrecords); /* Attempt to isolate the failures to checkpointing. */ if (i == (nrecords/100)) { enable_failures(opts->nops, 1000000); /* CHECK should expect failures. */ CHECK(session->checkpoint(session, NULL), true); disable_failures(); if (failed && opts->verbose) printf("checkpoint failed (expected).\n"); } } /* * Closing handles after an extreme fail is likely to cause * cascading failures (or crashes), so recommended practice is * to immediately exit. We're interested in testing both with * and without the recommended practice. */ if (failed) { if (!close_test) { fprintf(stderr, "exit early.\n"); exit(0); } else fprintf(stderr, "closing after failure.\n"); } free(bigref); CHECK(maincur->close(maincur), false); CHECK(maincur2->close(maincur2), false); CHECK(session->close(session, NULL), false); }
int main(int argc, char *argv[]) { struct sigaction sa; struct stat sb; FILE *fp; REPORT c_rep[MAX_TH], l_rep[MAX_TH], o_rep[MAX_TH]; WT_CONNECTION *conn; WT_CURSOR *cur_coll, *cur_local, *cur_oplog; WT_RAND_STATE rnd; WT_SESSION *session; pid_t pid; uint64_t absent_coll, absent_local, absent_oplog, count, key, last_key; uint64_t stable_fp, stable_val; uint32_t i, nth, timeout; int ch, status, ret; const char *working_dir; char buf[512], fname[64], kname[64], statname[1024]; char ts_string[WT_TS_HEX_STRING_SIZE]; bool fatal, rand_th, rand_time, verify_only; (void)testutil_set_progname(argv); compat = inmem = false; use_ts = true; nth = MIN_TH; rand_th = rand_time = true; timeout = MIN_TIME; verify_only = false; working_dir = "WT_TEST.timestamp-abort"; while ((ch = __wt_getopt(progname, argc, argv, "Ch:LmT:t:vz")) != EOF) switch (ch) { case 'C': compat = true; break; case 'h': working_dir = __wt_optarg; break; case 'L': table_pfx = "lsm"; break; case 'm': inmem = true; break; case 'T': rand_th = false; nth = (uint32_t)atoi(__wt_optarg); if (nth > MAX_TH) { fprintf(stderr, "Number of threads is larger than the" " maximum %" PRId32 "\n", MAX_TH); return (EXIT_FAILURE); } break; case 't': rand_time = false; timeout = (uint32_t)atoi(__wt_optarg); break; case 'v': verify_only = true; break; case 'z': use_ts = false; break; default: usage(); } argc -= __wt_optind; if (argc != 0) usage(); testutil_work_dir_from_path(home, sizeof(home), working_dir); testutil_check(pthread_rwlock_init(&ts_lock, NULL)); /* * If the user wants to verify they need to tell us how many threads * there were so we can find the old record files. */ if (verify_only && rand_th) { fprintf(stderr, "Verify option requires specifying number of threads\n"); exit (EXIT_FAILURE); } if (!verify_only) { testutil_make_work_dir(home); __wt_random_init_seed(NULL, &rnd); if (rand_time) { timeout = __wt_random(&rnd) % MAX_TIME; if (timeout < MIN_TIME) timeout = MIN_TIME; } if (rand_th) { nth = __wt_random(&rnd) % MAX_TH; if (nth < MIN_TH) nth = MIN_TH; } printf("Parent: compatibility: %s, " "in-mem log sync: %s, timestamp in use: %s\n", compat ? "true" : "false", inmem ? "true" : "false", use_ts ? "true" : "false"); printf("Parent: Create %" PRIu32 " threads; sleep %" PRIu32 " seconds\n", nth, timeout); printf("CONFIG: %s%s%s%s -h %s -T %" PRIu32 " -t %" PRIu32 "\n", progname, compat ? " -C" : "", inmem ? " -m" : "", !use_ts ? " -z" : "", working_dir, nth, timeout); /* * Fork a child to insert as many items. We will then randomly * kill the child, run recovery and make sure all items we wrote * exist after recovery runs. */ memset(&sa, 0, sizeof(sa)); sa.sa_handler = handler; testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); testutil_checksys((pid = fork()) < 0); if (pid == 0) { /* child */ run_workload(nth); return (EXIT_SUCCESS); } /* parent */ /* * Sleep for the configured amount of time before killing * the child. Start the timeout from the time we notice that * the file has been created. That allows the test to run * correctly on really slow machines. */ testutil_check(__wt_snprintf( statname, sizeof(statname), "%s/%s", home, ckpt_file)); while (stat(statname, &sb) != 0) testutil_sleep_wait(1, pid); sleep(timeout); sa.sa_handler = SIG_DFL; testutil_checksys(sigaction(SIGCHLD, &sa, NULL)); /* * !!! It should be plenty long enough to make sure more than * one log file exists. If wanted, that check would be added * here. */ printf("Kill child\n"); testutil_checksys(kill(pid, SIGKILL) != 0); testutil_checksys(waitpid(pid, &status, 0) == -1); } /* * !!! If we wanted to take a copy of the directory before recovery, * this is the place to do it. Don't do it all the time because * it can use a lot of disk space, which can cause test machine * issues. */ if (chdir(home) != 0) testutil_die(errno, "parent chdir: %s", home); /* * The tables can get very large, so while we'd ideally like to * copy the entire database, we only copy the log files for now. * Otherwise it can take far too long to run the test, particularly * in automated testing. */ testutil_check(__wt_snprintf(buf, sizeof(buf), "rm -rf ../%s.SAVE && mkdir ../%s.SAVE && " "cp -p * ../%s.SAVE", home, home, home)); if ((status = system(buf)) < 0) testutil_die(status, "system: %s", buf); printf("Open database, run recovery and verify content\n"); /* * Open the connection which forces recovery to be run. */ testutil_check(wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn)); testutil_check(conn->open_session(conn, NULL, NULL, &session)); /* * Open a cursor on all the tables. */ testutil_check(__wt_snprintf( buf, sizeof(buf), "%s:%s", table_pfx, uri_collection)); testutil_check(session->open_cursor(session, buf, NULL, NULL, &cur_coll)); testutil_check(__wt_snprintf( buf, sizeof(buf), "%s:%s", table_pfx, uri_local)); testutil_check(session->open_cursor(session, buf, NULL, NULL, &cur_local)); testutil_check(__wt_snprintf( buf, sizeof(buf), "%s:%s", table_pfx, uri_oplog)); testutil_check(session->open_cursor(session, buf, NULL, NULL, &cur_oplog)); /* * Find the biggest stable timestamp value that was saved. */ stable_val = 0; if (use_ts) { testutil_check( conn->query_timestamp(conn, ts_string, "get=recovery")); testutil_assert( sscanf(ts_string, "%" SCNx64, &stable_val) == 1); printf("Got stable_val %" PRIu64 "\n", stable_val); } count = 0; absent_coll = absent_local = absent_oplog = 0; fatal = false; for (i = 0; i < nth; ++i) { initialize_rep(&c_rep[i]); initialize_rep(&l_rep[i]); initialize_rep(&o_rep[i]); testutil_check(__wt_snprintf( fname, sizeof(fname), RECORDS_FILE, i)); if ((fp = fopen(fname, "r")) == NULL) testutil_die(errno, "fopen: %s", fname); /* * For every key in the saved file, verify that the key exists * in the table after recovery. If we're doing in-memory * log buffering we never expect a record missing in the middle, * but records may be missing at the end. If we did * write-no-sync, we expect every key to have been recovered. */ for (last_key = INVALID_KEY;; ++count, last_key = key) { ret = fscanf(fp, "%" SCNu64 "%" SCNu64 "\n", &stable_fp, &key); if (last_key == INVALID_KEY) { c_rep[i].first_key = key; l_rep[i].first_key = key; o_rep[i].first_key = key; } if (ret != EOF && ret != 2) { /* * If we find a partial line, consider it * like an EOF. */ if (ret == 1 || ret == 0) break; testutil_die(errno, "fscanf"); } if (ret == EOF) break; /* * If we're unlucky, the last line may be a partially * written key at the end that can result in a false * negative error for a missing record. Detect it. */ if (last_key != INVALID_KEY && key != last_key + 1) { printf("%s: Ignore partial record %" PRIu64 " last valid key %" PRIu64 "\n", fname, key, last_key); break; } testutil_check(__wt_snprintf( kname, sizeof(kname), "%" PRIu64, key)); cur_coll->set_key(cur_coll, kname); cur_local->set_key(cur_local, kname); cur_oplog->set_key(cur_oplog, kname); /* * The collection table should always only have the * data as of the checkpoint. */ if ((ret = cur_coll->search(cur_coll)) != 0) { if (ret != WT_NOTFOUND) testutil_die(ret, "search"); /* * If we don't find a record, the stable * timestamp written to our file better be * larger than the saved one. */ if (!inmem && stable_fp != 0 && stable_fp <= stable_val) { printf("%s: COLLECTION no record with " "key %" PRIu64 " record ts %" PRIu64 " <= stable ts %" PRIu64 "\n", fname, key, stable_fp, stable_val); absent_coll++; } if (c_rep[i].first_miss == INVALID_KEY) c_rep[i].first_miss = key; c_rep[i].absent_key = key; } else if (c_rep[i].absent_key != INVALID_KEY && c_rep[i].exist_key == INVALID_KEY) { /* * If we get here we found a record that exists * after absent records, a hole in our data. */ c_rep[i].exist_key = key; fatal = true; } else if (!inmem && stable_fp != 0 && stable_fp > stable_val) { /* * If we found a record, the stable timestamp * written to our file better be no larger * than the checkpoint one. */ printf("%s: COLLECTION record with " "key %" PRIu64 " record ts %" PRIu64 " > stable ts %" PRIu64 "\n", fname, key, stable_fp, stable_val); fatal = true; } /* * The local table should always have all data. */ if ((ret = cur_local->search(cur_local)) != 0) { if (ret != WT_NOTFOUND) testutil_die(ret, "search"); if (!inmem) printf("%s: LOCAL no record with key %" PRIu64 "\n", fname, key); absent_local++; if (l_rep[i].first_miss == INVALID_KEY) l_rep[i].first_miss = key; l_rep[i].absent_key = key; } else if (l_rep[i].absent_key != INVALID_KEY && l_rep[i].exist_key == INVALID_KEY) { /* * We should never find an existing key after * we have detected one missing. */ l_rep[i].exist_key = key; fatal = true; } /* * The oplog table should always have all data. */ if ((ret = cur_oplog->search(cur_oplog)) != 0) { if (ret != WT_NOTFOUND) testutil_die(ret, "search"); if (!inmem) printf("%s: OPLOG no record with key %" PRIu64 "\n", fname, key); absent_oplog++; if (o_rep[i].first_miss == INVALID_KEY) o_rep[i].first_miss = key; o_rep[i].absent_key = key; } else if (o_rep[i].absent_key != INVALID_KEY && o_rep[i].exist_key == INVALID_KEY) { /* * We should never find an existing key after * we have detected one missing. */ o_rep[i].exist_key = key; fatal = true; } } c_rep[i].last_key = last_key; l_rep[i].last_key = last_key; o_rep[i].last_key = last_key; testutil_checksys(fclose(fp) != 0); print_missing(&c_rep[i], fname, "COLLECTION"); print_missing(&l_rep[i], fname, "LOCAL"); print_missing(&o_rep[i], fname, "OPLOG"); } testutil_check(conn->close(conn, NULL)); if (!inmem && absent_coll) { printf("COLLECTION: %" PRIu64 " record(s) absent from %" PRIu64 "\n", absent_coll, count); fatal = true; } if (!inmem && absent_local) { printf("LOCAL: %" PRIu64 " record(s) absent from %" PRIu64 "\n", absent_local, count); fatal = true; } if (!inmem && absent_oplog) { printf("OPLOG: %" PRIu64 " record(s) absent from %" PRIu64 "\n", absent_oplog, count); fatal = true; } testutil_check(pthread_rwlock_destroy(&ts_lock)); if (fatal) return (EXIT_FAILURE); printf("%" PRIu64 " records verified\n", count); return (EXIT_SUCCESS); }
int main(int argc, char *argv[]) { static const struct { u_int workers; u_int uris; bool cache_cursors; } runs[] = { { 1, 1, false}, { 1, 1, true}, { 8, 1, false}, { 8, 1, true}, { 16, 1, false}, { 16, 1, true}, { 16, WT_ELEMENTS(uri_list), false}, { 16, WT_ELEMENTS(uri_list), true}, {200, 100, false}, {200, 100, true}, {200, WT_ELEMENTS(uri_list), false}, {200, WT_ELEMENTS(uri_list), true}, {300, 100, false}, {300, 100, true}, {600, WT_ELEMENTS(uri_list), false}, {600, WT_ELEMENTS(uri_list), true}, }; WT_RAND_STATE rnd; u_int i, n; int ch; /* * Bypass this test for valgrind. It has a fairly low thread limit. */ if (testutil_is_flag_set("TESTUTIL_BYPASS_VALGRIND")) return (EXIT_SUCCESS); (void)testutil_set_progname(argv); __wt_random_init_seed(NULL, &rnd); while ((ch = __wt_getopt(argv[0], argc, argv, "v")) != EOF) { switch (ch) { case 'v': verbose = true; break; default: fprintf(stderr, "usage: %s [-v]\n", argv[0]); return (EXIT_FAILURE); } } (void)signal(SIGALRM, on_alarm); /* Each test in the table runs for a minute, run 5 tests at random. */ for (i = 0; i < 5; ++i) { n = __wt_random(&rnd) % WT_ELEMENTS(runs); workers = runs[n].workers; uris = runs[n].uris; run(runs[n].cache_cursors); } uri_teardown(); return (EXIT_SUCCESS); }
/* * modify_run * Run some tests: * 1. Create an initial value, a copy and a fake cursor to use with the * WiredTiger routines. Generate a set of modify vectors and apply them to * the item stored in the cursor using the modify apply API. Also apply the * same modify vector to one of the copies using a helper routine written * to test the modify API. The final value generated with the modify API * and the helper routine should match. * * 2. Use the initial value and the modified value generated above as * inputs into the calculate-modify API to generate a set of modify * vectors. Apply this generated vector to the initial value using the * modify apply API to obtain a final value. The final value generated * should match the modified value that was used as input to the * calculate-modify API. */ static void modify_run(bool verbose) { WT_CURSOR *cursor, _cursor; WT_DECL_RET; WT_ITEM *localA, _localA, *localB, _localB; size_t len; int i, j; /* Initialize the RNG. */ __wt_random_init_seed(NULL, &rnd); /* Set up replacement information. */ modify_repl_init(); /* We need three WT_ITEMs, one of them part of a fake cursor. */ localA = &_localA; memset(&_localA, 0, sizeof(_localA)); localB = &_localB; memset(&_localB, 0, sizeof(_localB)); cursor = &_cursor; memset(&_cursor, 0, sizeof(_cursor)); cursor->value_format = "u"; #define NRUNS 10000 for (i = 0; i < NRUNS; ++i) { /* Create an initial value. */ len = (size_t)(__wt_random(&rnd) % MAX_REPL_BYTES); testutil_check(__wt_buf_set(NULL, localA, modify_repl, len)); for (j = 0; j < 1000; ++j) { /* Copy the current value into the second item. */ testutil_check(__wt_buf_set( NULL, localB, localA->data, localA->size)); /* * Create a random set of modify vectors, run the * underlying library modification function, then * compare the result against our implementation * of modify. */ modify_build(); testutil_check(__wt_buf_set( NULL, &cursor->value, localA->data, localA->size)); testutil_check(__wt_modify_apply_api( NULL, cursor, entries, nentries)); slow_apply_api(localA); compare(localA, &cursor->value); /* * Call the WiredTiger function to build a modification * vector for the change, and repeat the test using the * WiredTiger modification vector, then compare results * against our implementation of modify. */ nentries = WT_ELEMENTS(entries); ret = wiredtiger_calc_modify(NULL, localB, localA, WT_MAX(localB->size, localA->size) + 100, entries, &nentries); if (ret == WT_NOTFOUND) continue; testutil_check(ret); testutil_check(__wt_buf_set( NULL, &cursor->value, localB->data, localB->size)); testutil_check(__wt_modify_apply_api( NULL, cursor, entries, nentries)); compare(localA, &cursor->value); } if (verbose) { printf("%d (%d%%)\r", i, (i * 100) / NRUNS); fflush(stdout); } } if (verbose) printf("%d (100%%)\n", i); __wt_buf_free(NULL, localA); __wt_buf_free(NULL, localB); __wt_buf_free(NULL, &cursor->value); }
int main(void) { WT_RAND_STATE rnd; size_t len; uint32_t hw, sw; u_int i, j; uint8_t *data; /* Allocate aligned memory for the data. */ data = dcalloc(DATASIZE, sizeof(uint8_t)); /* Initialize the RNG. */ testutil_check(__wt_random_init_seed(NULL, &rnd)); /* Initialize the WiredTiger library checksum functions. */ __wt_cksum_init(); /* * Some simple known checksums. */ len = 1; hw = __wt_cksum(data, len); check(hw, (uint32_t)0x527d5351, len, "nul x1: hardware"); sw = cksum_sw(data, len); check(sw, (uint32_t)0x527d5351, len, "nul x1: software"); len = 2; hw = __wt_cksum(data, len); check(hw, (uint32_t)0xf16177d2, len, "nul x2: hardware"); sw = cksum_sw(data, len); check(sw, (uint32_t)0xf16177d2, len, "nul x2: software"); len = 3; hw = __wt_cksum(data, len); check(hw, (uint32_t)0x6064a37a, len, "nul x3: hardware"); sw = cksum_sw(data, len); check(sw, (uint32_t)0x6064a37a, len, "nul x3: software"); len = 4; hw = __wt_cksum(data, len); check(hw, (uint32_t)0x48674bc7, len, "nul x4: hardware"); sw = cksum_sw(data, len); check(sw, (uint32_t)0x48674bc7, len, "nul x4: software"); len = strlen("123456789"); memcpy(data, "123456789", len); hw = __wt_cksum(data, len); check(hw, (uint32_t)0xe3069283, len, "known string #1: hardware"); sw = cksum_sw(data, len); check(sw, (uint32_t)0xe3069283, len, "known string #1: software"); len = strlen("The quick brown fox jumps over the lazy dog"); memcpy(data, "The quick brown fox jumps over the lazy dog", len); hw = __wt_cksum(data, len); check(hw, (uint32_t)0x22620404, len, "known string #2: hardware"); sw = cksum_sw(data, len); check(sw, (uint32_t)0x22620404, len, "known string #2: software"); /* * Checksums of power-of-two data chunks. */ for (i = 0, len = 512; i < 1000; ++i) { for (j = 0; j < len; ++j) data[j] = __wt_random(&rnd) & 0xff; hw = __wt_cksum(data, len); sw = cksum_sw(data, len); check(hw, sw, len, "random power-of-two"); len *= 2; if (len > DATASIZE) len = 512; } /* * Checksums of random data chunks. */ for (i = 0; i < 1000; ++i) { len = __wt_random(&rnd) % DATASIZE; for (j = 0; j < len; ++j) data[j] = __wt_random(&rnd) & 0xff; hw = __wt_cksum(data, len); sw = cksum_sw(data, len); check(hw, sw, len, "random"); } free(data); return (EXIT_SUCCESS); }