/* * config_isolation -- * Isolation configuration. */ static void config_isolation(void) { CONFIG *cp; const char *cstr; /* * Isolation: choose something if isolation wasn't specified. */ cp = config_find("isolation", strlen("isolation")); if (!(cp->flags & C_PERM)) { /* Avoid "maybe uninitialized" warnings. */ switch (MMRAND(1, 4)) { case 1: cstr = "isolation=random"; break; case 2: cstr = "isolation=read-uncommitted"; break; case 3: cstr = "isolation=read-committed"; break; case 4: default: cstr = "isolation=snapshot"; break; } config_single(cstr, 0); } }
void value_gen(uint8_t *val, uint32_t *sizep, uint64_t keyno) { /* * Fixed-length records: take the low N bits from the last digit of * the record number. */ if (g.type == FIX) { switch (g.c_bitcnt) { case 8: val[0] = MMRAND(1, 0xff); break; case 7: val[0] = MMRAND(1, 0x7f); break; case 6: val[0] = MMRAND(1, 0x3f); break; case 5: val[0] = MMRAND(1, 0x1f); break; case 4: val[0] = MMRAND(1, 0x0f); break; case 3: val[0] = MMRAND(1, 0x07); break; case 2: val[0] = MMRAND(1, 0x03); break; case 1: val[0] = 1; break; } *sizep = 1; return; } /* * WiredTiger doesn't store zero-length data items in row-store files, * test that by inserting a zero-length data item every so often. * LSM doesn't support zero length items. */ if (keyno % 63 == 0 && strcmp("lsm", g.c_data_source) != 0) { val[0] = '\0'; *sizep = 0; return; } /* * Start the data with a 10-digit number. * * For row and non-repeated variable-length column-stores, change the * leading number to ensure every data item is unique. For repeated * variable-length column-stores (that is, to test run-length encoding), * use the same data value all the time. */ if ((g.type == ROW || g.type == VAR) && g.c_repeat_data_pct != 0 && MMRAND(1, 100) > g.c_repeat_data_pct) { (void)strcpy((char *)val, "DUPLICATEV"); val[10] = '/'; *sizep = val_dup_data_len; } else { (void)sprintf((char *)val, "%010" PRIu64, keyno); val[10] = '/'; *sizep = MMRAND(g.c_value_min, g.c_value_max); } }
/* * config_compression -- * Compression configuration. */ static void config_compression(void) { CONFIG *cp; const char *cstr; /* * Compression: choose something if compression wasn't specified, * otherwise confirm the appropriate shared library is available. * We don't include LZO in the test compression choices, we don't * yet have an LZO module of our own. */ cp = config_find("compression", strlen("compression")); if (!(cp->flags & C_PERM)) { cstr = "compression=none"; switch (MMRAND(1, 10)) { case 1: case 2: case 3: /* 30% */ break; case 4: case 5: /* 20% */ if (access(BZIP_PATH, R_OK) == 0) cstr = "compression=bzip"; break; case 6: /* 10% */ if (access(BZIP_PATH, R_OK) == 0) cstr = "compression=bzip-raw"; break; case 7: case 8: /* 20% */ if (access(SNAPPY_PATH, R_OK) == 0) cstr = "compression=snappy"; break; case 9: case 10: /* 20% */ if (access(ZLIB_PATH, R_OK) == 0) cstr = "compression=zlib"; break; } config_single(cstr, 0); } switch (g.c_compression_flag) { case COMPRESS_BZIP: case COMPRESS_BZIP_RAW: if (access(BZIP_PATH, R_OK) != 0) die(0, "bzip library not found or not readable"); break; case COMPRESS_LZO: if (access(LZO_PATH, R_OK) != 0) die(0, "LZO library not found or not readable"); break; case COMPRESS_SNAPPY: if (access(SNAPPY_PATH, R_OK) != 0) die(0, "snappy library not found or not readable"); break; case COMPRESS_ZLIB: if (access(ZLIB_PATH, R_OK) != 0) die(0, "zlib library not found or not readable"); break; } }
void value_gen(uint8_t *val, uint32_t *sizep, uint64_t keyno) { static const char *dup_data = "duplicate data item"; /* * Fixed-length records: take the low N bits from the last digit of * the record number. */ if (g.c_file_type == FIX) { switch (g.c_bitcnt) { case 8: val[0] = MMRAND(1, 0xff); break; case 7: val[0] = MMRAND(1, 0x7f); break; case 6: val[0] = MMRAND(1, 0x3f); break; case 5: val[0] = MMRAND(1, 0x1f); break; case 4: val[0] = MMRAND(1, 0x0f); break; case 3: val[0] = MMRAND(1, 0x07); break; case 2: val[0] = MMRAND(1, 0x03); break; case 1: val[0] = 1; break; } *sizep = 1; return; } /* * WiredTiger doesn't store zero-length data items in row-store files, * test that by inserting a zero-length data item every so often. */ if (++keyno % 63 == 0) { val[0] = '\0'; *sizep = 0; return; } /* * Start the data with a 10-digit number. * * For row and non-repeated variable-length column-stores, change the * leading number to ensure every data item is unique. For repeated * variable-length column-stores (that is, to test run-length encoding), * use the same data value all the time. */ if (g.c_file_type == VAR && g.c_repeat_data_pct != 0 && (u_int)wts_rand() % 100 > g.c_repeat_data_pct) { (void)strcpy((char *)val, dup_data); *sizep = (uint32_t)strlen(dup_data); return; } sprintf((char *)val, "%010" PRIu64, keyno); val[10] = '/'; *sizep = MMRAND(g.c_value_min, g.c_value_max); }
void key_len_setup() { size_t i; /* * The key is a variable length item with a leading 10-digit value. * Since we have to be able re-construct it from the record number * (when doing row lookups), we pre-load a set of random lengths in * a lookup table, and then use the record number to choose one of * the pre-loaded lengths. * * Fill in the random key lengths. */ for (i = 0; i < sizeof(g.key_rand_len) / sizeof(g.key_rand_len[0]); ++i) g.key_rand_len[i] = (uint16_t)MMRAND(g.c_key_min, g.c_key_max); }
/* * config_checksum -- * Checksum configuration. */ static void config_checksum(void) { CONFIG *cp; /* Choose a checksum mode if nothing was specified. */ cp = config_find("checksum", strlen("checksum")); if (!(cp->flags & C_PERM)) switch (MMRAND(1, 10)) { case 1: /* 10% */ config_single("checksum=on", 0); break; case 2: /* 10% */ config_single("checksum=off", 0); break; default: /* 80% */ config_single("checksum=uncompressed", 0); break; } }
void val_gen_setup(uint8_t **valp) { uint8_t *val; size_t i, len; /* * Set initial buffer contents to recognizable text. * * Add a few extra bytes in order to guarantee we can always offset * into the buffer by a few extra bytes, used to generate different * data for column-store run-length encoded files. */ len = g.c_value_max + 20; if ((val = malloc(len)) == NULL) syserr("malloc"); for (i = 0; i < len; ++i) val[i] = (uint8_t)("ABCDEFGHIJKLMNOPQRSTUVWXYZ"[i % 26]); *valp = val; val_dup_data_len = MMRAND(g.c_value_min, g.c_value_max); }
void key_gen(uint8_t *key, uint32_t *sizep, uint64_t keyno, int insert) { int len, suffix; /* * The key always starts with a 10-digit string (the specified cnt) * followed by two digits, a random number between 1 and 15 if it's * an insert, otherwise 00. */ suffix = insert ? (int)MMRAND(1, 15) : 0; len = sprintf((char *)key, "%010" PRIu64 ".%02d", keyno, suffix); /* * In a column-store, the key is only used for BDB, and so it doesn't * need a random length. */ if (g.c_file_type == ROW) { key[len] = '/'; len = g.key_rand_len[keyno % (sizeof(g.key_rand_len) / sizeof(g.key_rand_len[0]))]; } *sizep = (uint32_t)len; }
void wts_open(void) { WT_CONNECTION *conn; WT_SESSION *session; uint32_t maxintlpage, maxintlitem, maxleafpage, maxleafitem; int ret; const char *ext1, *ext2; char config[512], *end, *p; /* If the bzip2 compression module has been built, use it. */ #define EXTPATH "../../ext" ext1 = EXTPATH "compressors/bzip2_compress/.libs/bzip2_compress.so"; if (access(ext1, R_OK) != 0) { ext1 = ""; g.c_bzip = 0; } ext2 = EXTPATH "/collators/reverse/.libs/reverse_collator.so"; /* * Open configuration -- put command line configuration options at the * end so they can override "standard" configuration. */ snprintf(config, sizeof(config), "create,error_prefix=\"%s\",cache_size=%" PRIu32 "MB,sync=false," "extensions=[\"%s\",\"%s\"],%s", g.progname, g.c_cache, ext1, ext2, g.config_open == NULL ? "" : g.config_open); if ((ret = wiredtiger_open("RUNDIR", &event_handler, config, &conn)) != 0) die(ret, "wiredtiger_open"); if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) die(ret, "connection.open_session"); maxintlpage = 1U << g.c_intl_page_max; /* Make sure at least 2 internal page per thread can fit in cache. */ while (2 * g.c_threads * maxintlpage > g.c_cache << 20) maxintlpage >>= 1; maxintlitem = MMRAND(maxintlpage / 50, maxintlpage / 40); if (maxintlitem < 40) maxintlitem = 40; maxleafpage = 1U << g.c_leaf_page_max; /* Make sure at least one leaf page per thread can fit in cache. */ while (g.c_threads * (maxintlpage + maxleafpage) > g.c_cache << 20) maxleafpage >>= 1; maxleafitem = MMRAND(maxleafpage / 50, maxleafpage / 40); if (maxleafitem < 40) maxleafitem = 40; p = config; end = config + sizeof(config); p += snprintf(p, (size_t)(end - p), "key_format=%s," "internal_page_max=%d,internal_item_max=%d," "leaf_page_max=%d,leaf_item_max=%d", (g.type == ROW) ? "u" : "r", maxintlpage, maxintlitem, maxleafpage, maxleafitem); if (g.c_bzip) p += snprintf(p, (size_t)(end - p), ",block_compressor=\"bzip2_compress\""); switch (g.type) { case FIX: p += snprintf(p, (size_t)(end - p), ",value_format=%dt", g.c_bitcnt); break; case ROW: if (g.c_huffman_key) p += snprintf(p, (size_t)(end - p), ",huffman_key=english"); if (g.c_reverse) p += snprintf(p, (size_t)(end - p), ",collator=reverse"); /* FALLTHROUGH */ case VAR: if (g.c_huffman_value) p += snprintf(p, (size_t)(end - p), ",huffman_value=english"); if (g.c_dictionary) p += snprintf(p, (size_t)(end - p), ",dictionary=%d", MMRAND(123, 517)); break; } if ((ret = session->create(session, g.uri, config)) != 0) die(ret, "session.create: %s", g.uri); if ((ret = session->close(session, NULL)) != 0) die(ret, "session.close"); g.wts_conn = conn; }
static void * ops(void *arg) { TINFO *tinfo; WT_CONNECTION *conn; WT_CURSOR *cursor, *cursor_insert; WT_SESSION *session; WT_ITEM key, value; uint64_t cnt, keyno, sync_op, thread_ops; uint32_t op; uint8_t *keybuf, *valbuf; u_int np; int dir, insert, notfound, ret, sync_drop; char sync_name[64]; conn = g.wts_conn; tinfo = arg; /* Set up the default key and value buffers. */ memset(&key, 0, sizeof(key)); key_gen_setup(&keybuf); memset(&value, 0, sizeof(value)); val_gen_setup(&valbuf); /* Open a session. */ if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) die(ret, "connection.open_session"); /* * Open two cursors: one configured for overwriting and one configured * for append if we're dealing with a column-store. * * The reason is when testing with existing records, we don't track if * a record was deleted or not, which means we must use cursor->insert * with overwriting configured. But, in column-store files where we're * testing with new, appended records, we don't want to have to specify * the record number, which requires an append configuration. */ cursor = cursor_insert = NULL; if ((ret = session->open_cursor(session, WT_TABLENAME, NULL, "overwrite", &cursor)) != 0) die(ret, "session.open_cursor"); if ((g.c_file_type == FIX || g.c_file_type == VAR) && (ret = session->open_cursor(session, WT_TABLENAME, NULL, "append", &cursor_insert)) != 0) die(ret, "session.open_cursor"); /* Each thread does its share of the total operations. */ thread_ops = g.c_ops / g.c_threads; /* Pick an operation where we'll do a sync and create the name. */ sync_drop = 0; sync_op = MMRAND(1, thread_ops); snprintf(sync_name, sizeof(sync_name), "snapshot=thread-%d", tinfo->id); for (cnt = 0; cnt < thread_ops; ++cnt) { if (SINGLETHREADED && cnt % 100 == 0) track("read/write ops", 0ULL, tinfo); if (cnt == sync_op) { if (sync_drop && (int)MMRAND(1, 4) == 1) { if ((ret = session->drop( session, WT_TABLENAME, sync_name)) != 0) die(ret, "session.drop: %s: %s", WT_TABLENAME, sync_name); sync_drop = 0; } else { if ((ret = session->checkpoint( session, sync_name)) != 0) die(ret, "session.checkpoint: %s", sync_name); sync_drop = 1; } /* * Pick the next sync operation, try for roughly five * snapshot operations per thread run. */ sync_op += MMRAND(1, thread_ops) / 5; } insert = notfound = 0; keyno = MMRAND(1, g.rows); key.data = keybuf; value.data = valbuf; /* * Perform some number of operations: the percentage of deletes, * inserts and writes are specified, reads are the rest. The * percentages don't have to add up to 100, a high percentage * of deletes will mean fewer inserts and writes. A read * operation always follows a modification to confirm it worked. */ op = (uint32_t)(wts_rand() % 100); if (op < g.c_delete_pct) { ++tinfo->remove; switch (g.c_file_type) { case ROW: /* * If deleting a non-existent record, the cursor * won't be positioned, and so can't do a next. */ row_remove(cursor, &key, keyno, ¬found); break; case FIX: case VAR: col_remove(cursor, &key, keyno, ¬found); break; } } else if (op < g.c_delete_pct + g.c_insert_pct) { ++tinfo->insert; switch (g.c_file_type) { case ROW: row_update(cursor, &key, &value, keyno, 1); break; case FIX: case VAR: /* * Reset the standard cursor so it doesn't keep * pages pinned. */ if ((ret = cursor->reset(cursor)) != 0) die(ret, "cursor.reset"); col_insert(cursor_insert, &key, &value, &keyno); insert = 1; break; } } else if ( op < g.c_delete_pct + g.c_insert_pct + g.c_write_pct) { ++tinfo->update; switch (g.c_file_type) { case ROW: row_update(cursor, &key, &value, keyno, 0); break; case FIX: case VAR: col_update(cursor, &key, &value, keyno); break; } } else { ++tinfo->search; read_row(cursor, &key, keyno); continue; } /* * If we did any operation, we've set the cursor, do a small * number of next/prev cursor operations in a random direction. */ dir = (int)MMRAND(0, 1); for (np = 0; np < MMRAND(1, 8); ++np) { if (notfound) break; nextprev( insert ? cursor_insert : cursor, dir, ¬found); } if (insert && (ret = cursor_insert->reset(cursor_insert)) != 0) die(ret, "cursor.reset"); /* Read the value we modified to confirm the operation. */ read_row(cursor, &key, keyno); } if ((ret = session->close(session, NULL)) != 0) die(ret, "session.close"); free(keybuf); free(valbuf); tinfo->state = TINFO_COMPLETE; return (NULL); }
/* * backup -- * Periodically do a backup and verify it. */ void * backup(void *arg) { WT_CONNECTION *conn; WT_CURSOR *backup_cursor; WT_SESSION *session; u_int period; int ret; const char *key; (void)(arg); conn = g.wts_conn; /* Backups aren't supported for non-standard data sources. */ if (DATASOURCE("helium") || DATASOURCE("kvsbdb")) return (NULL); /* Open a session. */ if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) die(ret, "connection.open_session"); /* * Perform a backup at somewhere under 10 seconds (so we get at * least one done), and then at 45 second intervals. */ for (period = MMRAND(1, 10);; period = 45) { /* Sleep for short periods so we don't make the run wait. */ while (period > 0 && !g.workers_finished) { --period; sleep(1); } if (g.workers_finished) break; /* Lock out named checkpoints */ if ((ret = pthread_rwlock_wrlock(&g.backup_lock)) != 0) die(ret, "pthread_rwlock_wrlock: backup lock"); /* Re-create the backup directory. */ if ((ret = system(g.home_backup_init)) != 0) die(ret, "backup directory creation failed"); /* * open_cursor can return EBUSY if a metadata operation is * currently happening - retry in that case. */ while ((ret = session->open_cursor(session, "backup:", NULL, NULL, &backup_cursor)) == EBUSY) sleep(1); if (ret != 0) die(ret, "session.open_cursor: backup"); while ((ret = backup_cursor->next(backup_cursor)) == 0) { if ((ret = backup_cursor->get_key(backup_cursor, &key)) != 0) die(ret, "cursor.get_key"); copy_file(key); } if ((ret = backup_cursor->close(backup_cursor)) != 0) die(ret, "cursor.close"); if ((ret = pthread_rwlock_unlock(&g.backup_lock)) != 0) die(ret, "pthread_rwlock_unlock: backup lock"); check_copy(); } if ((ret = session->close(session, NULL)) != 0) die(ret, "session.close"); return (NULL); }
/* * config_setup -- * Initialize configuration for a run. */ void config_setup(void) { CONFIG *cp; /* Clear any temporary values. */ config_clear(); /* * Choose a data source type and a file type: they're interrelated (LSM * trees are only compatible with row-store) and other items depend on * them. */ if (!config_find_is_perm("data_source", strlen("data_source"))) switch (MMRAND(0, 2)) { case 0: config_single("data_source=file", 0); break; case 1: #if 0 config_single("data_source=lsm", 0); break; #endif case 2: config_single("data_source=table", 0); break; } if (!config_find_is_perm("file_type", strlen("file_type"))) { if (strcmp(g.c_data_source, "lsm") == 0) config_single("file_type=row", 0); else switch (MMRAND(0, 2)) { case 0: config_single("file_type=fix", 0); break; case 1: config_single("file_type=var", 0); break; case 2: config_single("file_type=row", 0); break; } } g.type = config_translate(g.c_file_type); /* * If data_source and file_type were both "permanent", we may still * have a mismatch. */ if (g.type != ROW && strcmp(g.c_data_source, "lsm") == 0) { fprintf(stderr, "%s: lsm data_source is only compatible with row file_type\n", g.progname); exit(EXIT_FAILURE); } /* Build the object name. */ if ((g.uri = malloc( strlen(g.c_data_source) + strlen(WT_NAME) + 2)) == NULL) syserr("malloc"); strcpy(g.uri, g.c_data_source); strcat(g.uri, ":"); strcat(g.uri, WT_NAME); /* Default single-threaded half of the time. */ cp = config_find("threads", strlen("threads")); if (!(cp->flags & C_PERM)) *cp->v = MMRAND(0, 1) ? 1: CONF_RAND(cp); /* Fill in random values for the rest of the run. */ for (cp = c; cp->name != NULL; ++cp) { if (cp->flags & (C_IGNORE | C_PERM | C_TEMP)) continue; /* * Boolean flags are 0 or 1, but only set N in 100 where the * variable's min value is N. Set the flag if we rolled >= * the min, 0 otherwise. */ if (cp->flags & C_BOOL) *cp->v = MMRAND(1, 100) <= cp->min ? 1 : 0; else *cp->v = CONF_RAND(cp); } config_compression(); /* Clear operations values if the whole run is read-only. */ if (g.c_ops == 0) for (cp = c; cp->name != NULL; ++cp) if (cp->flags & C_OPS) *cp->v = 0; /* Multi-threaded runs cannot be replayed. */ if (g.replay && !SINGLETHREADED) die(0, "-r is incompatible with threaded runs"); /* * Periodically, set the delete percentage to 0 so salvage gets run, * as long as the delete percentage isn't nailed down. */ if (!g.replay && g.run_cnt % 10 == 0) { cp = config_find("delete_pct", strlen("delete_pct")); if (cp->name != NULL && !(cp->flags & (C_IGNORE | C_PERM | C_TEMP))) g.c_delete_pct = 0; } /* Reset the key count. */ g.key_cnt = 0; }
/* * hot_backup -- * Periodically do a hot backup and verify it. */ void * hot_backup(void *arg) { WT_CONNECTION *conn; WT_CURSOR *backup_cursor; WT_SESSION *session; u_int period; int ret; const char *key; (void)arg; /* If hot backups aren't configured, we're done. */ if (!g.c_hot_backups) return (NULL); /* Hot backups aren't supported for non-standard data sources. */ if (DATASOURCE("kvsbdb") || DATASOURCE("memrata")) return (NULL); conn = g.wts_conn; /* Open a session. */ if ((ret = conn->open_session( conn, NULL, NULL, &session)) != 0) die(ret, "connection.open_session"); /* * Perform a hot backup at somewhere under 10 seconds (so we get at * least one done), and then at 45 second intervals. */ for (period = MMRAND(1, 10);; period = 45) { /* Sleep for short periods so we don't make the run wait. */ while (period > 0 && !g.threads_finished) { --period; sleep(1); } if (g.threads_finished) break; /* Lock out named checkpoints */ if ((ret = pthread_rwlock_wrlock(&g.backup_lock)) != 0) die(ret, "pthread_rwlock_wrlock: hot-backup lock"); /* Re-create the backup directory. */ (void)system("cd RUNDIR && rm -rf BACKUP"); if (mkdir(RUNDIR_BACKUP, 0777) != 0) die(errno, "mkdir: %s", RUNDIR_BACKUP); /* * open_cursor can return EBUSY if a metadata operation is * currently happening - retry in that case. */ while ((ret = session->open_cursor(session, "backup:", NULL, NULL, &backup_cursor)) == EBUSY) sleep(1); if (ret != 0) die(ret, "session.open_cursor: backup"); while ((ret = backup_cursor->next(backup_cursor)) == 0) { if ((ret = backup_cursor->get_key(backup_cursor, &key)) != 0) die(ret, "cursor.get_key"); hot_copy(key); } if ((ret = backup_cursor->close(backup_cursor)) != 0) die(ret, "cursor.close"); if ((ret = pthread_rwlock_unlock(&g.backup_lock)) != 0) die(ret, "pthread_rwlock_unlock: hot-backup lock"); check_copy(); } if ((ret = session->close(session, NULL)) != 0) die(ret, "session.close"); return (NULL); }
static void * ops(void *arg) { TINFO *tinfo; WT_CONNECTION *conn; WT_CURSOR *cursor, *cursor_insert; WT_SESSION *session; WT_ITEM key, value; uint64_t cnt, keyno, ckpt_op, session_op, thread_ops; uint32_t op; uint8_t *keybuf, *valbuf; u_int np; int dir, insert, intxn, notfound, ret; char *ckpt_config, config[64]; tinfo = arg; conn = g.wts_conn; keybuf = valbuf = NULL; /* Set up the default key and value buffers. */ key_gen_setup(&keybuf); val_gen_setup(&valbuf); /* * Each thread does its share of the total operations, and make sure * that it's not 0 (testing runs: threads might be larger than ops). */ thread_ops = 100 + g.c_ops / g.c_threads; /* * Select the first operation where we'll create sessions and cursors, * perform checkpoint operations. */ ckpt_op = MMRAND(1, thread_ops); session_op = 0; session = NULL; cursor = cursor_insert = NULL; for (intxn = 0, cnt = 0; cnt < thread_ops; ++cnt) { if (SINGLETHREADED && cnt % 100 == 0) track("ops", 0ULL, tinfo); /* * We can't checkpoint or swap sessions/cursors while in a * transaction, resolve any running transaction. Otherwise, * reset the cursor: we may block waiting for a lock and there * is no reason to keep pages pinned. */ if (cnt == ckpt_op || cnt == session_op) { if (intxn) { if ((ret = session->commit_transaction( session, NULL)) != 0) die(ret, "session.commit_transaction"); ++tinfo->commit; intxn = 0; } else if (cursor != NULL && (ret = cursor->reset(cursor)) != 0) die(ret, "cursor.reset"); } /* Open up a new session and cursors. */ if (cnt == session_op || session == NULL || cursor == NULL) { if (session != NULL && (ret = session->close(session, NULL)) != 0) die(ret, "session.close"); if ((ret = conn->open_session( conn, NULL, NULL, &session)) != 0) die(ret, "connection.open_session"); /* * Open two cursors: one configured for overwriting and * one configured for append if we're dealing with a * column-store. * * The reason is when testing with existing records, we * don't track if a record was deleted or not, which * means we must use cursor->insert with overwriting * configured. But, in column-store files where we're * testing with new, appended records, we don't want to * have to specify the record number, which requires an * append configuration. */ if ((ret = session->open_cursor(session, g.uri, NULL, "overwrite", &cursor)) != 0) die(ret, "session.open_cursor"); if ((g.type == FIX || g.type == VAR) && (ret = session->open_cursor(session, g.uri, NULL, "append", &cursor_insert)) != 0) die(ret, "session.open_cursor"); /* Pick the next session/cursor close/open. */ session_op += SINGLETHREADED ? MMRAND(1, thread_ops) : 100 * MMRAND(1, 50); } /* Checkpoint the database. */ if (cnt == ckpt_op) { /* * LSM and data-sources don't support named checkpoints, * else 25% of the time we name the checkpoint. */ if (DATASOURCE("lsm") || DATASOURCE("kvsbdb") || DATASOURCE("memrata") || MMRAND(1, 4) == 1) ckpt_config = NULL; else { (void)snprintf(config, sizeof(config), "name=thread-%d", tinfo->id); ckpt_config = config; } /* Named checkpoints lock out hot backups */ if (ckpt_config != NULL && (ret = pthread_rwlock_wrlock(&g.backup_lock)) != 0) die(ret, "pthread_rwlock_wrlock: hot-backup lock"); if ((ret = session->checkpoint(session, ckpt_config)) != 0) die(ret, "session.checkpoint%s%s", ckpt_config == NULL ? "" : ": ", ckpt_config == NULL ? "" : ckpt_config); if (ckpt_config != NULL && (ret = pthread_rwlock_unlock(&g.backup_lock)) != 0) die(ret, "pthread_rwlock_wrlock: hot-backup lock"); /* * Pick the next checkpoint operation, try for roughly * five checkpoint operations per thread run. */ ckpt_op += MMRAND(1, thread_ops) / 5; } /* * If we're not single-threaded and we're not in a transaction, * start a transaction 80% of the time. */ if (!SINGLETHREADED && !intxn && MMRAND(1, 10) >= 8) { if ((ret = session->begin_transaction(session, NULL)) != 0) die(ret, "session.begin_transaction"); intxn = 1; } insert = notfound = 0; keyno = MMRAND(1, g.rows); key.data = keybuf; value.data = valbuf; /* * Perform some number of operations: the percentage of deletes, * inserts and writes are specified, reads are the rest. The * percentages don't have to add up to 100, a high percentage * of deletes will mean fewer inserts and writes. Modifications * are always followed by a read to confirm it worked. */ op = (uint32_t)(rng() % 100); if (op < g.c_delete_pct) { ++tinfo->remove; switch (g.type) { case ROW: /* * If deleting a non-existent record, the cursor * won't be positioned, and so can't do a next. */ if (row_remove(cursor, &key, keyno, ¬found)) goto deadlock; break; case FIX: case VAR: if (col_remove(cursor, &key, keyno, ¬found)) goto deadlock; break; } } else if (op < g.c_delete_pct + g.c_insert_pct) { ++tinfo->insert; switch (g.type) { case ROW: if (row_insert(cursor, &key, &value, keyno)) goto deadlock; insert = 1; break; case FIX: case VAR: /* * We can only append so many new records, if * we've reached that limit, update a record * instead of doing an insert. */ if (g.append_cnt >= g.append_max) goto skip_insert; /* * Reset the standard cursor so it doesn't keep * pages pinned. */ if ((ret = cursor->reset(cursor)) != 0) die(ret, "cursor.reset"); /* Insert, then reset the insert cursor. */ if (col_insert( cursor_insert, &key, &value, &keyno)) goto deadlock; if ((ret = cursor_insert->reset(cursor_insert)) != 0) die(ret, "cursor.reset"); insert = 1; break; } } else if ( op < g.c_delete_pct + g.c_insert_pct + g.c_write_pct) { ++tinfo->update; switch (g.type) { case ROW: if (row_update(cursor, &key, &value, keyno)) goto deadlock; break; case FIX: case VAR: skip_insert: if (col_update(cursor, &key, &value, keyno)) goto deadlock; break; } } else { ++tinfo->search; if (read_row(cursor, &key, keyno)) goto deadlock; continue; } /* * The cursor is positioned if we did any operation other than * insert, do a small number of next/prev cursor operations in * a random direction. */ if (!insert) { dir = (int)MMRAND(0, 1); for (np = 0; np < MMRAND(1, 8); ++np) { if (notfound) break; if (nextprev(cursor, dir, ¬found)) goto deadlock; } } /* Read the value we modified to confirm the operation. */ ++tinfo->search; if (read_row(cursor, &key, keyno)) goto deadlock; /* * If we're in the transaction, commit 40% of the time and * rollback 10% of the time. */ if (intxn) switch (MMRAND(1, 10)) { case 1: case 2: case 3: case 4: /* 40% */ if ((ret = session->commit_transaction( session, NULL)) != 0) die(ret, "session.commit_transaction"); ++tinfo->commit; intxn = 0; break; case 5: /* 10% */ if (0) { deadlock: ++tinfo->deadlock; } if ((ret = session->rollback_transaction( session, NULL)) != 0) die(ret, "session.commit_transaction"); ++tinfo->rollback; intxn = 0; break; default: break; } } if (session != NULL && (ret = session->close(session, NULL)) != 0) die(ret, "session.close"); free(keybuf); free(valbuf); tinfo->state = TINFO_COMPLETE; return (NULL); }
void wts_open(void) { WT_CONNECTION *conn; WT_SESSION *session; uint32_t maxintlpage, maxintlitem, maxleafpage, maxleafitem; int ret; char config[2048], *end, *p; /* * Open configuration. * * Put configuration file configuration options second to last. Put * command line configuration options at the end. Do this so they * override the standard configuration. */ snprintf(config, sizeof(config), "create,cache_size=%" PRIu32 "MB," "error_prefix=\"%s\"," "extensions=[\"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],%s,%s," "statistics=true,statistics_log=(wait=5)," "sync=false,", g.c_cache, g.progname, REVERSE_PATH, access(BZIP_PATH, R_OK) == 0 ? BZIP_PATH : "", access(LZO_PATH, R_OK) == 0 ? LZO_PATH : "", (access(RAW_PATH, R_OK) == 0 && access(BZIP_PATH, R_OK) == 0) ? RAW_PATH : "", access(SNAPPY_PATH, R_OK) == 0 ? SNAPPY_PATH : "", g.c_config_open == NULL ? "" : g.c_config_open, g.config_open == NULL ? "" : g.config_open); if ((ret = wiredtiger_open("RUNDIR", &event_handler, config, &conn)) != 0) die(ret, "wiredtiger_open"); g.wts_conn = conn; if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) die(ret, "connection.open_session"); /* * Create the object. * * Make sure at least 2 internal page per thread can fit in cache. */ maxintlpage = 1U << g.c_intl_page_max; while (2 * g.c_threads * maxintlpage > g.c_cache << 20) maxintlpage >>= 1; maxintlitem = MMRAND(maxintlpage / 50, maxintlpage / 40); if (maxintlitem < 40) maxintlitem = 40; /* Make sure at least one leaf page per thread can fit in cache. */ maxleafpage = 1U << g.c_leaf_page_max; while (g.c_threads * (maxintlpage + maxleafpage) > g.c_cache << 20) maxleafpage >>= 1; maxleafitem = MMRAND(maxleafpage / 50, maxleafpage / 40); if (maxleafitem < 40) maxleafitem = 40; p = config; end = config + sizeof(config); p += snprintf(p, (size_t)(end - p), "key_format=%s," "internal_page_max=%d,internal_item_max=%d," "leaf_page_max=%d,leaf_item_max=%d", (g.type == ROW) ? "u" : "r", maxintlpage, maxintlitem, maxleafpage, maxleafitem); switch (g.type) { case FIX: p += snprintf(p, (size_t)(end - p), ",value_format=%dt", g.c_bitcnt); break; case ROW: if (g.c_huffman_key) p += snprintf(p, (size_t)(end - p), ",huffman_key=english"); if (!g.c_prefix) p += snprintf(p, (size_t)(end - p), ",prefix_compression=false"); if (g.c_reverse) p += snprintf(p, (size_t)(end - p), ",collator=reverse"); /* FALLTHROUGH */ case VAR: if (g.c_huffman_value) p += snprintf(p, (size_t)(end - p), ",huffman_value=english"); if (g.c_dictionary) p += snprintf(p, (size_t)(end - p), ",dictionary=%d", MMRAND(123, 517)); break; } /* Configure checksums (not configurable from the command line). */ switch MMRAND(1, 10) { case 1: /* 10% */ p += snprintf(p, (size_t)(end - p), ",checksum=\"on\""); break; case 2: /* 10% */ p += snprintf(p, (size_t)(end - p), ",checksum=\"off\""); break; default: /* 80% */ p += snprintf( p, (size_t)(end - p), ",checksum=\"uncompressed\""); break; } /* Configure compression. */ switch (g.compression) { case COMPRESS_NONE: break; case COMPRESS_BZIP: p += snprintf(p, (size_t)(end - p), ",block_compressor=\"bzip2\""); break; case COMPRESS_LZO: p += snprintf(p, (size_t)(end - p), ",block_compressor=\"LZO1B-6\""); break; case COMPRESS_RAW: p += snprintf(p, (size_t)(end - p), ",block_compressor=\"raw\""); break; case COMPRESS_SNAPPY: p += snprintf(p, (size_t)(end - p), ",block_compressor=\"snappy\""); break; } /* Configure internal key truncation. */ p += snprintf( p, (size_t)(end - p), ",internal_key_truncate=%s", g.c_internal_key_truncation ? "true" : "false"); /* Configure Btree page key gap. */ p += snprintf(p, (size_t)(end - p), ",key_gap=%u", g.c_key_gap); /* Configure Btree split page percentage. */ p += snprintf(p, (size_t)(end - p), ",split_pct=%u", g.c_split_pct); if ((ret = session->create(session, g.uri, config)) != 0) die(ret, "session.create: %s", g.uri); if ((ret = session->close(session, NULL)) != 0) die(ret, "session.close"); }
/* * config_setup -- * Initialize configuration for a run. */ void config_setup(void) { CONFIG *cp; /* Clear any temporary values. */ config_clear(); /* * Choose a data source type and a file type: they're interrelated (LSM * trees are only compatible with row-store) and other items depend on * them. */ if (!config_find_is_perm("data_source", strlen("data_source"))) switch (MMRAND(1, 3)) { case 1: config_single("data_source=file", 0); break; case 2: config_single("data_source=lsm", 0); break; case 3: config_single("data_source=table", 0); break; } if (!config_find_is_perm("file_type", strlen("file_type"))) switch (DATASOURCE("lsm") ? 3 : MMRAND(1, 3)) { case 1: config_single("file_type=fix", 0); break; case 2: config_single("file_type=var", 0); break; case 3: config_single("file_type=row", 0); break; } config_map_file_type(g.c_file_type, &g.type); /* * If data_source and file_type were both "permanent", we may still * have a mismatch. */ if (DATASOURCE("lsm") && g.type != ROW) { fprintf(stderr, "%s: lsm data_source is only compatible with row file_type\n", g.progname); exit(EXIT_FAILURE); } /* * Build the top-level object name: we're overloading data_source in * our configuration, LSM or KVS devices are "tables", but files are * tested as well. */ if ((g.uri = malloc(256)) == NULL) syserr("malloc"); strcpy(g.uri, DATASOURCE("file") ? "file:" : "table:"); if (DATASOURCE("helium")) strcat(g.uri, "dev1/"); strcat(g.uri, WT_NAME); /* Default single-threaded 10% of the time. */ cp = config_find("threads", strlen("threads")); if (!(cp->flags & C_PERM)) *cp->v = MMRAND(1, 100) < 10 ? 1: CONF_RAND(cp); /* Fill in random values for the rest of the run. */ for (cp = c; cp->name != NULL; ++cp) { if (cp->flags & (C_IGNORE | C_PERM | C_TEMP)) continue; /* * Boolean flags are 0 or 1, but only set N in 100 where the * variable's min value is N. Set the flag if we rolled >= * the min, 0 otherwise. */ if (cp->flags & C_BOOL) *cp->v = MMRAND(1, 100) <= cp->min ? 1 : 0; else *cp->v = CONF_RAND(cp); } /* Required shared libraries. */ if (DATASOURCE("helium") && access(HELIUM_PATH, R_OK) != 0) die(errno, "Levyx/helium shared library: %s", HELIUM_PATH); if (DATASOURCE("kvsbdb") && access(KVS_BDB_PATH, R_OK) != 0) die(errno, "kvsbdb shared library: %s", KVS_BDB_PATH); /* Some data-sources don't support user-specified collations. */ if (DATASOURCE("helium") || DATASOURCE("kvsbdb")) g.c_reverse = 0; config_checksum(); config_compression(); /* Clear operations values if the whole run is read-only. */ if (g.c_ops == 0) for (cp = c; cp->name != NULL; ++cp) if (cp->flags & C_OPS) *cp->v = 0; /* * Periodically, set the delete percentage to 0 so salvage gets run, * as long as the delete percentage isn't nailed down. */ if (!g.replay && g.run_cnt % 10 == 0) { cp = config_find("delete_pct", strlen("delete_pct")); if (cp->name != NULL && !(cp->flags & (C_IGNORE | C_PERM | C_TEMP))) g.c_delete_pct = 0; } /* * If this is an LSM run, set the cache size and crank up the insert * percentage. */ if (DATASOURCE("lsm")) { cp = config_find("cache", strlen("cache")); if (!(cp->flags & C_PERM)) g.c_cache = 30 * g.c_chunk_size; cp = config_find("insert_pct", strlen("insert_pct")); if (cp->name != NULL && !(cp->flags & (C_IGNORE | C_PERM | C_TEMP))) g.c_insert_pct = MMRAND(50, 85); } /* * Key/value minimum/maximum are related, correct unless specified by * the configuration. */ cp = config_find("key_min", strlen("key_min")); if (!(cp->flags & C_PERM) && g.c_key_min > g.c_key_max) g.c_key_min = g.c_key_max; cp = config_find("key_max", strlen("key_max")); if (!(cp->flags & C_PERM) && g.c_key_max < g.c_key_min) g.c_key_max = g.c_key_min; if (g.c_key_min > g.c_key_max) die(EINVAL, "key_min may not be larger than key_max"); cp = config_find("value_min", strlen("value_min")); if (!(cp->flags & C_PERM) && g.c_value_min > g.c_value_max) g.c_value_min = g.c_value_max; cp = config_find("value_max", strlen("value_max")); if (!(cp->flags & C_PERM) && g.c_value_max < g.c_value_min) g.c_value_max = g.c_value_min; if (g.c_value_min > g.c_value_max) die(EINVAL, "value_min may not be larger than value_max"); /* Reset the key count. */ g.key_cnt = 0; }