/*
* Each thread inserts a set of keys into the record store database. The keys
* are generated in such a way that there are large gaps in the key range.
*/
static void *
thread_func(void *arg)
{
TEST_OPTS *opts;
WT_CURSOR *cursor, *idx_cursor;
WT_SESSION *session;
uint64_t i, ins_rotor, ins_thr_idx, thr_idx, ts;
uint64_t *obj_data;
opts = (TEST_OPTS *)arg;
thr_idx = __wt_atomic_fetch_addv64(&opts->next_threadid, 1);
ts = g_ts;
obj_data = dcalloc(
(NR_OBJECTS/NR_THREADS + 1) * NR_FIELDS, sizeof(*obj_data));
testutil_check(opts->conn->open_session(
opts->conn, NULL, NULL, &session));
testutil_check(session->open_cursor(
session, opts->uri, NULL, NULL, &cursor));
testutil_check(session->open_cursor(
session, "table:index", NULL, NULL, &idx_cursor));
for (ins_rotor = 1; ins_rotor < 10; ++ins_rotor) {
for (ins_thr_idx = thr_idx, i = 0; ins_thr_idx < NR_OBJECTS;
ins_thr_idx += NR_THREADS, i += NR_FIELDS) {
testutil_check(
session->begin_transaction(session, "sync=false"));
cursor->set_key(cursor, ins_thr_idx << 40 | ins_rotor);
cursor->set_value(cursor, ts,
obj_data[i+0], obj_data[i+1], obj_data[i+2],
obj_data[i+3], obj_data[i+4], obj_data[i+5],
obj_data[i+6], obj_data[i+7]);
testutil_check(cursor->insert(cursor));
idx_cursor->set_key(
idx_cursor, ins_thr_idx << 40 | ts);
idx_cursor->set_value(idx_cursor, ins_rotor);
testutil_check(idx_cursor->insert(idx_cursor));
testutil_check(
session->commit_transaction(session, NULL));
/* change object fields */
++obj_data[i + ((ins_thr_idx + ins_rotor) % NR_FIELDS)];
++obj_data[i +
((ins_thr_idx + ins_rotor + 1) % NR_FIELDS)];
++g_ts;
/* 5K updates/sec */
(void)usleep(1000000ULL * NR_THREADS / 5000);
}
}
testutil_check(session->close(session, NULL));
free(obj_data);
return (NULL);
}
/*
* Create and keep all the strings used to populate the bloom filter, so that
* we can do validation with the same set of entries.
*/
void
populate_entries(void)
{
uint32_t i, j;
uint8_t **entries;
srand(g.c_srand);
entries = dcalloc(g.c_ops, sizeof(uint8_t *));
for (i = 0; i < g.c_ops; i++) {
entries[i] = dcalloc(g.c_key_max, sizeof(uint8_t));
for (j = 0; j < g.c_key_max; j++)
entries[i][j] = 'a' + ((uint8_t)rand() % 26);
}
g.entries = entries;
}
/*
* dstrndup --
* Call emulating strndup, dying on failure. Don't use actual strndup here
* as it is not supported within MSVC.
*/
void *
dstrndup(const char *str, size_t len)
{
char *p;
p = dcalloc(len + 1, sizeof(char));
memcpy(p, str, len);
return (p);
}
void InitPStream(PStream * pst)
{
void InitDWin(PStream *);
double *dcalloc(int, int);
double **ddcalloc(int, int, int, int);
double ***dddcalloc(int, int, int, int, int, int);
int half, full;
int i, m;
InitDWin(pst);
half = pst->range * 2;
full = pst->range * 4 + 1;
pst->vSize = (pst->order + 1) * pst->dw.num;
pst->sm.length = LENGTH;
while (pst->sm.length < pst->range + pst->dw.maxw[WRIGHT])
pst->sm.length *= 2;
pst->mean = dcalloc(pst->vSize * 2, 0);
pst->ivar = pst->mean + pst->vSize;
pst->sm.mseq = ddcalloc(pst->sm.length, pst->vSize, 0, 0);
pst->sm.ivseq = ddcalloc(pst->sm.length, pst->vSize, 0, 0);
pst->sm.c = ddcalloc(pst->sm.length, pst->order + 1, 0, 0);
pst->sm.P = dddcalloc(full, pst->sm.length, pst->order + 1, half, 0, 0);
pst->sm.pi =
ddcalloc(pst->range + pst->dw.maxw[WRIGHT] + 1, pst->order + 1,
pst->range, 0);
pst->sm.k =
ddcalloc(pst->range + pst->dw.maxw[WRIGHT] + 1, pst->order + 1,
pst->range, 0);
for (i = 0; i < pst->sm.length; i++)
for (m = 0; m < pst->vSize; m++)
pst->sm.ivseq[i][m] = 0.0;
for (i = 0; i < pst->sm.length; i++)
for (m = 0; m <= pst->order; m++)
pst->sm.P[0][i][m] = INFTY;
pst->sm.t = pst->range - 1;
pst->sm.mask = pst->sm.length - 1;
return;
}
static void InitPStreamChol(PStreamChol *pst, const float *dynwin, int fsize,
int order, int T)
{
// order of cepstrum
pst->order = order;
// windows for dynamic feature
InitDWin(pst, dynwin, fsize);
// dimension of observed vector
pst->vSize = (pst->order + 1) * pst->dw.num; // odim = dim * (1--3)
// memory allocation
pst->T = T; // number of frames
pst->width = pst->dw.maxw[WRIGHT] * 2 + 1; // width of R
pst->mseq = ddcalloc(T, pst->vSize, 0, 0); // [T][odim]
pst->ivseq = ddcalloc(T, pst->vSize, 0, 0); // [T][odim]
pst->R = ddcalloc(T, pst->width, 0, 0); // [T][width]
pst->r = dcalloc(T, 0); // [T]
pst->g = dcalloc(T, 0); // [T]
pst->c = ddcalloc(T, pst->order + 1, 0, 0); // [T][dim]
return;
}
/*
* corrupt_metadata --
* Corrupt the metadata by scribbling on the "corrupt" URI string.
*/
static void
corrupt_metadata(void)
{
FILE *fp;
struct stat sb;
long off;
size_t meta_size;
bool corrupted;
uint8_t *buf, *corrupt;
char path[256];
/*
* Open the file, read its contents. Find the string "corrupt" and
* modify one byte at that offset. That will cause a checksum error
* when WiredTiger next reads it.
*/
testutil_check(__wt_snprintf(
path, sizeof(path), "%s/%s", home, WT_METAFILE));
if ((fp = fopen(path, "r+")) == NULL)
testutil_die(errno, "fopen: %s", path);
testutil_check(fstat(fileno(fp), &sb));
meta_size = (size_t)sb.st_size;
buf = dcalloc(meta_size, 1);
if (fread(buf, 1, meta_size, fp) != meta_size)
testutil_die(errno, "fread: %" WT_SIZET_FMT, meta_size);
corrupted = false;
/*
* Corrupt all occurrences of the string in the file.
*/
while ((corrupt = byte_str(buf, meta_size, CORRUPT)) != NULL) {
corrupted = true;
testutil_assert(*(char *)corrupt != 'X');
*(char *)corrupt = 'X';
off = (long)(corrupt - buf);
if (fseek(fp, off, SEEK_SET) != 0)
testutil_die(errno, "fseek: %ld", off);
if (fwrite("X", 1, 1, fp) != 1)
testutil_die(errno, "fwrite");
}
if (!corrupted)
testutil_die(errno, "corrupt string did not occur");
if (fclose(fp) != 0)
testutil_die(errno, "fclose");
free(buf);
}
void
run(void)
{
WT_BLOOM *bloomp;
WT_ITEM item;
WT_SESSION_IMPL *sess;
uint32_t fp, i;
int ret;
const char *uri = "file:my_bloom.bf";
/* Use the internal session handle to access private APIs. */
sess = (WT_SESSION_IMPL *)g.wt_session;
testutil_check(__wt_bloom_create(
sess, uri, NULL, g.c_ops, g.c_factor, g.c_k, &bloomp));
item.size = g.c_key_max;
for (i = 0; i < g.c_ops; i++) {
item.data = g.entries[i];
if ((ret = __wt_bloom_insert(bloomp, &item)) != 0)
testutil_die(ret, "__wt_bloom_insert: %" PRIu32, i);
}
testutil_check(__wt_bloom_finalize(bloomp));
for (i = 0; i < g.c_ops; i++) {
item.data = g.entries[i];
if ((ret = __wt_bloom_get(bloomp, &item)) != 0) {
fprintf(stderr,
"get failed at record: %" PRIu32 "\n", i);
testutil_die(ret, "__wt_bloom_get");
}
}
testutil_check(__wt_bloom_close(bloomp));
testutil_check(g.wt_session->checkpoint(g.wt_session, NULL));
testutil_check(__wt_bloom_open(
sess, uri, g.c_factor, g.c_k, NULL, &bloomp));
for (i = 0; i < g.c_ops; i++) {
item.data = g.entries[i];
testutil_check(__wt_bloom_get(bloomp, &item));
}
/*
* Try out some values we didn't insert - choose a different size to
* ensure the value doesn't overlap with existing values.
*/
item.size = g.c_key_max + 10;
item.data = dcalloc(item.size, 1);
memset((void *)item.data, 'a', item.size);
for (i = 0, fp = 0; i < g.c_ops; i++) {
((uint8_t *)item.data)[i % item.size] =
'a' + ((uint8_t)rand() % 26);
if ((ret = __wt_bloom_get(bloomp, &item)) == 0)
++fp;
if (ret != 0 && ret != WT_NOTFOUND)
testutil_die(ret, "__wt_bloom_get");
}
free((void *)item.data);
printf(
"Out of %" PRIu32 " ops, got %" PRIu32 " false positives, %.4f%%\n",
g.c_ops, fp, 100.0 * fp/g.c_ops);
testutil_check(__wt_bloom_drop(bloomp, NULL));
}
static void
run_workload(uint32_t nth)
{
WT_CONNECTION *conn;
WT_SESSION *session;
WT_THREAD_DATA *td;
wt_thread_t *thr;
uint32_t i;
int ret;
char envconf[512];
thr = dcalloc(nth+1, sizeof(*thr));
td = dcalloc(nth+1, sizeof(WT_THREAD_DATA));
if (chdir(home) != 0)
testutil_die(errno, "Child chdir: %s", home);
if (inmem)
strcpy(envconf, ENV_CONFIG_DEF);
else
strcpy(envconf, ENV_CONFIG_TXNSYNC);
if (compat)
strcat(envconf, ENV_CONFIG_COMPAT);
if ((ret = wiredtiger_open(NULL, NULL, envconf, &conn)) != 0)
testutil_die(ret, "wiredtiger_open");
if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
testutil_die(ret, "WT_CONNECTION:open_session");
/*
* Create all the tables.
*/
if ((ret = session->create(session, uri_collection,
"key_format=S,value_format=u,log=(enabled=false)")) != 0)
testutil_die(ret, "WT_SESSION.create: %s", uri_collection);
if ((ret = session->create(session,
uri_local, "key_format=S,value_format=u")) != 0)
testutil_die(ret, "WT_SESSION.create: %s", uri_local);
if ((ret = session->create(session,
uri_oplog, "key_format=S,value_format=u")) != 0)
testutil_die(ret, "WT_SESSION.create: %s", uri_oplog);
/*
* Don't log the stable timestamp table so that we know what timestamp
* was stored at the checkpoint.
*/
if ((ret = session->create(session, stable_store,
"key_format=Q,value_format=Q,log=(enabled=false)")) != 0)
testutil_die(ret, "WT_SESSION.create: %s", stable_store);
if ((ret = session->close(session, NULL)) != 0)
testutil_die(ret, "WT_SESSION:close");
/*
* Thread 0 is the checkpoint thread.
*/
td[0].conn = conn;
td[0].id = 0;
printf("Create checkpoint thread\n");
testutil_check(__wt_thread_create(
NULL, &thr[0], thread_ckpt_run, &td[0]));
for (i = 1; i <= nth; ++i) {
td[i].conn = conn;
td[i].start = (UINT64_MAX / nth) * (i - 1);
td[i].id = i;
testutil_check(__wt_thread_create(
NULL, &thr[i], thread_run, &td[i]));
}
/*
* The threads never exit, so the child will just wait here until
* it is killed.
*/
printf("Create %" PRIu32 " writer threads\n", nth);
fflush(stdout);
for (i = 0; i <= nth; ++i)
testutil_check(__wt_thread_join(NULL, thr[i]));
/*
* NOTREACHED
*/
free(thr);
free(td);
exit(EXIT_SUCCESS);
}
static void
run_workload(uint32_t nth)
{
WT_CONNECTION *conn;
WT_SESSION *session;
THREAD_DATA *td;
wt_thread_t *thr;
uint32_t ckpt_id, i, ts_id;
char envconf[512], uri[128];
thr = dcalloc(nth+2, sizeof(*thr));
td = dcalloc(nth+2, sizeof(THREAD_DATA));
if (chdir(home) != 0)
testutil_die(errno, "Child chdir: %s", home);
if (inmem)
(void)__wt_snprintf(envconf, sizeof(envconf),
ENV_CONFIG_DEF, SESSION_MAX);
else
(void)__wt_snprintf(envconf, sizeof(envconf),
ENV_CONFIG_TXNSYNC, SESSION_MAX);
if (compat)
strcat(envconf, ENV_CONFIG_COMPAT);
testutil_check(wiredtiger_open(NULL, NULL, envconf, &conn));
testutil_check(conn->open_session(conn, NULL, NULL, &session));
/*
* Create all the tables.
*/
testutil_check(__wt_snprintf(
uri, sizeof(uri), "%s:%s", table_pfx, uri_collection));
testutil_check(session->create(session, uri,
"key_format=S,value_format=u,log=(enabled=false)"));
testutil_check(__wt_snprintf(
uri, sizeof(uri), "%s:%s", table_pfx, uri_local));
testutil_check(session->create(session,
uri, "key_format=S,value_format=u"));
testutil_check(__wt_snprintf(
uri, sizeof(uri), "%s:%s", table_pfx, uri_oplog));
testutil_check(session->create(session,
uri, "key_format=S,value_format=u"));
/*
* Don't log the stable timestamp table so that we know what timestamp
* was stored at the checkpoint.
*/
testutil_check(session->close(session, NULL));
/*
* The checkpoint thread and the timestamp threads are added at the end.
*/
ckpt_id = nth;
td[ckpt_id].conn = conn;
td[ckpt_id].info = nth;
printf("Create checkpoint thread\n");
testutil_check(__wt_thread_create(
NULL, &thr[ckpt_id], thread_ckpt_run, &td[ckpt_id]));
ts_id = nth + 1;
if (use_ts) {
td[ts_id].conn = conn;
td[ts_id].info = nth;
printf("Create timestamp thread\n");
testutil_check(__wt_thread_create(
NULL, &thr[ts_id], thread_ts_run, &td[ts_id]));
}
printf("Create %" PRIu32 " writer threads\n", nth);
for (i = 0; i < nth; ++i) {
td[i].conn = conn;
td[i].start = WT_BILLION * (uint64_t)i;
td[i].info = i;
testutil_check(__wt_thread_create(
NULL, &thr[i], thread_run, &td[i]));
}
/*
* The threads never exit, so the child will just wait here until
* it is killed.
*/
fflush(stdout);
for (i = 0; i <= ts_id; ++i)
testutil_check(__wt_thread_join(NULL, &thr[i]));
/*
* NOTREACHED
*/
free(thr);
free(td);
exit(EXIT_SUCCESS);
}
void InitDWin(PStream * pst)
{
double *dcalloc(int, int);
int i, j;
int fsize, leng;
double x, a0, a1, a2;
FILE *fp;
/* memory allocation */
if ((pst->dw.width = (int **) calloc(pst->dw.num, sizeof(int *))) == NULL) {
fprintf(stderr, "%s : Cannot allocate memory!\n", cmnd);
exit(1);
}
for (i = 0; i < pst->dw.num; i++)
if ((pst->dw.width[i] = (int *) calloc(2, sizeof(int))) == NULL) {
fprintf(stderr, "%s : Cannot allocate memory!\n", cmnd);
exit(1);
}
if ((pst->dw.coef =
(double **) calloc(pst->dw.num, sizeof(double *))) == NULL) {
fprintf(stderr, "%s : Cannot allocate memory!\n", cmnd);
exit(1);
}
/* window for static parameter */
pst->dw.width[0][WLEFT] = pst->dw.width[0][WRIGHT] = 0;
pst->dw.coef[0] = dcalloc(1, 0);
pst->dw.coef[0][0] = 1;
/* set delta coefficients */
if (pst->dw.calccoef == 0) {
for (i = 1; i < pst->dw.num; i++) {
if (pst->dw.fn[i][0] == ' ') {
fsize = str2darray(pst->dw.fn[i], &(pst->dw.coef[i]));
} else {
/* read from file */
fp = getfp(pst->dw.fn[i], "rb");
/* check the number of coefficients */
fseek(fp, 0L, 2);
fsize = ftell(fp) / sizeof(real);
fseek(fp, 0L, 0);
/* read coefficients */
pst->dw.coef[i] = dcalloc(fsize, 0);
freadf(pst->dw.coef[i], sizeof(**(pst->dw.coef)), fsize, fp);
}
/* set pointer */
leng = fsize / 2;
pst->dw.coef[i] += leng;
pst->dw.width[i][WLEFT] = -leng;
pst->dw.width[i][WRIGHT] = leng;
if (fsize % 2 == 0)
pst->dw.width[i][WRIGHT]--;
}
} else if (pst->dw.calccoef == 1) {
for (i = 1; i < pst->dw.num; i++) {
leng = atoi(pst->dw.fn[i]);
if (leng < 1) {
fprintf(stderr,
"%s : Width for regression coefficient shuould be more than 1!\n",
cmnd);
exit(1);
}
pst->dw.width[i][WLEFT] = -leng;
pst->dw.width[i][WRIGHT] = leng;
pst->dw.coef[i] = dcalloc(leng * 2 + 1, 0);
pst->dw.coef[i] += leng;
}
leng = atoi(pst->dw.fn[1]);
for (a1 = 0, j = -leng; j <= leng; a1 += j * j, j++);
for (j = -leng; j <= leng; j++)
pst->dw.coef[1][j] = (double) j / (double) a1;
if (pst->dw.num > 2) {
leng = atoi(pst->dw.fn[2]);
for (a0 = a1 = a2 = 0, j = -leng; j <= leng;
a0++, a1 += j * j, a2 += j * j * j * j, j++);
for (j = -leng; j <= leng; j++)
pst->dw.coef[2][j] =
((double) (a0 * j * j - a1)) / ((double) (a2 * a0 - a1 * a1)) /
2;
}
}
pst->dw.maxw[WLEFT] = pst->dw.maxw[WRIGHT] = 0;
for (i = 0; i < pst->dw.num; i++) {
if (pst->dw.maxw[WLEFT] > pst->dw.width[i][WLEFT])
pst->dw.maxw[WLEFT] = pst->dw.width[i][WLEFT];
if (pst->dw.maxw[WRIGHT] < pst->dw.width[i][WRIGHT])
pst->dw.maxw[WRIGHT] = pst->dw.width[i][WRIGHT];
}
return;
}
int
main(void)
{
WT_RAND_STATE rnd;
size_t len;
uint32_t hw, sw;
u_int i, j;
uint8_t *data;
/* Allocate aligned memory for the data. */
data = dcalloc(DATASIZE, sizeof(uint8_t));
/* Initialize the RNG. */
testutil_check(__wt_random_init_seed(NULL, &rnd));
/* Initialize the WiredTiger library checksum functions. */
__wt_cksum_init();
/*
* Some simple known checksums.
*/
len = 1;
hw = __wt_cksum(data, len);
check(hw, (uint32_t)0x527d5351, len, "nul x1: hardware");
sw = cksum_sw(data, len);
check(sw, (uint32_t)0x527d5351, len, "nul x1: software");
len = 2;
hw = __wt_cksum(data, len);
check(hw, (uint32_t)0xf16177d2, len, "nul x2: hardware");
sw = cksum_sw(data, len);
check(sw, (uint32_t)0xf16177d2, len, "nul x2: software");
len = 3;
hw = __wt_cksum(data, len);
check(hw, (uint32_t)0x6064a37a, len, "nul x3: hardware");
sw = cksum_sw(data, len);
check(sw, (uint32_t)0x6064a37a, len, "nul x3: software");
len = 4;
hw = __wt_cksum(data, len);
check(hw, (uint32_t)0x48674bc7, len, "nul x4: hardware");
sw = cksum_sw(data, len);
check(sw, (uint32_t)0x48674bc7, len, "nul x4: software");
len = strlen("123456789");
memcpy(data, "123456789", len);
hw = __wt_cksum(data, len);
check(hw, (uint32_t)0xe3069283, len, "known string #1: hardware");
sw = cksum_sw(data, len);
check(sw, (uint32_t)0xe3069283, len, "known string #1: software");
len = strlen("The quick brown fox jumps over the lazy dog");
memcpy(data, "The quick brown fox jumps over the lazy dog", len);
hw = __wt_cksum(data, len);
check(hw, (uint32_t)0x22620404, len, "known string #2: hardware");
sw = cksum_sw(data, len);
check(sw, (uint32_t)0x22620404, len, "known string #2: software");
/*
* Checksums of power-of-two data chunks.
*/
for (i = 0, len = 512; i < 1000; ++i) {
for (j = 0; j < len; ++j)
data[j] = __wt_random(&rnd) & 0xff;
hw = __wt_cksum(data, len);
sw = cksum_sw(data, len);
check(hw, sw, len, "random power-of-two");
len *= 2;
if (len > DATASIZE)
len = 512;
}
/*
* Checksums of random data chunks.
*/
for (i = 0; i < 1000; ++i) {
len = __wt_random(&rnd) % DATASIZE;
for (j = 0; j < len; ++j)
data[j] = __wt_random(&rnd) & 0xff;
hw = __wt_cksum(data, len);
sw = cksum_sw(data, len);
check(hw, sw, len, "random");
}
free(data);
return (EXIT_SUCCESS);
}
