int
testFiles(char *filename, char *prefix, uint32 merSize) {
char *prefixfilename = new char [strlen(prefix) + 32];
// Create existDB e and save it to disk
//
existDB *e = new existDB(filename, merSize, existDBnoFlags | existDBcounts, 0, ~uint32ZERO);
sprintf(prefixfilename, "%s.1", prefix);
e->saveState(prefixfilename);
// Create existDB f by loading the saved copy from disk
//
existDB *f = new existDB(prefixfilename);
// Create a fresh existDB g (to check if we corrup the original when saved)
//
existDB *g = new existDB(filename, merSize, existDBnoFlags | existDBcounts, 0, ~uint32ZERO);
speedCounter *C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, true);
fprintf(stderr, "Need to iterate over %7.2f Mmers.\n", (uint64MASK(2 * merSize) + 1) / 1000000.0);
for (uint64 d=0, m=uint64MASK(2 * merSize); m--; ) {
bool ee = e->exists(m);
bool ef = f->exists(m);
bool eg = g->exists(m);
uint32 ce = e->count(m);
uint32 cf = f->count(m);
uint32 cg = g->count(m);
if ((ee != ef) || (ef != eg) || (ee != eg))
fprintf(stderr, "mer "uint64HEX" not found : e=%d f=%d g=%d\n", m, ee, ef, eg);
if ((ce != cf) || (cf != cg) || (ce != cg))
fprintf(stderr, "mer "uint64HEX" count differs : e=%u f=%u g=%u (exists=%d)\n", m, ce, cf, cg, ee);
if ((m & 0xffffff) == 0) {
// Been a while since a report, so report.
d = 1;
}
if ((ce > 1) && (d == 1)) {
// Report anything not unique, to make sure that we're testing real counts and not just existence.
fprintf(stderr, "mer "uint64HEX" : e=%u f=%u g=%u (exists=%d)\n", m, ce, cf, cg, ee);
d = 0;
}
C->tick();
}
delete e;
delete C;
return(0);
}
int
testExhaustive(char *filename, char *merylname, uint32 merSize) {
existDB *E = new existDB(filename, merSize, existDBnoFlags, 0, ~uint32ZERO);
merylStreamReader *M = new merylStreamReader(merylname);
speedCounter *C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, true);
uint64 found = uint64ZERO;
uint64 expected = uint64ZERO;
FILE *DUMP = 0L;
DUMP = fopen("testExhaustive.ms.dump", "w");
while (M->nextMer()) {
if (E->exists(M->theFMer())) {
expected++;
fprintf(DUMP, uint64HEX"\n", (uint64)M->theFMer());
} else {
fprintf(DUMP, uint64HEX" MISSED!\n", (uint64)M->theFMer());
}
}
fclose(DUMP);
fprintf(stderr, "Found "uint64FMT" mers in the meryl database.\n", expected);
fprintf(stderr, "Need to iterate over %7.2f Mmers.\n", (uint64MASK(2 * merSize) + 1) / 1000000.0);
DUMP = fopen("testExhaustive.ck.dump", "w");
for (uint64 m = uint64MASK(2 * merSize); m--; ) {
if (E->exists(m)) {
found++;
fprintf(DUMP, uint64HEX"\n", m);
}
C->tick();
}
fclose(DUMP);
delete C;
delete E;
delete M;
if (expected != found) {
fprintf(stderr, "Expected to find "uint64FMT" mers, but found "uint64FMT" instead.\n",
expected, found);
return(1);
} else {
return(0);
}
}
void
testFibonacciEncoding(void) {
time_t mtseed = time(0L);
mt_s *mtctx = 0L;
uint32 iterations = TEST_LENGTH / 4;
uint64 *bits = new uint64 [3 * iterations];
uint64 bpos = uint64ZERO;
uint32 failed = 0;
uint32 errors = 0;
fprintf(stderr, "Starting test of fibonacci encoding\n");
bpos = uint64ZERO;
mtctx = mtInit(mtseed);
failed = 0;
for (uint32 j=0; j < iterations; j++) {
uint64 siz1 = (mtRandom32(mtctx) % 63) + 1;
uint64 val1 = mtRandom64(mtctx) & uint64MASK(siz1);
uint64 siz2 = siz1;
setFibonacciEncodedNumber(bits, bpos, &siz1, val1);
uint64 val2 = getFibonacciEncodedNumber(bits, bpos, &siz2);
if ((val1 != val2) || (siz1 != siz2)) {
fprintf(stderr, "fibEnc #1 failed on "uint32FMT": got "uint64FMT" expected "uint64FMT"\n", j, val2, val1);
failed++;
}
bpos += siz1;
}
if (failed) {
fprintf(stderr, "fibEnc #1 failed "uint32FMT" times.\n", failed);
errors++;
}
bpos = uint64ZERO;
mtctx = mtInit(mtseed);
failed = 0;
for (uint32 j=0; j < iterations; j++) {
uint64 siz1 = (mtRandom32(mtctx) % 63) + 1;
uint64 val1 = mtRandom64(mtctx) & uint64MASK(siz1);
uint64 val2 = getFibonacciEncodedNumber(bits, bpos, &siz1);
if (val1 != val2) {
fprintf(stderr, "fibEnc #2 failed on "uint32FMT": got "uint64FMT" expected "uint64FMT"\n", j, val2, val1);
failed++;
}
bpos += siz1;
}
if (failed) {
fprintf(stderr, "fibEnc #2 failed "uint32FMT" times.\n", failed);
errors++;
}
delete [] bits;
if (errors)
exit(1);
}
void
testBinaryEncoding(void) {
time_t mtseed = time(0L);
mt_s *mtctx = 0L;
uint32 iterations = TEST_LENGTH;
uint64 *bits = new uint64 [iterations + 2];
uint64 bpos = uint64ZERO;
uint64 *V = new uint64 [iterations];
uint64 *C = new uint64 [iterations];
uint64 *S = new uint64 [iterations];
uint32 failed = 0;
uint32 errors = 0;
fprintf(stderr, "Starting test of binary encoding\n");
bpos = uint64ZERO;
mtctx = mtInit(mtseed);
// Build some values to stuff into the bits
for (uint32 j=0; j < iterations; j++) {
S[j] = (mtRandom32(mtctx) % 63) + 1;
V[j] = mtRandom64(mtctx) & uint64MASK(S[j]);
//fprintf(stderr, "[%2d] S="uint64FMT" V="uint64HEX"\n", j, S[j], V[j]);
}
// Stuff them in, in blocks of some size. At the same time, decode
// (this has found bugs in the past).
failed = 0;
for (uint32 j=0; j < iterations; ) {
uint64 num = (mtRandom32(mtctx) % 8);
if (j + num > iterations)
num = iterations - j;
if (num == 0) {
setDecodedValue(bits, bpos, S[j], V[j]);
C[j] = getDecodedValue(bits, bpos, S[j]);
//fprintf(stderr, "[%2d] V="uint64HEX" C="uint64HEX" single\n", j, V[j], C[j]);
bpos += S[j];
} else {
uint64 newp1 = setDecodedValues(bits, bpos, num, S+j, V+j);
uint64 newp2 = getDecodedValues(bits, bpos, num, S+j, C+j);
if (newp1 != newp2) {
// not perfect; we should be checking the values too, but we do that later.
for (uint32 x=0; x<num; x++)
fprintf(stderr, "[%2d] #1 V="uint64HEX" C="uint64HEX" multiple "uint32FMT" %s\n",
j+x, V[j+x], C[j+x], num, (V[j+x] == C[j+x]) ? "" : "FAILED");
failed++;
}
bpos = newp2;
}
j += num;
if (num == 0)
j++;
}
if (failed) {
fprintf(stderr, "binEncoding #1 failed encoding "uint32FMT" times.\n", failed);
errors++;
}
// Check that V == C
failed = 0;
for (uint32 j=0; j<iterations; j++) {
if (V[j] != C[j]) {
fprintf(stderr, "[%2d] #2 V="uint64HEX" C="uint64HEX" S="uint32FMT"\n",
j, V[j], C[j], S[j]);
failed++;
}
}
if (failed) {
fprintf(stderr, "binEncoding #2 failed encode/decode "uint32FMT" times.\n", failed);
errors++;
}
// Decode independently, with different nums
bpos = 0; // reset to start of bits
for (uint32 j=0; j < iterations; ) {
uint64 num = (mtRandom32(mtctx) % 8);
if (j + num > iterations)
num = iterations - j;
if (num == 0) {
C[j] = getDecodedValue(bits, bpos, S[j]);
bpos += S[j];
} else {
bpos = getDecodedValues(bits, bpos, num, S+j, C+j);
}
j += num;
if (num == 0)
j++;
}
// Check that V == C
failed = 0;
for (uint32 j=0; j<iterations; j++) {
if (V[j] != C[j]) {
fprintf(stderr, "[%2d] #3 V="uint64HEX" C="uint64HEX" S="uint32FMT"\n",
j, V[j], C[j], S[j]);
failed++;
}
}
if (failed) {
fprintf(stderr, "binEncoding #3 failed decoding "uint32FMT" times.\n", failed);
errors++;
}
// Clean.
delete [] bits;
delete [] V;
delete [] C;
delete [] S;
if (errors)
exit(1);
}
void
testBinaryEncodingPrePost(void) {
time_t mtseed = time(0L);
mt_s *mtctx = 0L;
uint32 iterations = TEST_LENGTH;
uint64 *bits = new uint64 [2 * iterations];
uint64 bpos = uint64ZERO;
uint32 siz1 = uint64ZERO;
uint64 val1 = uint64ZERO;
uint64 val2 = uint64ZERO;
fprintf(stderr, "Starting test of binary encoding pre/post increment\n");
bpos = uint64ZERO;
mtctx = mtInit(mtseed);
for (uint32 j=0; j < iterations; j++) {
siz1 = (mtRandom32(mtctx) % 63) + 1;
val1 = mtRandom64(mtctx) & uint64MASK(siz1);
setDecodedValue(bits, bpos, siz1, val1);
val2 = postDecrementDecodedValue(bits, bpos, siz1);
if (val2 != val1) {
fprintf(stderr, "postDec1 failed: got "uint64FMT" expected "uint64FMT" siz="uint32FMT"\n",
val2, val1, siz1);
exit(1);
}
val2 = getDecodedValue(bits, bpos, siz1) + 1;
val2 &= uint64MASK(siz1);
if (val2 != val1) {
fprintf(stderr, "postDec2 failed: got "uint64FMT" expected "uint64FMT" siz="uint32FMT"\n",
val2, val1, siz1);
exit(1);
}
val2 = preDecrementDecodedValue(bits, bpos, siz1) + 2;
val2 &= uint64MASK(siz1);
if (val2 != val1) {
fprintf(stderr, "preDec failed: got "uint64FMT" expected "uint64FMT" siz="uint32FMT"\n",
val2, val1, siz1);
exit(1);
}
val2 = postIncrementDecodedValue(bits, bpos, siz1) + 2;
val2 &= uint64MASK(siz1);
if (val2 != val1) {
fprintf(stderr, "postInc failed: got "uint64FMT" expected "uint64FMT"\n", val2+2, val1-2);
exit(1);
}
val2 = getDecodedValue(bits, bpos, siz1) + 1;
val2 &= uint64MASK(siz1);
if (val2 != val1) {
fprintf(stderr, "postInc2 failed: got "uint64FMT" expected "uint64FMT" siz="uint32FMT"\n",
val2, val1, siz1);
exit(1);
}
val2 = preIncrementDecodedValue(bits, bpos, siz1);
// Should be back to original value, so no mask
if (val2 != val1) {
fprintf(stderr, "preInc failed: got "uint64FMT" expected "uint64FMT"\n", val2, val1);
exit(1);
}
switch (j % 4) {
case 0:
val2 = postDecrementDecodedValue(bits, bpos, siz1);
break;
case 1:
val2 = preDecrementDecodedValue(bits, bpos, siz1);
break;
case 2:
val2 = postIncrementDecodedValue(bits, bpos, siz1);
break;
case 3:
val2 = preIncrementDecodedValue(bits, bpos, siz1);
break;
}
bpos += siz1;
}
bpos = uint64ZERO;
mtctx = mtInit(mtseed);
//for (j=0; j < iterations; j++) {
//}
delete [] bits;
}