static void testBulkSetRecords(CuTest *testCase) {
/*
* Tests doing a bulk update of a set of records.
*/
setup();
int64_t i = 100, j = 110, k = 120, l = 130;
stKVDatabase_insertRecord(database, 1, &i, sizeof(int64_t));
stList *requests = stList_construct3(0, (void(*)(void *)) stKVDatabaseBulkRequest_destruct);
stList_append(requests, stKVDatabaseBulkRequest_constructInsertRequest(2, &j, sizeof(int64_t)));
stList_append(requests, stKVDatabaseBulkRequest_constructSetRequest(3, &k, sizeof(int64_t)));
stList_append(requests, stKVDatabaseBulkRequest_constructUpdateRequest(1, &l, sizeof(int64_t)));
stKVDatabase_bulkSetRecords(database, requests);
stList_destruct(requests);
int64_t *m = stKVDatabase_getRecord(database, 1);
CuAssertTrue(testCase, m != NULL);
CuAssertTrue(testCase, l == *m);
free(m);
m = stKVDatabase_getRecord(database, 2);
CuAssertTrue(testCase, m != NULL);
CuAssertTrue(testCase, j == *m);
free(m);
m = stKVDatabase_getRecord(database, 3);
CuAssertTrue(testCase, m != NULL);
CuAssertTrue(testCase, k == *m);
free(m);
teardown();
}
void cactusDisk_write(CactusDisk *cactusDisk) {
Flower *flower;
int64_t recordSize;
stList *removeRequests = stList_construct3(0, (void (*)(void *)) stIntTuple_destruct);
st_logDebug("Starting to write the cactus to disk\n");
stSortedSetIterator *it = stSortedSet_getIterator(cactusDisk->flowers);
//Sort flowers to update.
while ((flower = stSortedSet_getNext(it)) != NULL) {
cactusDisk_addUpdateRequest(cactusDisk, flower);
}
stSortedSet_destructIterator(it);
st_logDebug("Got the flowers to update\n");
//Remove nets that are marked for deletion..
it = stSortedSet_getIterator(cactusDisk->flowerNamesMarkedForDeletion);
char *nameString;
while ((nameString = stSortedSet_getNext(it)) != NULL) {
Name name = cactusMisc_stringToName(nameString);
if (containsRecord(cactusDisk, name)) {
stList_append(cactusDisk->updateRequests, stKVDatabaseBulkRequest_constructUpdateRequest(name, &name, 0)); //We set it to null in the first atomic operation.
stList_append(removeRequests, stIntTuple_construct1(name));
}
}
stSortedSet_destructIterator(it);
st_logDebug("Avoided updating nets marked for deletion\n");
// Insert and/or update meta-sequences.
it = stSortedSet_getIterator(cactusDisk->metaSequences);
MetaSequence *metaSequence;
while ((metaSequence = stSortedSet_getNext(it)) != NULL) {
void *vA =
binaryRepresentation_makeBinaryRepresentation(metaSequence,
(void (*)(void *, void (*)(const void * ptr, size_t size, size_t count))) metaSequence_writeBinaryRepresentation,
&recordSize);
//Compression
vA = compress(vA, &recordSize);
if (!containsRecord(cactusDisk, metaSequence_getName(metaSequence))) {
stList_append(cactusDisk->updateRequests,
stKVDatabaseBulkRequest_constructInsertRequest(metaSequence_getName(metaSequence), vA, recordSize));
} else {
stList_append(cactusDisk->updateRequests,
stKVDatabaseBulkRequest_constructUpdateRequest(metaSequence_getName(metaSequence), vA, recordSize));
}
free(vA);
}
stSortedSet_destructIterator(it);
st_logDebug("Got the sequences we are going to add to the database.\n");
if (!containsRecord(cactusDisk, CACTUS_DISK_PARAMETER_KEY)) { //We only write the parameters once.
//Finally the database info.
void *cactusDiskParameters =
binaryRepresentation_makeBinaryRepresentation(cactusDisk,
(void (*)(void *, void (*)(const void * ptr, size_t size, size_t count))) cactusDisk_writeBinaryRepresentation,
&recordSize);
//Compression
cactusDiskParameters = compress(cactusDiskParameters, &recordSize);
stList_append(cactusDisk->updateRequests,
stKVDatabaseBulkRequest_constructInsertRequest(CACTUS_DISK_PARAMETER_KEY, cactusDiskParameters,
recordSize));
free(cactusDiskParameters);
}
st_logDebug("Checked if need to write the initial parameters\n");
if (stList_length(cactusDisk->updateRequests) > 0) {
st_logDebug("Going to write %" PRIi64 " updates\n", stList_length(cactusDisk->updateRequests));
stTry
{
st_logDebug("Writing %" PRIi64 " updates\n", stList_length(cactusDisk->updateRequests));
assert(stList_length(cactusDisk->updateRequests) > 0);
stKVDatabase_bulkSetRecords(cactusDisk->database, cactusDisk->updateRequests);
}
stCatch(except)
{
stThrowNewCause(except, ST_KV_DATABASE_EXCEPTION_ID,
"Failed when trying to set records in updating the cactus disk");
}stTryEnd
;
}
Name cactusDisk_addString(CactusDisk *cactusDisk, const char *string) {
/*
* Adds a string to the database.
*/
if (cactusDisk->storeSequencesInAFile) {
if (cactusDisk->sequencesWriteFileHandle == NULL) {
//We do not allow the read file handle to be open at the same time.
if (cactusDisk->sequencesReadFileHandle != NULL) {
fclose(cactusDisk->sequencesReadFileHandle);
cactusDisk->sequencesReadFileHandle = NULL;
}
cactusDisk->sequencesWriteFileHandle = fopen(cactusDisk->absSequencesFileName, "a");
assert(cactusDisk->sequencesWriteFileHandle != NULL);
}
else {
//The read file handle should not be open at the same time.
assert(cactusDisk->sequencesReadFileHandle == NULL);
}
Name name = ftell(cactusDisk->sequencesWriteFileHandle) + 1;
//Extra temporary cheesy code to avoid potential overflow in fprintf
int64_t chunkSize = 1000000000; //1 gig approx chunks, to avoid a possible overflow issue with fprintf
int64_t length = strlen(string);
if (length > chunkSize) {
fprintf(cactusDisk->sequencesWriteFileHandle, ">");
for (int64_t i = 0; i < length;) {
int64_t j = i + chunkSize <= length ? chunkSize : length - i;
char *string2 = memcpy(st_malloc(sizeof(char) * (j + 1)), string + i, sizeof(char) * j);
string2[j] = '\0';
int64_t k = fprintf(cactusDisk->sequencesWriteFileHandle, "%s", string2);
(void) k;
assert(k == j);
free(string2);
i += j;
}
} else {
//Replacing this line
int64_t k = fprintf(cactusDisk->sequencesWriteFileHandle, ">%s", string);
(void) k;
assert(k == length + 1);
}
#ifndef NDEBUG
// Extra fsync may not be necessary.
fsync(fileno(cactusDisk->sequencesWriteFileHandle));
fclose(cactusDisk->sequencesWriteFileHandle);
cactusDisk->sequencesWriteFileHandle = NULL;
cactusDisk->sequencesReadFileHandle = fopen(cactusDisk->absSequencesFileName, "r");
char *string2 = getStringFromDisk(cactusDisk->sequencesReadFileHandle, name, 0, length);
for (int64_t i = 0; i < length; i++) {
assert(string[i] == string2[i]);
}
free(string2);
#endif
return name;
} else {
int64_t stringSize = strlen(string);
int64_t intervalSize = ceil((double) stringSize / CACTUS_DISK_SEQUENCE_CHUNK_SIZE);
Name name = cactusDisk_getUniqueIDInterval(cactusDisk, intervalSize);
stList *insertRequests = stList_construct3(0, (void (*)(void *)) stKVDatabaseBulkRequest_destruct);
for (int64_t i = 0; i * CACTUS_DISK_SEQUENCE_CHUNK_SIZE < stringSize; i++) {
int64_t j =
(i + 1) * CACTUS_DISK_SEQUENCE_CHUNK_SIZE < stringSize ?
CACTUS_DISK_SEQUENCE_CHUNK_SIZE : stringSize - i * CACTUS_DISK_SEQUENCE_CHUNK_SIZE;
char *subString = stString_getSubString(string, i * CACTUS_DISK_SEQUENCE_CHUNK_SIZE, j);
stList_append(insertRequests, stKVDatabaseBulkRequest_constructInsertRequest(name + i, subString, j + 1));
free(subString);
}
stTry
{
stKVDatabase_bulkSetRecords(cactusDisk->database, insertRequests);
}
stCatch(except)
{
stThrowNewCause(except, ST_KV_DATABASE_EXCEPTION_ID,
"An unknown database error occurred when we tried to add a string to the cactus disk");
}stTryEnd
;
stList_destruct(insertRequests);
return name;
}
}
static void testBulkGetRecords(CuTest* testCase) {
/*
* Tests the new bulk get functions
*/
setup();
int64_t i = 100, j = 110, k = 120, l = 130;
int64_t bigRecSize = 184500800;
int64_t* m = (int64_t*)st_malloc(bigRecSize);
int64_t ki = 4, kj = 5, kk = 3, kl = 1, km = 2;
stKVDatabase_insertRecord(database, 1, &i, sizeof(int64_t));
stList *requests = stList_construct3(0, (void(*)(void *)) stKVDatabaseBulkRequest_destruct);
stList_append(requests, stKVDatabaseBulkRequest_constructInsertRequest(ki, &i, sizeof(int64_t)));
stList_append(requests, stKVDatabaseBulkRequest_constructInsertRequest(kj, &j, sizeof(int64_t)));
stList_append(requests, stKVDatabaseBulkRequest_constructSetRequest(kk, &k, sizeof(int64_t)));
stList_append(requests, stKVDatabaseBulkRequest_constructUpdateRequest(kl, &l, sizeof(int64_t)));
stKVDatabase_bulkSetRecords(database, requests);
stList_destruct(requests);
stKVDatabase_setRecord(database, km, m, bigRecSize);
stList* keys = stList_construct2(5);
stList_set(keys, 0, &ki);
stList_set(keys, 1, &kj);
stList_set(keys, 2, &kk);
stList_set(keys, 3, &kl);
stList_set(keys, 4, &km);
stList* results = stKVDatabase_bulkGetRecords(database, keys);
CuAssertTrue(testCase, stList_length(results) == 5);
void* record;
int64_t size;
stKVDatabaseBulkResult* res0 = (stKVDatabaseBulkResult*)stList_get(results, 0);
record = stKVDatabaseBulkResult_getRecord(res0, &size);
CuAssertTrue(testCase, record != NULL);
CuAssertTrue(testCase, *(int64_t*)record == i && size == sizeof(int64_t));
stKVDatabaseBulkResult* res1 = (stKVDatabaseBulkResult*)stList_get(results, 1);
record = stKVDatabaseBulkResult_getRecord(res1, &size);
CuAssertTrue(testCase, record != NULL);
CuAssertTrue(testCase, *(int64_t*)record == j && size == sizeof(int64_t));
stKVDatabaseBulkResult* res2 = (stKVDatabaseBulkResult*)stList_get(results, 2);
record = stKVDatabaseBulkResult_getRecord(res2, &size);
CuAssertTrue(testCase, record != NULL);
CuAssertTrue(testCase, *(int64_t*)record == k && size == sizeof(int64_t));
stKVDatabaseBulkResult* res3 = (stKVDatabaseBulkResult*)stList_get(results, 3);
record = stKVDatabaseBulkResult_getRecord(res3, &size);
CuAssertTrue(testCase, record != NULL);
CuAssertTrue(testCase, *(int64_t*)record == l && size == sizeof(int64_t));
stKVDatabaseBulkResult* res4 = (stKVDatabaseBulkResult*)stList_get(results, 4);
record = stKVDatabaseBulkResult_getRecord(res4, &size);
CuAssertTrue(testCase, record != NULL);
CuAssertTrue(testCase, size == bigRecSize);
stList_destruct(results);
results = stKVDatabase_bulkGetRecordsRange(database, 1, 6);
CuAssertTrue(testCase, stList_length(results) == 6);
res0 = (stKVDatabaseBulkResult*)stList_get(results, 0);
record = stKVDatabaseBulkResult_getRecord(res0, &size);
CuAssertTrue(testCase, record != NULL);
CuAssertTrue(testCase, *(int64_t*)record == l && size == sizeof(int64_t));
res1 = (stKVDatabaseBulkResult*)stList_get(results, 1);
record = stKVDatabaseBulkResult_getRecord(res1, &size);
CuAssertTrue(testCase, record != NULL);
CuAssertTrue(testCase, size == bigRecSize);
res2 = (stKVDatabaseBulkResult*)stList_get(results, 2);
record = stKVDatabaseBulkResult_getRecord(res2, &size);
CuAssertTrue(testCase, record != NULL);
CuAssertTrue(testCase, *(int64_t*)record == k && size == sizeof(int64_t));
res3 = (stKVDatabaseBulkResult*)stList_get(results, 3);
record = stKVDatabaseBulkResult_getRecord(res3, &size);
CuAssertTrue(testCase, record != NULL);
CuAssertTrue(testCase, *(int64_t*)record == i && size == sizeof(int64_t));
res4 = (stKVDatabaseBulkResult*)stList_get(results, 4);
record = stKVDatabaseBulkResult_getRecord(res4, &size);
CuAssertTrue(testCase, record != NULL);
CuAssertTrue(testCase, *(int64_t*)record == j && size == sizeof(int64_t));
stKVDatabaseBulkResult* res5 = (stKVDatabaseBulkResult*)stList_get(results, 5);
record = stKVDatabaseBulkResult_getRecord(res5, &size);
CuAssertTrue(testCase, record == NULL);
stList_destruct(results);
free(m);
teardown();
}
