static stList *getRecords(CactusDisk *cactusDisk, stList *objectNames, char *type) {
if (stList_length(objectNames) == 0) {
return stList_construct3(0, NULL);
}
stList *records = NULL;
stTry
{
records = stKVDatabase_bulkGetRecords(cactusDisk->database, objectNames);
}
stCatch(except)
{
stThrowNewCause(except, ST_KV_DATABASE_EXCEPTION_ID,
"An unknown database error occurred when getting a bulk set of %s", type);
}stTryEnd
;
assert(records != NULL);
assert(stList_length(objectNames) == stList_length(records));
stList_setDestructor(records, free);
for (int64_t i = 0; i < stList_length(objectNames); i++) {
Name objectName = *((int64_t *) stList_get(objectNames, i));
int64_t recordSize;
void *record;
stKVDatabaseBulkResult *result = stList_get(records, i);
assert(result != NULL);
if (!stCache_containsRecord(cactusDisk->cache, objectName, 0, INT64_MAX)) {
record = stKVDatabaseBulkResult_getRecord(result, &recordSize);
assert(recordSize >= 0);
assert(record != NULL);
record = decompress(record, &recordSize);
stCache_setRecord(cactusDisk->cache, objectName, 0, recordSize, record);
} else {
record = stCache_getRecord(cactusDisk->cache, objectName, 0, INT64_MAX, &recordSize);
assert(recordSize >= 0);
assert(record != NULL);
}
stKVDatabaseBulkResult_destruct(result);
stList_set(records, i, record);
}
return records;
}
static void cacheSubstringsFromDB(CactusDisk *cactusDisk, stList *substrings) {
/*
* Caches the given set of substrings in the cactusDisk cache.
*/
if (cactusDisk->storeSequencesInAFile) {
if (cactusDisk->sequencesReadFileHandle == NULL) {
if(cactusDisk->sequencesWriteFileHandle != NULL) {
fsync(fileno(cactusDisk->sequencesWriteFileHandle));
fclose(cactusDisk->sequencesWriteFileHandle);
cactusDisk->sequencesWriteFileHandle = NULL;
}
cactusDisk->sequencesReadFileHandle = fopen(cactusDisk->absSequencesFileName, "r");
assert(cactusDisk->sequencesReadFileHandle != NULL);
}
else {
assert(cactusDisk->sequencesWriteFileHandle == NULL);
}
for (int64_t i = 0; i < stList_length(substrings); i++) {
Substring *substring = stList_get(substrings, i);
char *string = getStringFromDisk(cactusDisk->sequencesReadFileHandle, substring->name, substring->start,
substring->length);
stCache_setRecord(cactusDisk->stringCache, substring->name, substring->start, substring->length, string);
#ifndef NDEBUG
int64_t bytesRead;
char *string2 = stCache_getRecord(cactusDisk->stringCache, substring->name, substring->start,
substring->length, &bytesRead);
assert(bytesRead == substring->length);
for (int64_t j = 0; j < substring->length; j++) {
assert(string2[j] == string[j]);
}
free(string2);
#endif
free(string);
}
} else {
stList *getRequests = stList_construct3(0, free);
for (int64_t i = 0; i < stList_length(substrings); i++) {
Substring *substring = stList_get(substrings, i);
int64_t intervalSize = (substring->length + substring->start - 1) / CACTUS_DISK_SEQUENCE_CHUNK_SIZE
- substring->start / CACTUS_DISK_SEQUENCE_CHUNK_SIZE + 1;
Name shiftedName = substring->name + substring->start / CACTUS_DISK_SEQUENCE_CHUNK_SIZE;
for (int64_t j = 0; j < intervalSize; j++) {
int64_t *k = st_malloc(sizeof(int64_t));
k[0] = shiftedName + j;
stList_append(getRequests, k);
}
}
if (stList_length(getRequests) == 0) {
stList_destruct(getRequests);
return;
}
stList *records = NULL;
stTry
{
records = stKVDatabase_bulkGetRecords(cactusDisk->database, getRequests);
}
stCatch(except)
{
stThrowNewCause(except, ST_KV_DATABASE_EXCEPTION_ID,
"An unknown database error occurred when getting a sequence string");
}stTryEnd
;
assert(records != NULL);
assert(stList_length(records) == stList_length(getRequests));
stList_destruct(getRequests);
stListIterator *recordsIt = stList_getIterator(records);
for (int64_t i = 0; i < stList_length(substrings); i++) {
Substring *substring = stList_get(substrings, i);
int64_t intervalSize = (substring->length + substring->start - 1) / CACTUS_DISK_SEQUENCE_CHUNK_SIZE
- substring->start / CACTUS_DISK_SEQUENCE_CHUNK_SIZE + 1;
stList *strings = stList_construct();
while (intervalSize-- > 0) {
int64_t recordSize;
stKVDatabaseBulkResult *result = stList_getNext(recordsIt);
assert(result != NULL);
char *string = stKVDatabaseBulkResult_getRecord(result, &recordSize);
assert(string != NULL);
assert(strlen(string) == recordSize - 1);
stList_append(strings, string);
assert(recordSize <= CACTUS_DISK_SEQUENCE_CHUNK_SIZE + 1);
}
assert(stList_length(strings) > 0);
char *joinedString = stString_join2("", strings);
stCache_setRecord(cactusDisk->stringCache, substring->name,
(substring->start / CACTUS_DISK_SEQUENCE_CHUNK_SIZE) * CACTUS_DISK_SEQUENCE_CHUNK_SIZE,
strlen(joinedString), joinedString);
free(joinedString);
stList_destruct(strings);
}
assert(stList_getNext(recordsIt) == NULL);
stList_destructIterator(recordsIt);
stList_destruct(records);
}
}
static void testBulkGetRecords(CuTest* testCase) {
/*
* Tests the new bulk get functions
*/
setup();
int64_t i = 100, j = 110, k = 120, l = 130;
int64_t bigRecSize = 184500800;
int64_t* m = (int64_t*)st_malloc(bigRecSize);
int64_t ki = 4, kj = 5, kk = 3, kl = 1, km = 2;
stKVDatabase_insertRecord(database, 1, &i, sizeof(int64_t));
stList *requests = stList_construct3(0, (void(*)(void *)) stKVDatabaseBulkRequest_destruct);
stList_append(requests, stKVDatabaseBulkRequest_constructInsertRequest(ki, &i, sizeof(int64_t)));
stList_append(requests, stKVDatabaseBulkRequest_constructInsertRequest(kj, &j, sizeof(int64_t)));
stList_append(requests, stKVDatabaseBulkRequest_constructSetRequest(kk, &k, sizeof(int64_t)));
stList_append(requests, stKVDatabaseBulkRequest_constructUpdateRequest(kl, &l, sizeof(int64_t)));
stKVDatabase_bulkSetRecords(database, requests);
stList_destruct(requests);
stKVDatabase_setRecord(database, km, m, bigRecSize);
stList* keys = stList_construct2(5);
stList_set(keys, 0, &ki);
stList_set(keys, 1, &kj);
stList_set(keys, 2, &kk);
stList_set(keys, 3, &kl);
stList_set(keys, 4, &km);
stList* results = stKVDatabase_bulkGetRecords(database, keys);
CuAssertTrue(testCase, stList_length(results) == 5);
void* record;
int64_t size;
stKVDatabaseBulkResult* res0 = (stKVDatabaseBulkResult*)stList_get(results, 0);
record = stKVDatabaseBulkResult_getRecord(res0, &size);
CuAssertTrue(testCase, record != NULL);
CuAssertTrue(testCase, *(int64_t*)record == i && size == sizeof(int64_t));
stKVDatabaseBulkResult* res1 = (stKVDatabaseBulkResult*)stList_get(results, 1);
record = stKVDatabaseBulkResult_getRecord(res1, &size);
CuAssertTrue(testCase, record != NULL);
CuAssertTrue(testCase, *(int64_t*)record == j && size == sizeof(int64_t));
stKVDatabaseBulkResult* res2 = (stKVDatabaseBulkResult*)stList_get(results, 2);
record = stKVDatabaseBulkResult_getRecord(res2, &size);
CuAssertTrue(testCase, record != NULL);
CuAssertTrue(testCase, *(int64_t*)record == k && size == sizeof(int64_t));
stKVDatabaseBulkResult* res3 = (stKVDatabaseBulkResult*)stList_get(results, 3);
record = stKVDatabaseBulkResult_getRecord(res3, &size);
CuAssertTrue(testCase, record != NULL);
CuAssertTrue(testCase, *(int64_t*)record == l && size == sizeof(int64_t));
stKVDatabaseBulkResult* res4 = (stKVDatabaseBulkResult*)stList_get(results, 4);
record = stKVDatabaseBulkResult_getRecord(res4, &size);
CuAssertTrue(testCase, record != NULL);
CuAssertTrue(testCase, size == bigRecSize);
stList_destruct(results);
results = stKVDatabase_bulkGetRecordsRange(database, 1, 6);
CuAssertTrue(testCase, stList_length(results) == 6);
res0 = (stKVDatabaseBulkResult*)stList_get(results, 0);
record = stKVDatabaseBulkResult_getRecord(res0, &size);
CuAssertTrue(testCase, record != NULL);
CuAssertTrue(testCase, *(int64_t*)record == l && size == sizeof(int64_t));
res1 = (stKVDatabaseBulkResult*)stList_get(results, 1);
record = stKVDatabaseBulkResult_getRecord(res1, &size);
CuAssertTrue(testCase, record != NULL);
CuAssertTrue(testCase, size == bigRecSize);
res2 = (stKVDatabaseBulkResult*)stList_get(results, 2);
record = stKVDatabaseBulkResult_getRecord(res2, &size);
CuAssertTrue(testCase, record != NULL);
CuAssertTrue(testCase, *(int64_t*)record == k && size == sizeof(int64_t));
res3 = (stKVDatabaseBulkResult*)stList_get(results, 3);
record = stKVDatabaseBulkResult_getRecord(res3, &size);
CuAssertTrue(testCase, record != NULL);
CuAssertTrue(testCase, *(int64_t*)record == i && size == sizeof(int64_t));
res4 = (stKVDatabaseBulkResult*)stList_get(results, 4);
record = stKVDatabaseBulkResult_getRecord(res4, &size);
CuAssertTrue(testCase, record != NULL);
CuAssertTrue(testCase, *(int64_t*)record == j && size == sizeof(int64_t));
stKVDatabaseBulkResult* res5 = (stKVDatabaseBulkResult*)stList_get(results, 5);
record = stKVDatabaseBulkResult_getRecord(res5, &size);
CuAssertTrue(testCase, record == NULL);
stList_destruct(results);
free(m);
teardown();
}
