static stList *getRecords(CactusDisk *cactusDisk, stList *objectNames, char *type) { if (stList_length(objectNames) == 0) { return stList_construct3(0, NULL); } stList *records = NULL; stTry { records = stKVDatabase_bulkGetRecords(cactusDisk->database, objectNames); } stCatch(except) { stThrowNewCause(except, ST_KV_DATABASE_EXCEPTION_ID, "An unknown database error occurred when getting a bulk set of %s", type); }stTryEnd ; assert(records != NULL); assert(stList_length(objectNames) == stList_length(records)); stList_setDestructor(records, free); for (int64_t i = 0; i < stList_length(objectNames); i++) { Name objectName = *((int64_t *) stList_get(objectNames, i)); int64_t recordSize; void *record; stKVDatabaseBulkResult *result = stList_get(records, i); assert(result != NULL); if (!stCache_containsRecord(cactusDisk->cache, objectName, 0, INT64_MAX)) { record = stKVDatabaseBulkResult_getRecord(result, &recordSize); assert(recordSize >= 0); assert(record != NULL); record = decompress(record, &recordSize); stCache_setRecord(cactusDisk->cache, objectName, 0, recordSize, record); } else { record = stCache_getRecord(cactusDisk->cache, objectName, 0, INT64_MAX, &recordSize); assert(recordSize >= 0); assert(record != NULL); } stKVDatabaseBulkResult_destruct(result); stList_set(records, i, record); } return records; }
static void cacheSubstringsFromDB(CactusDisk *cactusDisk, stList *substrings) { /* * Caches the given set of substrings in the cactusDisk cache. */ if (cactusDisk->storeSequencesInAFile) { if (cactusDisk->sequencesReadFileHandle == NULL) { if(cactusDisk->sequencesWriteFileHandle != NULL) { fsync(fileno(cactusDisk->sequencesWriteFileHandle)); fclose(cactusDisk->sequencesWriteFileHandle); cactusDisk->sequencesWriteFileHandle = NULL; } cactusDisk->sequencesReadFileHandle = fopen(cactusDisk->absSequencesFileName, "r"); assert(cactusDisk->sequencesReadFileHandle != NULL); } else { assert(cactusDisk->sequencesWriteFileHandle == NULL); } for (int64_t i = 0; i < stList_length(substrings); i++) { Substring *substring = stList_get(substrings, i); char *string = getStringFromDisk(cactusDisk->sequencesReadFileHandle, substring->name, substring->start, substring->length); stCache_setRecord(cactusDisk->stringCache, substring->name, substring->start, substring->length, string); #ifndef NDEBUG int64_t bytesRead; char *string2 = stCache_getRecord(cactusDisk->stringCache, substring->name, substring->start, substring->length, &bytesRead); assert(bytesRead == substring->length); for (int64_t j = 0; j < substring->length; j++) { assert(string2[j] == string[j]); } free(string2); #endif free(string); } } else { stList *getRequests = stList_construct3(0, free); for (int64_t i = 0; i < stList_length(substrings); i++) { Substring *substring = stList_get(substrings, i); int64_t intervalSize = (substring->length + substring->start - 1) / CACTUS_DISK_SEQUENCE_CHUNK_SIZE - substring->start / CACTUS_DISK_SEQUENCE_CHUNK_SIZE + 1; Name shiftedName = substring->name + substring->start / CACTUS_DISK_SEQUENCE_CHUNK_SIZE; for (int64_t j = 0; j < intervalSize; j++) { int64_t *k = st_malloc(sizeof(int64_t)); k[0] = shiftedName + j; stList_append(getRequests, k); } } if (stList_length(getRequests) == 0) { stList_destruct(getRequests); return; } stList *records = NULL; stTry { records = stKVDatabase_bulkGetRecords(cactusDisk->database, getRequests); } stCatch(except) { stThrowNewCause(except, ST_KV_DATABASE_EXCEPTION_ID, "An unknown database error occurred when getting a sequence string"); }stTryEnd ; assert(records != NULL); assert(stList_length(records) == stList_length(getRequests)); stList_destruct(getRequests); stListIterator *recordsIt = stList_getIterator(records); for (int64_t i = 0; i < stList_length(substrings); i++) { Substring *substring = stList_get(substrings, i); int64_t intervalSize = (substring->length + substring->start - 1) / CACTUS_DISK_SEQUENCE_CHUNK_SIZE - substring->start / CACTUS_DISK_SEQUENCE_CHUNK_SIZE + 1; stList *strings = stList_construct(); while (intervalSize-- > 0) { int64_t recordSize; stKVDatabaseBulkResult *result = stList_getNext(recordsIt); assert(result != NULL); char *string = stKVDatabaseBulkResult_getRecord(result, &recordSize); assert(string != NULL); assert(strlen(string) == recordSize - 1); stList_append(strings, string); assert(recordSize <= CACTUS_DISK_SEQUENCE_CHUNK_SIZE + 1); } assert(stList_length(strings) > 0); char *joinedString = stString_join2("", strings); stCache_setRecord(cactusDisk->stringCache, substring->name, (substring->start / CACTUS_DISK_SEQUENCE_CHUNK_SIZE) * CACTUS_DISK_SEQUENCE_CHUNK_SIZE, strlen(joinedString), joinedString); free(joinedString); stList_destruct(strings); } assert(stList_getNext(recordsIt) == NULL); stList_destructIterator(recordsIt); stList_destruct(records); } }
static void testBulkGetRecords(CuTest* testCase) { /* * Tests the new bulk get functions */ setup(); int64_t i = 100, j = 110, k = 120, l = 130; int64_t bigRecSize = 184500800; int64_t* m = (int64_t*)st_malloc(bigRecSize); int64_t ki = 4, kj = 5, kk = 3, kl = 1, km = 2; stKVDatabase_insertRecord(database, 1, &i, sizeof(int64_t)); stList *requests = stList_construct3(0, (void(*)(void *)) stKVDatabaseBulkRequest_destruct); stList_append(requests, stKVDatabaseBulkRequest_constructInsertRequest(ki, &i, sizeof(int64_t))); stList_append(requests, stKVDatabaseBulkRequest_constructInsertRequest(kj, &j, sizeof(int64_t))); stList_append(requests, stKVDatabaseBulkRequest_constructSetRequest(kk, &k, sizeof(int64_t))); stList_append(requests, stKVDatabaseBulkRequest_constructUpdateRequest(kl, &l, sizeof(int64_t))); stKVDatabase_bulkSetRecords(database, requests); stList_destruct(requests); stKVDatabase_setRecord(database, km, m, bigRecSize); stList* keys = stList_construct2(5); stList_set(keys, 0, &ki); stList_set(keys, 1, &kj); stList_set(keys, 2, &kk); stList_set(keys, 3, &kl); stList_set(keys, 4, &km); stList* results = stKVDatabase_bulkGetRecords(database, keys); CuAssertTrue(testCase, stList_length(results) == 5); void* record; int64_t size; stKVDatabaseBulkResult* res0 = (stKVDatabaseBulkResult*)stList_get(results, 0); record = stKVDatabaseBulkResult_getRecord(res0, &size); CuAssertTrue(testCase, record != NULL); CuAssertTrue(testCase, *(int64_t*)record == i && size == sizeof(int64_t)); stKVDatabaseBulkResult* res1 = (stKVDatabaseBulkResult*)stList_get(results, 1); record = stKVDatabaseBulkResult_getRecord(res1, &size); CuAssertTrue(testCase, record != NULL); CuAssertTrue(testCase, *(int64_t*)record == j && size == sizeof(int64_t)); stKVDatabaseBulkResult* res2 = (stKVDatabaseBulkResult*)stList_get(results, 2); record = stKVDatabaseBulkResult_getRecord(res2, &size); CuAssertTrue(testCase, record != NULL); CuAssertTrue(testCase, *(int64_t*)record == k && size == sizeof(int64_t)); stKVDatabaseBulkResult* res3 = (stKVDatabaseBulkResult*)stList_get(results, 3); record = stKVDatabaseBulkResult_getRecord(res3, &size); CuAssertTrue(testCase, record != NULL); CuAssertTrue(testCase, *(int64_t*)record == l && size == sizeof(int64_t)); stKVDatabaseBulkResult* res4 = (stKVDatabaseBulkResult*)stList_get(results, 4); record = stKVDatabaseBulkResult_getRecord(res4, &size); CuAssertTrue(testCase, record != NULL); CuAssertTrue(testCase, size == bigRecSize); stList_destruct(results); results = stKVDatabase_bulkGetRecordsRange(database, 1, 6); CuAssertTrue(testCase, stList_length(results) == 6); res0 = (stKVDatabaseBulkResult*)stList_get(results, 0); record = stKVDatabaseBulkResult_getRecord(res0, &size); CuAssertTrue(testCase, record != NULL); CuAssertTrue(testCase, *(int64_t*)record == l && size == sizeof(int64_t)); res1 = (stKVDatabaseBulkResult*)stList_get(results, 1); record = stKVDatabaseBulkResult_getRecord(res1, &size); CuAssertTrue(testCase, record != NULL); CuAssertTrue(testCase, size == bigRecSize); res2 = (stKVDatabaseBulkResult*)stList_get(results, 2); record = stKVDatabaseBulkResult_getRecord(res2, &size); CuAssertTrue(testCase, record != NULL); CuAssertTrue(testCase, *(int64_t*)record == k && size == sizeof(int64_t)); res3 = (stKVDatabaseBulkResult*)stList_get(results, 3); record = stKVDatabaseBulkResult_getRecord(res3, &size); CuAssertTrue(testCase, record != NULL); CuAssertTrue(testCase, *(int64_t*)record == i && size == sizeof(int64_t)); res4 = (stKVDatabaseBulkResult*)stList_get(results, 4); record = stKVDatabaseBulkResult_getRecord(res4, &size); CuAssertTrue(testCase, record != NULL); CuAssertTrue(testCase, *(int64_t*)record == j && size == sizeof(int64_t)); stKVDatabaseBulkResult* res5 = (stKVDatabaseBulkResult*)stList_get(results, 5); record = stKVDatabaseBulkResult_getRecord(res5, &size); CuAssertTrue(testCase, record == NULL); stList_destruct(results); free(m); teardown(); }