static stKVDatabaseConf *constructFromString(const char *xmlString) {
stHash *hash = hackParseXmlString(xmlString);
stKVDatabaseConf *databaseConf = NULL;
const char *type = getXmlValueRequired(hash, "conf_type");
const char *dbTag = getXmlValueRequired(hash, "db_tag");
if (!stString_eq(type, dbTag)) {
stThrowNew(ST_KV_DATABASE_EXCEPTION_ID, "Database XML tag \"%s\" did not match st_kv_database_conf type attribute", dbTag, type);
}
if (stString_eq(type, "tokyo_cabinet")) {
databaseConf = stKVDatabaseConf_constructTokyoCabinet(getXmlValueRequired(hash, "database_dir"));
} else if (stString_eq(type, "kyoto_tycoon")) {
databaseConf = stKVDatabaseConf_constructKyotoTycoon(getXmlValueRequired(hash, "host"),
getXmlPort(hash),
getXmlTimeout(hash),
getXMLMaxKTRecordSize(hash),
getXMLMaxKTBulkSetSize(hash),
getXMLMaxKTBulkSetNumRecords(hash),
getXmlValueRequired(hash, "database_dir"),
stHash_search(hash, "database_name"));
} else if (stString_eq(type, "mysql")) {
databaseConf = stKVDatabaseConf_constructMySql(getXmlValueRequired(hash, "host"), getXmlPort(hash),
getXmlValueRequired(hash, "user"), getXmlValueRequired(hash, "password"),
getXmlValueRequired(hash, "database_name"), getXmlValueRequired(hash, "table_name"));
} else {
stThrowNew(ST_KV_DATABASE_EXCEPTION_ID, "invalid database type \"%s\"", type);
}
stHash_destruct(hash);
return databaseConf;
}
/* Parse XML string into a hash. This parses all attributes of all tags
* into values. st_kv_database_conf type is stored as conf_type,
* database tag is stores as db_tag. This does minimal error checking
* and is really lame.
*/
static stHash *hackParseXmlString(const char *xmlString) {
stHash *hash = stHash_construct3(stHash_stringKey, stHash_stringEqualKey, free, free);
char *toReplace[5] = { "</", "<", "/>", ">", "=" };
char *cA = stString_replace(xmlString, toReplace[0], " "), *cA2;
for (int64_t i = 1; i < 5; i++) {
cA2 = stString_replace(cA, toReplace[i], " ");
free(cA);
cA = cA2;
}
getExpectedToken(&cA2, "st_kv_database_conf");
stHash_insert(hash, stString_copy("conf_type"), getKeyValue(&cA2, "type"));
stHash_insert(hash, stString_copy("db_tag"), getNextToken(&cA2));
char *key;
while (((key = getNextToken(&cA2)) != NULL) && !stString_eq(key, "st_kv_database_conf")) {
char *value = getNextToken(&cA2);
if (value == NULL) {
stThrowNew(ST_KV_DATABASE_EXCEPTION_ID, "failed to to get value for key \"%s\"", key);
}
if (stHash_search(hash, key) != NULL) {
stThrowNew(ST_KV_DATABASE_EXCEPTION_ID, "got a duplicate entry in the database conf string \"%s\"", key);
}
stHash_insert(hash, key, value);
}
if(!stString_eq(key, "st_kv_database_conf")) {
stThrowNew(ST_KV_DATABASE_EXCEPTION_ID, "got an unexpected final entry \"%s\"", key);
}
free(key);
free(cA);
return hash;
}
static void stSet_verifySetsHaveSameFunctions(stSet *set1, stSet *set2) {
if (!stSet_hashersEqual(set1, set2)) {
stThrowNew(SET_EXCEPTION_ID, "Comparator functions are not equal for "
"two sets.");
}
if (!stSet_equalitiesEqual(set1, set2)) {
stThrowNew(SET_EXCEPTION_ID, "Hash functions are not equal for "
"two sets.");
}
}
static const char *getXmlValueRequired(stHash *hash, const char *key) {
const char *value = stHash_search(hash, (char*)key);
if (value == NULL) {
stThrowNew(ST_KV_DATABASE_EXCEPTION_ID, "did not find a \"%s\" value in the database XML string", key);
}
return value;
}
stSortedSetIterator *stSortedSet_getIteratorFrom(stSortedSet *items, void *item) {
stSortedSetIterator *iterator = stSortedSet_getIterator(items);
if(avl_t_find(&iterator->traverser, items->sortedSet, item) == NULL) {
stThrowNew(SORTED_SET_EXCEPTION_ID, "Tried to create an iterator with an item that is not in the list of items");
}
stSortedSet_getPrevious(iterator);
return iterator;
}
stKVDatabaseConf *stKVDatabaseConf_constructMySql(const char *host, unsigned port, const char *user, const char *password,
const char *databaseName, const char *tableName) {
#ifndef HAVE_MYSQL
stThrowNew(ST_KV_DATABASE_EXCEPTION_ID,
"requested MySQL database, however sonlib is not compiled with MySql support");
#endif
return constructSql(stKVDatabaseTypeMySql, host, port, user, password, databaseName, tableName);
}
static void removeRecord(stKVDatabase *database, int64_t key) {
MySqlDb *dbImpl = database->dbImpl;
sqlExec(dbImpl, "delete from %s where id=%lld", dbImpl->table, (long long)key);
my_ulonglong numRows = mysql_affected_rows(dbImpl->conn);
if (numRows == 0) {
stThrowNew(ST_KV_DATABASE_EXCEPTION_ID, "remove of non-existent key %lld", (long long)key);
}
}
static void getExpectedToken(char **tokenStream, const char *expected) {
char *cA = getNextToken(tokenStream);
if (!stString_eq(cA, expected)) {
stThrowNew(ST_KV_DATABASE_EXCEPTION_ID,
"BUG: expected the token: %s in database XML string, but I got: %s from the stream %s",
expected, cA, *tokenStream);
}
free(cA);
}
/* immediate execution of a statement that doesn't return results, formatting
* arguments into query */
static void sqlExec(MySqlDb *dbImpl, char *query, ...) {
va_list args;
va_start(args, query);
MYSQL_RES *rs = queryStartv(dbImpl, query, args);
va_end(args);
if (queryNext(dbImpl, rs) != NULL) {
stThrowNew(ST_KV_DATABASE_EXCEPTION_ID, "SQL command should not have returned a result: \"%s\"", query);
}
queryEnd(dbImpl, rs);
}
static void *getPartialRecord(stKVDatabase *database, int64_t key, int64_t zeroBasedByteOffset, int64_t sizeInBytes, int64_t recordSize) {
MySqlDb *dbImpl = database->dbImpl;
MYSQL_RES *rs = queryStart(dbImpl, "select substring(data, %lld, %lld) from %s where id=%lld", (long long)zeroBasedByteOffset+1, (long long)sizeInBytes, dbImpl->table, (long long)key);
char **row = queryNext(dbImpl, rs);
void *data = NULL;
int64_t readLen = 0;
if (row != NULL) {
readLen = queryLength(dbImpl, rs);
data = stSafeCCopyMem(row[0], readLen);
}
queryEnd(dbImpl, rs);
if (readLen != sizeInBytes) {
stThrowNew(ST_KV_DATABASE_EXCEPTION_ID, "partial read of key %lld, expected %lld bytes got %lld bytes", (long long)key, (long long)sizeInBytes, (long long)readLen);
}
return data;
}
stSortedSet *stSortedSet_getDifference(stSortedSet *sortedSet1, stSortedSet *sortedSet2) {
if(!stSortedSet_comparatorsEqual(sortedSet1, sortedSet2)) {
stThrowNew(SORTED_SET_EXCEPTION_ID, "Comparators are not equal for creating the sorted set difference");
}
stSortedSet *sortedSet3 = stSortedSet_construct3(stSortedSet_getComparator(sortedSet1)->compareFn, NULL);
//Add those from sortedSet1 only if they are not in sortedSet2
stSortedSetIterator *it= stSortedSet_getIterator(sortedSet1);
void *o;
while((o = stSortedSet_getNext(it)) != NULL) {
if(stSortedSet_search(sortedSet2, o) == NULL) {
stSortedSet_insert(sortedSet3, o);
}
}
stSortedSet_destructIterator(it);
return sortedSet3;
}
static void cactusDisk_loadFromBinaryRepresentation(void **binaryString, CactusDisk *cactusDisk, stKVDatabaseConf *conf) {
cactusDisk->sequencesReadFileHandle = NULL;
cactusDisk->sequencesWriteFileHandle = NULL; //I think these lines are not needed.
cactusDisk->sequencesFileName = NULL;
cactusDisk->absSequencesFileName = NULL;
assert(binaryRepresentation_peekNextElementType(*binaryString) == CODE_CACTUS_DISK);
binaryRepresentation_popNextElementType(binaryString);
cactusDisk->storeSequencesInAFile = binaryRepresentation_getBool(binaryString);
if (cactusDisk->storeSequencesInAFile) {
cactusDisk->sequencesFileName = binaryRepresentation_getString(binaryString);
if (stKVDatabaseConf_getDir(conf) == NULL) {
stThrowNew(CACTUS_DISK_EXCEPTION_ID,
"The database conf does not contain a directory in which the sequence file is to be found!\n");
}
cactusDisk->absSequencesFileName = stString_print("%s/%s", stKVDatabaseConf_getDir(conf),
cactusDisk->sequencesFileName);
}
assert(binaryRepresentation_peekNextElementType(*binaryString) == CODE_CACTUS_DISK);
binaryRepresentation_popNextElementType(binaryString);
}
stSortedSet *stSortedSet_getUnion(stSortedSet *sortedSet1, stSortedSet *sortedSet2) {
if(!stSortedSet_comparatorsEqual(sortedSet1, sortedSet2)) {
stThrowNew(SORTED_SET_EXCEPTION_ID, "Comparators are not equal for creating the union of two sorted sets");
}
stSortedSet *sortedSet3 = stSortedSet_construct3(stSortedSet_getComparator(sortedSet1)->compareFn, NULL);
//Add those from sortedSet1
stSortedSetIterator *it= stSortedSet_getIterator(sortedSet1);
void *o;
while((o = stSortedSet_getNext(it)) != NULL) {
stSortedSet_insert(sortedSet3, o);
}
stSortedSet_destructIterator(it);
//Add those from sortedSet2
it= stSortedSet_getIterator(sortedSet2);
while((o = stSortedSet_getNext(it)) != NULL) {
stSortedSet_insert(sortedSet3, o);
}
stSortedSet_destructIterator(it);
return sortedSet3;
}
/* connect to a database server */
static MySqlDb *connect(stKVDatabaseConf *conf) {
MySqlDb *dbImpl = stSafeCCalloc(sizeof(MySqlDb));
if ((dbImpl->conn = mysql_init(NULL)) == NULL) {
disconnect(dbImpl);
stThrowNew(ST_KV_DATABASE_EXCEPTION_ID, "mysql_init failed");
}
if (mysql_real_connect(dbImpl->conn, stKVDatabaseConf_getHost(conf), stKVDatabaseConf_getUser(conf), stKVDatabaseConf_getPassword(conf), stKVDatabaseConf_getDatabaseName(conf), stKVDatabaseConf_getPort(conf), NULL, 0) == NULL) {
stExcept *ex = createMySqlExcept(dbImpl, "failed to connect to MySql database: %s on %s as user %s",
stKVDatabaseConf_getDatabaseName(conf), stKVDatabaseConf_getHost(conf), stKVDatabaseConf_getUser(conf));
disconnect(dbImpl);
stThrow(ex);
}
dbImpl->table = stString_copy(stKVDatabaseConf_getTableName(conf));
// disable report of notes, so only warnings and errors come back
sqlExec(dbImpl, "set sql_notes=0");
// set max sizes of an sql statment to 1G. This must also be specified
// for the server by adding "max_allowed_packet = 1G" to the [mysqld]
// section of my.cnf
sqlExec(dbImpl, "set global max_allowed_packet=1073741824");
// set the idle timeout to a week
int waitTimeout = 7 * 24 * 60 * 60; // 1 week
sqlExec(dbImpl, "set wait_timeout=%d", waitTimeout);
// set the read timeout to an hour
int readTimeout = 60 * 60; // 1 hour
sqlExec(dbImpl, "set net_read_timeout=%d", readTimeout);
// NOTE: commit will not return an error, this does row-level locking on
// the select done before the update
sqlExec(dbImpl, "set autocommit = 0;");
sqlExec(dbImpl, "set session transaction isolation level serializable;");
return dbImpl;
}
static void thrower2() {
stThrowNew(ERR2, "error in %s", "thrower2");
}
/* check if the sorted set is in a state that it can be modified */
static void checkModifiable(stSortedSet *sortedSet) {
if (sortedSet->numberOfLiveIterators > 0) {
//assert(0);
stThrowNew(SORTED_SET_EXCEPTION_ID, "attempt to modify an stSortedSet while iterators are active");
}
}
static CactusDisk *cactusDisk_constructPrivate(stKVDatabaseConf *conf, bool create, const char *sequencesFileName) {
//sequencesFileName = NULL; //Disable the ability to store the sequences on disk.
CactusDisk *cactusDisk = st_calloc(1, sizeof(CactusDisk));
//construct lists of in memory objects
cactusDisk->metaSequences = stSortedSet_construct3(cactusDisk_constructMetaSequencesP, NULL);
cactusDisk->flowers = stSortedSet_construct3(cactusDisk_constructFlowersP, NULL);
cactusDisk->flowerNamesMarkedForDeletion = stSortedSet_construct3((int (*)(const void *, const void *)) strcmp,
free);
cactusDisk->updateRequests = stList_construct3(0, (void (*)(void *)) stKVDatabaseBulkRequest_destruct);
//Now open the database
cactusDisk->database = stKVDatabase_construct(conf, create);
cactusDisk->cache = stCache_construct();
cactusDisk->stringCache = stCache_construct();
//initialise the unique ids.
int64_t seed = (clock() << 24) | (time(NULL) << 16) | (getpid() & 65535); //Likely to be unique
st_logDebug("The cactus disk is seeding the random number generator with the value %" PRIi64 "\n", seed);
st_randomSeed(seed);
cactusDisk->uniqueNumber = 0;
cactusDisk->maxUniqueNumber = 0;
//Now load any stuff..
if (containsRecord(cactusDisk, CACTUS_DISK_PARAMETER_KEY)) {
if (create) {
stThrowNew(CACTUS_DISK_EXCEPTION_ID, "Tried to create a cactus disk, but the cactus disk already exists");
}
if (sequencesFileName != NULL) {
stThrowNew(CACTUS_DISK_EXCEPTION_ID,
"A sequences file name is specified, but the cactus disk is not being created");
}
void *record = getRecord(cactusDisk, CACTUS_DISK_PARAMETER_KEY, "cactus_disk parameters");
void *record2 = record;
cactusDisk_loadFromBinaryRepresentation(&record, cactusDisk, conf);
free(record2);
} else {
assert(create);
if (sequencesFileName == NULL) {
cactusDisk->storeSequencesInAFile = 0;
cactusDisk->sequencesFileName = NULL;
cactusDisk->sequencesReadFileHandle = NULL;
cactusDisk->sequencesWriteFileHandle = NULL;
cactusDisk->absSequencesFileName = NULL;
} else {
if (stKVDatabaseConf_getDir(conf) == NULL) {
stThrowNew(CACTUS_DISK_EXCEPTION_ID,
"The database conf does not contain a directory in which the sequence file is to be found!\n");
}
cactusDisk->storeSequencesInAFile = 1;
cactusDisk->sequencesFileName = stString_copy(sequencesFileName);
cactusDisk->absSequencesFileName = stString_print("%s/%s", stKVDatabaseConf_getDir(conf),
cactusDisk->sequencesFileName);
//Make sure the file exists
cactusDisk->sequencesReadFileHandle = fopen(cactusDisk->absSequencesFileName, "w");
assert(cactusDisk->sequencesReadFileHandle != NULL);
fclose(cactusDisk->sequencesReadFileHandle); //Flush it first time.
cactusDisk->sequencesReadFileHandle = NULL;
cactusDisk->sequencesWriteFileHandle = NULL;
}
}
return cactusDisk;
}
