int rl_dirty_hash(struct rlite *db, unsigned char **hash)
{
long i;
int retval = RL_OK;
rl_page *page;
SHA1_CTX sha;
unsigned char *data = NULL;
if (db->write_pages_len == 0) {
*hash = NULL;
goto cleanup;
}
RL_MALLOC(data, db->page_size * sizeof(unsigned char));
RL_MALLOC(*hash, sizeof(unsigned char) * 20);
SHA1Init(&sha);
for (i = 0; i < db->write_pages_len; i++) {
page = db->write_pages[i];
memset(data, 0, db->page_size);
if (page->type) {
retval = page->type->serialize(db, page->obj, data);
}
SHA1Update(&sha, data, db->page_size);
}
SHA1Final(*hash, &sha);
cleanup:
rl_free(data);
if (retval != RL_OK) {
rl_free(*hash);
*hash = NULL;
}
return retval;
}
int rl_key_set(rlite *db, const unsigned char *key, long keylen, unsigned char type, long value_page, unsigned long long expires, long version)
{
int retval;
rl_key *key_obj = NULL;
unsigned char *digest = NULL;
RL_CALL2(rl_key_delete, RL_OK, RL_NOT_FOUND, db, key, keylen);
RL_MALLOC(digest, sizeof(unsigned char) * 20);
RL_CALL(sha1, RL_OK, key, keylen, digest);
rl_btree *btree;
RL_CALL(rl_get_key_btree, RL_OK, db, &btree, 1);
RL_MALLOC(key_obj, sizeof(*key_obj))
RL_CALL(rl_multi_string_set, RL_OK, db, &key_obj->string_page, key, keylen);
key_obj->type = type;
key_obj->value_page = value_page;
key_obj->expires = expires;
// reserving version=0 for non existent keys
if (version == 0) {
version = 1;
}
key_obj->version = version;
RL_CALL(rl_btree_add_element, RL_OK, db, btree, db->databases[rl_get_selected_db(db)], digest, key_obj);
retval = RL_OK;
cleanup:
if (retval != RL_OK) {
rl_free(digest);
rl_free(key_obj);
}
return retval;
}
int rl_header_deserialize(struct rlite *db, void **UNUSED(obj), void *UNUSED(context), unsigned char *data)
{
int retval = RL_OK;
int identifier_len = strlen((char *)identifier);
if (memcmp(data, identifier, identifier_len) != 0) {
fprintf(stderr, "Unexpected header, expecting %s\n", identifier);
return RL_INVALID_STATE;
}
db->page_size = get_4bytes(&data[identifier_len]);
db->initial_next_empty_page =
db->next_empty_page = get_4bytes(&data[identifier_len + 4]);
db->initial_number_of_pages =
db->number_of_pages = get_4bytes(&data[identifier_len + 8]);
db->initial_number_of_databases =
db->number_of_databases = get_4bytes(&data[identifier_len + 12]);
rl_free(db->databases);
rl_free(db->initial_databases);
RL_MALLOC(db->databases, sizeof(long) * (db->number_of_databases + RLITE_INTERNAL_DB_COUNT));
RL_MALLOC(db->initial_databases, sizeof(long) * (db->number_of_databases + RLITE_INTERNAL_DB_COUNT));
long i, pos = identifier_len + 16;
for (i = 0; i < db->number_of_databases + RLITE_INTERNAL_DB_COUNT; i++) {
db->initial_databases[i] =
db->databases[i] = get_4bytes(&data[pos]);
pos += 4;
}
cleanup:
return retval;
}
static int add_member(rlite *db, rl_btree *scores, long scores_page, rl_skiplist *skiplist, long skiplist_page, double score, unsigned char *member, long memberlen)
{
int retval;
unsigned char *digest = NULL;
void *value_ptr;
RL_MALLOC(value_ptr, sizeof(double));
digest = rl_malloc(sizeof(unsigned char) * 20);
if (!digest) {
rl_free(value_ptr);
retval = RL_OUT_OF_MEMORY;
goto cleanup;
}
*(double *)value_ptr = score;
retval = sha1(member, memberlen, digest);
if (retval != RL_OK) {
rl_free(value_ptr);
rl_free(digest);
goto cleanup;
}
RL_CALL(rl_btree_add_element, RL_OK, db, scores, scores_page, digest, value_ptr);
retval = rl_skiplist_add(db, skiplist, skiplist_page, score, member, memberlen);
if (retval != RL_OK) {
// This failure is critical. The btree already has the element, but
// the skiplist failed to add it. If left as is, it would be in an
// inconsistent state. Dropping all the transaction in progress.
rl_discard(db);
goto cleanup;
}
cleanup:
return retval;
}
int rl_list_iterator_next(rl_list_iterator *iterator, void **element)
{
int retval;
if (iterator->node == NULL) {
retval = RL_END;
goto cleanup;
}
if (element) {
RL_MALLOC(*element, iterator->list->type->element_size);
memcpy(*element, iterator->node->elements[iterator->node_position], iterator->list->type->element_size);
}
iterator->node_position += iterator->direction;
if (iterator->node_position < 0 || iterator->node_position == iterator->node->size) {
long next_node_page = iterator->direction == 1 ? iterator->node->right : iterator->node->left;
RL_CALL(rl_list_node_nocache_destroy, RL_OK, iterator->db, iterator->node);
iterator->node = NULL;
if (next_node_page) {
void *_node;
RL_CALL(rl_read, RL_FOUND, iterator->db, iterator->list->type->list_node_type, next_node_page, iterator->list, &_node, 0);
iterator->node = _node;
iterator->node_position = iterator->direction == 1 ? 0 : (iterator->node->size - 1);
}
}
if (iterator->node && iterator->node_position < -1) {
retval = RL_UNEXPECTED;
goto cleanup;
}
retval = RL_OK;
cleanup:
if (retval != RL_OK) {
rl_list_iterator_destroy(iterator->db, iterator);
}
return retval;
}
int rl_list_iterator_create(rlite *db, rl_list_iterator **_iterator, rl_list *list, int direction)
{
void *_node;
int retval;
rl_list_iterator *iterator = NULL;
RL_MALLOC(iterator, sizeof(*iterator));
iterator->db = db;
iterator->list = list;
iterator->node = NULL;
if (direction < 0) {
iterator->direction = -1;
RL_CALL(rl_read, RL_FOUND, db, list->type->list_node_type, list->right, list, &_node, 0);
iterator->node = _node;
iterator->node_position = iterator->node->size - 1;
}
else {
iterator->direction = 1;
RL_CALL(rl_read, RL_FOUND, db, list->type->list_node_type, list->left, list, &_node, 0);
iterator->node = _node;
iterator->node_position = 0;
}
*_iterator = iterator;
retval = RL_OK;
cleanup:
if (iterator && retval != RL_OK) {
rl_list_iterator_destroy(db, iterator);
}
return retval;
}
int rl_key_delete(struct rlite *db, const unsigned char *key, long keylen)
{
int retval;
void *tmp;
unsigned char *digest;
rl_btree *btree = NULL;
rl_key *key_obj = NULL;
RL_MALLOC(digest, sizeof(unsigned char) * 20);
RL_CALL(sha1, RL_OK, key, keylen, digest);
RL_CALL(rl_get_key_btree, RL_OK, db, &btree, 0);
retval = rl_btree_find_score(db, btree, digest, &tmp, NULL, NULL);
if (retval == RL_FOUND) {
int selected_database = rl_get_selected_db(db);
key_obj = tmp;
RL_CALL(rl_multi_string_delete, RL_OK, db, key_obj->string_page);
retval = rl_btree_remove_element(db, btree, db->databases[selected_database], digest);
if (retval == RL_DELETED) {
db->databases[selected_database] = 0;
retval = RL_OK;
}
else if (retval != RL_OK) {
goto cleanup;
}
}
cleanup:
rl_free(digest);
return retval;
}
static int rl_zset_create(rlite *db, long levels_page_number, rl_btree **btree, long *btree_page, rl_skiplist **_skiplist, long *skiplist_page)
{
rl_list *levels;
rl_btree *scores = NULL;
rl_skiplist *skiplist = NULL;
long scores_page_number;
long skiplist_page_number;
int retval;
RL_CALL(rl_btree_create, RL_OK, db, &scores, &rl_btree_type_hash_sha1_double);
scores_page_number = db->next_empty_page;
RL_CALL(rl_write, RL_OK, db, &rl_data_type_btree_hash_sha1_double, scores_page_number, scores);
RL_CALL(rl_skiplist_create, RL_OK, db, &skiplist);
skiplist_page_number = db->next_empty_page;
RL_CALL(rl_write, RL_OK, db, &rl_data_type_skiplist, skiplist_page_number, skiplist);
RL_CALL(rl_list_create, RL_OK, db, &levels, &rl_list_type_long);
RL_CALL(rl_write, RL_OK, db, &rl_data_type_list_long, levels_page_number, levels);
long *scores_element;
RL_MALLOC(scores_element, sizeof(long));
*scores_element = scores_page_number;
RL_CALL(rl_list_add_element, RL_OK, db, levels, levels_page_number, scores_element, 0);
long *skiplist_element;
RL_MALLOC(skiplist_element, sizeof(long))
*skiplist_element = skiplist_page_number;
RL_CALL(rl_list_add_element, RL_OK, db, levels, levels_page_number, skiplist_element, 1);
if (btree) {
*btree = scores;
}
if (btree_page) {
*btree_page = scores_page_number;
}
if (_skiplist) {
*_skiplist = skiplist;
}
if (skiplist_page) {
*skiplist_page = skiplist_page_number;
}
cleanup:
return retval;
}
int rl_string_deserialize(rlite *db, void **obj, void *UNUSED(context), unsigned char *data)
{
int retval;
unsigned char *new_data;
RL_MALLOC(new_data, sizeof(char) * db->page_size);
memcpy(new_data, data, sizeof(char) * db->page_size);
*obj = new_data;
retval = RL_OK;
cleanup:
return retval;
}
int rl_list_node_create(rlite *UNUSED(db), rl_list *list, rl_list_node **_node)
{
int retval;
rl_list_node *node;
RL_MALLOC(node, sizeof(rl_list_node));
RL_MALLOC(node->elements, sizeof(void *) * list->max_node_size);
node->size = 0;
node->left = 0;
node->right = 0;
*_node = node;
retval = RL_OK;
cleanup:
if (retval != RL_OK) {
if (node) {
rl_free(node->elements);
}
rl_free(node);
}
return retval;
}
int rl_create_db(rlite *db)
{
int retval, i;
RL_CALL(rl_ensure_pages, RL_OK, db);
db->initial_next_empty_page =
db->next_empty_page = 1;
db->initial_number_of_pages =
db->number_of_pages = 1;
db->selected_database = 0;
db->selected_internal = RLITE_INTERNAL_DB_NO;
db->initial_number_of_databases =
db->number_of_databases = 16;
RL_MALLOC(db->databases, sizeof(long) * (db->number_of_databases + RLITE_INTERNAL_DB_COUNT));
RL_MALLOC(db->initial_databases, sizeof(long) * (db->number_of_databases + RLITE_INTERNAL_DB_COUNT));
for (i = 0; i < db->number_of_databases + RLITE_INTERNAL_DB_COUNT; i++) {
db->initial_databases[i] =
db->databases[i] = 0;
}
cleanup:
return retval;
}
int rl_list_deserialize(rlite *UNUSED(db), void **obj, void *context, unsigned char *data)
{
rl_list *list;
int retval = RL_OK;
RL_MALLOC(list, sizeof(*list));
list->type = context;
list->left = get_4bytes(data);
list->right = get_4bytes(&data[4]);
list->max_node_size = get_4bytes(&data[8]);
list->size = get_4bytes(&data[12]);
*obj = list;
cleanup:
return retval;
}
int rl_commit(struct rlite *db)
{
int retval;
RL_CALL(rl_write_apply_wal, RL_OK, db);
db->initial_next_empty_page = db->next_empty_page;
db->initial_number_of_pages = db->number_of_pages;
db->initial_number_of_databases = db->number_of_databases;
rl_free(db->initial_databases);
RL_MALLOC(db->initial_databases, sizeof(long) * (db->number_of_databases + RLITE_INTERNAL_DB_COUNT));
memcpy(db->initial_databases, db->databases, sizeof(long) * (db->number_of_databases + RLITE_INTERNAL_DB_COUNT));
rl_discard(db);
cleanup:
return retval;
}
UWPMessage* CreateUWPMessage(void)
{
UWPMessage *msg = (UWPMessage *)RL_MALLOC(sizeof(UWPMessage));
msg->type = UWP_MSG_NONE;
Vector2 v0 = { 0, 0 };
msg->paramVector0 = v0;
msg->paramInt0 = 0;
msg->paramInt1 = 0;
msg->paramChar0 = 0;
msg->paramFloat0 = 0;
msg->paramDouble0 = 0;
msg->paramBool0 = false;
return msg;
}
static int rl_get_zscore(rlite *db, rl_btree *scores, unsigned char *member, long memberlen, double *score)
{
unsigned char *digest = NULL;
int retval;
RL_MALLOC(digest, sizeof(unsigned char) * 20);
RL_CALL(sha1, RL_OK, member, memberlen, digest);
void *value;
RL_CALL(rl_btree_find_score, RL_FOUND, db, scores, digest, &value, NULL, NULL);
*score = *(double *)value;
retval = RL_FOUND;
cleanup:
rl_free(digest);
return retval;
}
int rl_multi_string_set(struct rlite *db, long *number, const unsigned char *data, long size)
{
int retval;
long *page = NULL;
rl_list *list = NULL;
RL_CALL(rl_list_create, RL_OK, db, &list, &rl_list_type_long);
*number = db->next_empty_page;
RL_CALL(rl_write, RL_OK, db, &rl_data_type_list_long, *number, list);
RL_MALLOC(page, sizeof(*page));
*page = size;
RL_CALL(rl_list_add_element, RL_OK, db, list, *number, page, -1);
page = NULL;
RL_CALL(append, RL_OK, db, list, *number, data, size);
cleanup:
return retval;
}
int rl_multi_string_append(struct rlite *db, long number, const unsigned char *data, long datasize, long *newlength)
{
rl_list *list = NULL;
unsigned char *tmp_data;
void *tmp;
int retval;
long size, cpsize;
long string_page_number;
RL_CALL(rl_read, RL_FOUND, db, &rl_data_type_list_long, number, &rl_list_type_long, &tmp, 0);
list = tmp;
RL_CALL(rl_list_get_element, RL_FOUND, db, list, &tmp, 0);
size = *(long *)tmp;
RL_MALLOC(tmp, sizeof(long));
*(long *)tmp = size + datasize;
RL_CALL(rl_list_add_element, RL_OK, db, list, number, tmp, 0);
RL_CALL(rl_list_remove_element, RL_OK, db, list, number, 1);
if (newlength) {
*newlength = size + datasize;
}
size = size % db->page_size;
if (size > 0) {
RL_CALL(rl_list_get_element, RL_FOUND, db, list, &tmp, -1);
string_page_number = *(long *)tmp;
RL_CALL(rl_string_get, RL_OK, db, &tmp_data, string_page_number);
cpsize = db->page_size - size;
if (cpsize > datasize) {
cpsize = datasize;
}
memcpy(&tmp_data[size], data, cpsize * sizeof(unsigned char));
RL_CALL(rl_write, RL_OK, db, &rl_data_type_string, string_page_number, tmp_data);
data = &data[cpsize];
datasize -= cpsize;
}
if (datasize > 0) {
RL_CALL(append, RL_OK, db, list, number, data, datasize);
}
retval = RL_OK;
cleanup:
return retval;
}
int rl_is_balanced(rlite *db)
{
int retval;
long i, selected_database = db->selected_database;
short *pages = NULL;
long missing_pages = 0;
RL_MALLOC(pages, sizeof(short) * db->number_of_pages);
for (i = 1; i < db->number_of_pages; i++) {
pages[i] = 0;
}
for (i = 0; i < db->number_of_databases + RLITE_INTERNAL_DB_COUNT; i++) {
if (db->databases[i] == 0) {
continue;
}
pages[db->databases[i]] = 1;
RL_CALL(rl_select, RL_OK, db, i);
RL_CALL(rl_database_is_balanced, RL_OK, db, pages);
}
RL_CALL(rl_select, RL_OK, db, selected_database);
long page_number = db->next_empty_page;
while (page_number != db->number_of_pages) {
pages[page_number] = 1;
RL_CALL(rl_long_get, RL_OK, db, &page_number, page_number);
}
for (i = 1; i < db->number_of_pages; i++) {
if (pages[i] == 0) {
fprintf(stderr, "Found orphan page %ld\n", i);
missing_pages++;
}
}
if (missing_pages) {
fprintf(stderr, "Missing %ld pages\n", missing_pages);
retval = RL_UNEXPECTED;
}
cleanup:
rl_free(pages);
return retval;
}
int rl_watch(struct rlite *db, struct watched_key** _watched_key, const unsigned char *key, long keylen) {
int retval;
struct watched_key* wkey = NULL;
RL_MALLOC(wkey, sizeof(struct watched_key));
wkey->database = rl_get_selected_db(db);
RL_CALL(sha1, RL_OK, key, keylen, wkey->digest);
RL_CALL2(rl_key_get_hash_ignore_expire, RL_FOUND, RL_NOT_FOUND, db, wkey->digest, NULL, NULL, NULL, NULL, &wkey->version, 1);
if (retval == RL_NOT_FOUND) {
wkey->version = 0;
}
*_watched_key = wkey;
retval = RL_OK;
cleanup:
if (retval != RL_OK) {
rl_free(wkey);
}
return retval;
}
int rl_list_create(rlite *db, rl_list **_list, rl_list_type *type)
{
int retval;
rl_list *list;
RL_MALLOC(list, sizeof(rl_list))
list->max_node_size = (db->page_size - 12) / type->element_size;
list->type = type;
rl_list_node *node;
RL_CALL(rl_list_node_create, RL_OK, db, list, &node);
node->left = 0;
node->right = 0;
list->size = 0;
list->left = db->next_empty_page;
RL_CALL(rl_write, RL_OK, db, list->type->list_node_type, db->next_empty_page, node);
list->right = list->left;
*_list = list;
retval = RL_OK;
cleanup:
if (retval != RL_OK) {
rl_list_destroy(db, list);
}
return retval;
}
int rl_list_node_deserialize_long(rlite *db, void **obj, void *context, unsigned char *data)
{
rl_list *list = context;
rl_list_node *node = NULL;
long i = 0, pos = 12;
int retval;
RL_CALL(rl_list_node_create, RL_OK, db, list, &node);
node->size = (long)get_4bytes(data);
node->left = (long)get_4bytes(&data[4]);
node->right = (long)get_4bytes(&data[8]);
for (i = 0; i < node->size; i++) {
RL_MALLOC(node->elements[i], sizeof(long))
*(long *)node->elements[i] = get_4bytes(&data[pos]);
pos += 4;
}
*obj = node;
retval = RL_OK;
cleanup:
if (retval != RL_OK && node) {
node->size = i;
rl_list_node_destroy(db, node);
}
return retval;
}
static int append(struct rlite *db, rl_list *list, long list_page_number, const unsigned char *data, long size)
{
int retval = RL_OK;
long *page = NULL;
long pos = 0, to_copy;
unsigned char *string = NULL;
while (pos < size) {
RL_MALLOC(page, sizeof(*page));
RL_CALL(rl_string_create, RL_OK, db, &string, page);
to_copy = db->page_size;
if (pos + to_copy > size) {
to_copy = size - pos;
}
memcpy(string, &data[pos], sizeof(unsigned char) * to_copy);
string = NULL;
RL_CALL(rl_list_add_element, RL_OK, db, list, list_page_number, page, -1);
page = NULL;
pos += to_copy;
}
cleanup:
rl_free(string);
rl_free(page);
return retval;
}
static int zunionstore_minmax(rlite *db, long keys_size, unsigned char **keys, long *keys_len, double *weights, int aggregate)
{
rl_btree *target_scores;
long target_scores_page, target_skiplist_page;
rl_skiplist *target_skiplist;
int retval;
rl_skiplist_iterator **iterators = NULL;;
rl_skiplist *skiplist;
double *scores = NULL, score;
unsigned char **members = NULL;
long *memberslen = NULL, i;
long position;
RL_CALL(rl_zset_get_objects, RL_OK, db, keys[0], keys_len[0], NULL, &target_scores, &target_scores_page, &target_skiplist, &target_skiplist_page, 1, 1);
RL_MALLOC(scores, sizeof(double) * keys_size);
RL_MALLOC(members, sizeof(unsigned char *) * keys_size);
for (i = 0; i < keys_size; i++) {
members[i] = NULL;
}
RL_MALLOC(memberslen, sizeof(long) * keys_size);
RL_MALLOC(iterators, sizeof(rl_skiplist_iterator *) * keys_size);
for (i = 0; i < keys_size; i++) {
iterators[i] = NULL;
}
for (i = 0; i < keys_size; i++) {
retval = rl_zset_get_objects(db, keys[i], keys_len[i], NULL, NULL, NULL, &skiplist, NULL, 0, 0);
if (retval == RL_NOT_FOUND) {
iterators[i] = NULL;
continue;
}
if (retval != RL_OK) {
goto cleanup;
}
RL_CALL(rl_skiplist_iterator_create, RL_OK, db, &iterators[i], skiplist, 0, aggregate == RL_ZSET_AGGREGATE_MAX ? -1 : 1, 0);
retval = rl_zset_iterator_next(iterators[i], &scores[i], &members[i], &memberslen[i]);
if (retval != RL_OK) {
iterators[i] = NULL;
if (retval != RL_END) {
goto cleanup;
}
}
}
while (1) {
position = -1;
for (i = 0; i < keys_size; i++) {
if (!iterators[i]) {
continue;
}
if (position == -1) {
position = i;
continue;
}
score = weights ? weights[position - 1] * scores[position] : scores[position];
if (isnan(score)) {
score = 0.0;
}
if ((aggregate == RL_ZSET_AGGREGATE_MAX && scores[i] > score) ||
(aggregate == RL_ZSET_AGGREGATE_MIN && scores[i] < score)) {
position = i;
}
}
if (position == -1) {
break;
}
score = weights ? weights[position - 1] * scores[position] : scores[position];
if (isnan(score)) {
score = 0.0;
}
retval = add_member(db, target_scores, target_scores_page, target_skiplist, target_skiplist_page, score, members[position], memberslen[position]);
if (retval != RL_FOUND && retval != RL_OK) {
goto cleanup;
}
rl_free(members[position]);
members[position] = NULL;
retval = rl_zset_iterator_next(iterators[position], &scores[position], &members[position], &memberslen[position]);
if (retval != RL_OK) {
iterators[position] = NULL;
if (retval != RL_END) {
goto cleanup;
}
}
}
if (target_scores->number_of_elements == 0) {
RL_CALL(rl_key_delete_with_value, RL_OK, db, keys[0], keys_len[0]);
}
retval = RL_OK;
cleanup:
if (iterators) {
for (i = 0; i < keys_size; i++) {
if (iterators[i]) {
rl_zset_iterator_destroy(iterators[i]);
}
rl_free(members[i]);
}
}
rl_free(iterators);
rl_free(memberslen);
rl_free(members);
rl_free(scores);
return retval;
}
int rl_zinterstore(rlite *db, long keys_size, unsigned char **keys, long *keys_len, double *_weights, int aggregate)
{
unsigned char *member = NULL;
rl_btree **btrees = NULL;
rl_skiplist **skiplists = NULL;
double weight = 1.0, weight_tmp;
double *weights = NULL;
rl_skiplist_node *node;
rl_skiplist_iterator *skiplist_iterator = NULL;
rl_btree_iterator *btree_iterator = NULL;
int retval;
long target_btree_page, target_skiplist_page;
long multi_string_page;
rl_btree *target_btree;
rl_skiplist *target_skiplist;
int found;
void *tmp;
double skiplist_score, tmp_score;
long memberlen;
unsigned char digest[20];
if (keys_size > 1) {
RL_MALLOC(btrees, sizeof(rl_btree *) * (keys_size - 1));
RL_MALLOC(skiplists, sizeof(rl_skiplist *) * (keys_size - 1));
}
else {
retval = RL_UNEXPECTED;
goto cleanup;
}
retval = rl_key_delete_with_value(db, keys[0], keys_len[0]);
if (retval != RL_OK && retval != RL_NOT_FOUND) {
goto cleanup;
}
rl_btree *btree, *btree_tmp;
rl_skiplist *skiplist = NULL, *skiplist_tmp;
long i;
// key in position 0 is the target key
// we'll store a pivot skiplist in btree/skiplist and the others in btrees/skiplists
// TODO: qsort instead
RL_MALLOC(weights, sizeof(double) * (keys_size - 2));
for (i = 1; i < keys_size; i++) {
weight_tmp = _weights ? _weights[i - 1] : 1.0;
RL_CALL2(rl_zset_get_objects, RL_OK, RL_WRONG_TYPE, db, keys[i], keys_len[i], NULL, &btree_tmp, NULL, &skiplist_tmp, NULL, 0, 0);
if (retval == RL_WRONG_TYPE) {
skiplist_tmp = NULL;
RL_CALL(rl_set_get_objects, RL_OK, db, keys[i], keys_len[i], NULL, &btree_tmp, 0, 0);
}
if (i == 1) {
btree = btree_tmp;
skiplist = skiplist_tmp;
weight = weight_tmp;
}
else if (btree_tmp->number_of_elements < btree->number_of_elements) {
weights[i - 2] = weight;
weight = weight_tmp;
btrees[i - 2] = btree;
btree = btree_tmp;
skiplists[i - 2] = skiplist;
skiplist = skiplist_tmp;
}
else {
weights[i - 2] = weight_tmp;
btrees[i - 2] = btree_tmp;
skiplists[i - 2] = skiplist_tmp;
}
}
RL_CALL(rl_zset_get_objects, RL_OK, db, keys[0], keys_len[0], NULL, &target_btree, &target_btree_page, &target_skiplist, &target_skiplist_page, 1, 1);
if (skiplist) {
RL_CALL(rl_skiplist_iterator_create, RL_OK, db, &skiplist_iterator, skiplist, 0, 0, 0);
} else {
RL_CALL(rl_btree_iterator_create, RL_OK, db, btree, &btree_iterator);
}
while ((retval = skiplist ? rl_skiplist_iterator_next(skiplist_iterator, &node) : rl_btree_iterator_next(btree_iterator, NULL, &tmp)) == RL_OK) {
found = 1;
if (skiplist) {
skiplist_score = node->score * weight;
multi_string_page = node->value;
} else {
skiplist_score = weight;
multi_string_page = *(long *)tmp;
rl_free(tmp);
}
for (i = 1; i < keys_size - 1; i++) {
RL_CALL(rl_multi_string_sha1, RL_OK, db, digest, multi_string_page);
retval = rl_btree_find_score(db, btrees[i - 1], digest, &tmp, NULL, NULL);
if (retval == RL_NOT_FOUND) {
found = 0;
break;
}
else if (retval == RL_FOUND) {
tmp_score = skiplist ? *(double *)tmp : 1.0;
if (aggregate == RL_ZSET_AGGREGATE_SUM) {
skiplist_score += tmp_score * weights[i - 1];
}
else if (
(aggregate == RL_ZSET_AGGREGATE_MIN && tmp_score * weights[i - 1] < skiplist_score) ||
(aggregate == RL_ZSET_AGGREGATE_MAX && tmp_score * weights[i - 1] > skiplist_score)
) {
skiplist_score = tmp_score * weights[i - 1];
}
}
else {
goto cleanup;
}
}
if (found) {
RL_CALL(rl_multi_string_get, RL_OK, db, multi_string_page, &member, &memberlen);
RL_CALL(add_member, RL_OK, db, target_btree, target_btree_page, target_skiplist, target_skiplist_page, isnan(skiplist_score) ? 0.0 : skiplist_score, member, memberlen);
rl_free(member);
member = NULL;
}
}
skiplist_iterator = NULL;
btree_iterator = NULL;
if (retval != RL_END) {
goto cleanup;
}
retval = RL_OK;
cleanup:
rl_free(member);
if (skiplist_iterator) {
rl_zset_iterator_destroy(skiplist_iterator);
}
if (btree_iterator) {
rl_btree_iterator_destroy(btree_iterator);
}
rl_free(weights);
rl_free(btrees);
rl_free(skiplists);
return retval;
}
int rl_write(struct rlite *db, rl_data_type *type, long page_number, void *obj)
{
// fprintf(stderr, "w %ld %s\n", page_number, type->name);
rl_page *page = NULL;
long pos;
int retval;
if (page_number == db->next_empty_page) {
RL_CALL(rl_alloc_page_number, RL_OK, db, NULL);
retval = rl_write(db, &rl_data_type_header, 0, NULL);
if (retval != RL_OK) {
goto cleanup;
}
}
retval = rl_search_cache(db, type, page_number, NULL, &pos, NULL, db->write_pages, db->write_pages_len);
if (retval == RL_FOUND) {
if (obj != db->write_pages[pos]->obj) {
if (db->write_pages[pos]->obj) {
db->write_pages[pos]->type->destroy(db, db->write_pages[pos]->obj);
}
db->write_pages[pos]->obj = obj;
db->write_pages[pos]->type = type;
}
retval = RL_OK;
}
else if (retval == RL_NOT_FOUND) {
if (db->driver_type == RL_FILE_DRIVER) {
RL_CALL(file_driver_fp, RL_OK, db);
}
rl_ensure_pages(db);
RL_MALLOC(page, sizeof(*page));
#ifdef RL_DEBUG
page->serialized_data = NULL;
#endif
page->page_number = page_number;
page->type = type;
page->obj = obj;
if (pos < db->write_pages_len) {
memmove(&db->write_pages[pos + 1], &db->write_pages[pos], sizeof(rl_page *) * (db->write_pages_len - pos));
}
db->write_pages[pos] = page;
db->write_pages_len++;
retval = rl_search_cache(db, type, page_number, NULL, &pos, NULL, db->read_pages, db->read_pages_len);
if (retval == RL_FOUND) {
#ifdef RL_DEBUG
rl_free(db->read_pages[pos]->serialized_data);
#endif
if (db->read_pages[pos]->obj != obj) {
db->read_pages[pos]->type->destroy(db, db->read_pages[pos]->obj);
}
rl_free(db->read_pages[pos]);
memmove(&db->read_pages[pos], &db->read_pages[pos + 1], sizeof(rl_page *) * (db->read_pages_len - pos));
db->read_pages_len--;
retval = RL_OK;
}
else if (retval != RL_NOT_FOUND) {
goto cleanup;
}
retval = RL_OK;
}
cleanup:
if (retval != RL_OK) {
if (obj) {
type->destroy(db, obj);
rl_purge_cache(db, page_number);
}
if (page_number != 0) {
rl_discard(db);
}
}
return retval;
}
int rl_discard(struct rlite *db)
{
long i;
void *tmp;
int retval = RL_OK;
rl_page *page;
if (db->driver_type == RL_FILE_DRIVER) {
rl_file_driver *driver = db->driver;
if (driver->fp) {
RL_CALL(rl_flock, RL_OK, driver->fp, RLITE_FLOCK_UN);
fclose(driver->fp);
driver->fp = NULL;
}
}
for (i = 0; i < db->read_pages_len; i++) {
page = db->read_pages[i];
if (page->type == NULL) {
// read only, from wal
rl_free(page->obj);
} else if (page->type->destroy && page->obj) {
page->type->destroy(db, page->obj);
}
#ifdef RL_DEBUG
rl_free(page->serialized_data);
#endif
rl_free(page);
}
for (i = 0; i < db->write_pages_len; i++) {
page = db->write_pages[i];
if (page->type && page->type->destroy && page->obj) {
page->type->destroy(db, page->obj);
}
#ifdef RL_DEBUG
rl_free(page->serialized_data);
#endif
rl_free(page);
}
db->read_pages_len = 0;
db->write_pages_len = 0;
db->next_empty_page = db->initial_next_empty_page;
db->number_of_pages = db->initial_number_of_pages;
db->number_of_databases = db->initial_number_of_databases;
rl_free(db->databases);
RL_MALLOC(db->databases, sizeof(long) * (db->number_of_databases + RLITE_INTERNAL_DB_COUNT)); // ?
if (db->initial_databases) {
memcpy(db->databases, db->initial_databases, sizeof(long) * (db->number_of_databases + RLITE_INTERNAL_DB_COUNT));
}
if (db->read_pages_alloc != DEFAULT_READ_PAGES_LEN) {
tmp = rl_realloc(db->read_pages, sizeof(rl_page *) * DEFAULT_READ_PAGES_LEN);
if (!tmp) {
retval = RL_OUT_OF_MEMORY;
goto cleanup;
}
db->read_pages = tmp;
db->read_pages_alloc = DEFAULT_READ_PAGES_LEN;
}
if (db->write_pages_alloc > 0) {
if (db->write_pages_alloc != DEFAULT_WRITE_PAGES_LEN) {
db->write_pages_alloc = DEFAULT_WRITE_PAGES_LEN;
tmp = rl_realloc(db->write_pages, sizeof(rl_page *) * DEFAULT_WRITE_PAGES_LEN);
if (!tmp) {
retval = RL_OUT_OF_MEMORY;
goto cleanup;
}
db->write_pages = tmp;
}
}
cleanup:
return retval;
}
int rl_dump(struct rlite *db, const unsigned char *key, long keylen, unsigned char **data, long *datalen)
{
int retval;
uint64_t crc;
unsigned char type;
unsigned char *value = NULL, *value2 = NULL;
unsigned char *buf = NULL;
long buflen;
long valuelen, value2len;
unsigned char **values = NULL;
long i = -1, *valueslen = NULL;
uint32_t length;
double score;
char f[40];
RL_CALL(rl_key_get, RL_FOUND, db, key, keylen, &type, NULL, NULL, NULL, NULL);
if (type == RL_TYPE_STRING) {
RL_CALL(rl_get, RL_OK, db, key, keylen, &value, &valuelen);
RL_MALLOC(buf, sizeof(unsigned char) * (16 + valuelen));
buf[0] = REDIS_RDB_TYPE_STRING;
buf[1] = (REDIS_RDB_32BITLEN << 6);
length = htonl(valuelen);
memcpy(&buf[2], &length, 4);
memcpy(&buf[6], value, valuelen);
buflen = valuelen + 6;
} else if (type == RL_TYPE_LIST) {
RL_CALL(rl_lrange, RL_OK, db, key, keylen, 0, -1, &valuelen, &values, &valueslen);
buflen = 16;
for (i = 0; i < valuelen; i++) {
buflen += 5 + valueslen[i];
}
RL_MALLOC(buf, sizeof(unsigned char) * buflen);
buf[0] = REDIS_RDB_TYPE_LIST;
buf[1] = (REDIS_RDB_32BITLEN << 6);
length = htonl(valuelen);
memcpy(&buf[2], &length, 4);
buflen = 6;
for (i = 0; i < valuelen; i++) {
buf[buflen++] = (REDIS_RDB_32BITLEN << 6);
length = htonl(valueslen[i]);
memcpy(&buf[buflen], &length, 4);
buflen += 4;
memcpy(&buf[buflen], values[i], valueslen[i]);
buflen += valueslen[i];
}
} else if (type == RL_TYPE_SET) {
rl_set_iterator *iterator;
RL_CALL(rl_smembers, RL_OK, db, &iterator, key, keylen);
buflen = 16;
length = 0;
while ((retval = rl_set_iterator_next(iterator, NULL, &valuelen)) == RL_OK) {
buflen += 5 + valuelen;
length++;
}
if (retval != RL_END) {
goto cleanup;
}
RL_MALLOC(buf, sizeof(unsigned char) * buflen);
buf[0] = REDIS_RDB_TYPE_SET;
buf[1] = (REDIS_RDB_32BITLEN << 6);
length = htonl(length);
memcpy(&buf[2], &length, 4);
buflen = 6;
RL_CALL(rl_smembers, RL_OK, db, &iterator, key, keylen);
while ((retval = rl_set_iterator_next(iterator, &value, &valuelen)) == RL_OK) {
buf[buflen++] = (REDIS_RDB_32BITLEN << 6);
length = htonl(valuelen);
memcpy(&buf[buflen], &length, 4);
buflen += 4;
memcpy(&buf[buflen], value, valuelen);
buflen += valuelen;
rl_free(value);
value = NULL;
}
if (retval != RL_END) {
goto cleanup;
}
} else if (type == RL_TYPE_ZSET) {
rl_zset_iterator *iterator;
RL_CALL(rl_zrange, RL_OK, db, key, keylen, 0, -1, &iterator);
buflen = 16;
length = 0;
while ((retval = rl_zset_iterator_next(iterator, &score, NULL, &valuelen)) == RL_OK) {
buflen += 6 + valuelen + snprintf(f, 40, "%lf", score);
length++;
}
if (retval != RL_END) {
goto cleanup;
}
RL_MALLOC(buf, sizeof(unsigned char) * buflen);
buf[0] = REDIS_RDB_TYPE_ZSET;
buf[1] = (REDIS_RDB_32BITLEN << 6);
length = htonl(length);
memcpy(&buf[2], &length, 4);
buflen = 6;
RL_CALL(rl_zrange, RL_OK, db, key, keylen, 0, -1, &iterator);
while ((retval = rl_zset_iterator_next(iterator, &score, &value, &valuelen)) == RL_OK) {
buf[buflen++] = (REDIS_RDB_32BITLEN << 6);
length = htonl(valuelen);
memcpy(&buf[buflen], &length, 4);
buflen += 4;
memcpy(&buf[buflen], value, valuelen);
buflen += valuelen;
rl_free(value);
value = NULL;
valuelen = snprintf(f, 40, "%lf", score);
buf[buflen++] = valuelen;
memcpy(&buf[buflen], f, valuelen);
buflen += valuelen;
}
if (retval != RL_END) {
goto cleanup;
}
} else if (type == RL_TYPE_HASH) {
rl_hash_iterator *iterator;
RL_CALL(rl_hgetall, RL_OK, db, &iterator, key, keylen);
buflen = 16;
length = 0;
while ((retval = rl_hash_iterator_next(iterator, NULL, &value2len, NULL, &valuelen)) == RL_OK) {
buflen += 10 + valuelen + value2len;
length++;
}
if (retval != RL_END) {
goto cleanup;
}
RL_MALLOC(buf, sizeof(unsigned char) * buflen);
buf[0] = REDIS_RDB_TYPE_HASH;
buf[1] = (REDIS_RDB_32BITLEN << 6);
length = htonl(length);
memcpy(&buf[2], &length, 4);
buflen = 6;
RL_CALL(rl_hgetall, RL_OK, db, &iterator, key, keylen);
while ((retval = rl_hash_iterator_next(iterator, &value, &valuelen, &value2, &value2len)) == RL_OK) {
buf[buflen++] = (REDIS_RDB_32BITLEN << 6);
length = htonl(valuelen);
memcpy(&buf[buflen], &length, 4);
buflen += 4;
memcpy(&buf[buflen], value, valuelen);
buflen += valuelen;
rl_free(value);
value = NULL;
buf[buflen++] = (REDIS_RDB_32BITLEN << 6);
length = htonl(value2len);
memcpy(&buf[buflen], &length, 4);
buflen += 4;
memcpy(&buf[buflen], value2, value2len);
buflen += value2len;
rl_free(value2);
value2 = NULL;
}
} else {
retval = RL_UNEXPECTED;
goto cleanup;
}
buf[buflen++] = REDIS_RDB_VERSION;
buf[buflen++] = REDIS_RDB_VERSION >> 8;
crc = rl_crc64(0, buf, buflen);
memrev64ifbe(&crc);
memcpy(&buf[buflen], &crc, 8);
buflen += 8;
*data = buf;
*datalen = buflen;
retval = RL_OK;
cleanup:
if (values) {
for (i = 0; i < valuelen; i++) {
rl_free(values[i]);
}
rl_free(values);
}
rl_free(valueslen);
rl_free(value);
rl_free(value2);
return retval;
}
int rl_multi_string_getrange(struct rlite *db, long number, unsigned char **_data, long *size, long start, long stop)
{
long totalsize;
rl_list *list = NULL;
rl_list_node *node = NULL;
void *_list, *tmp;
unsigned char *data = NULL;
int retval;
RL_CALL(rl_read, RL_FOUND, db, &rl_data_type_list_long, number, &rl_list_type_long, &_list, 0);
list = _list;
unsigned char *tmp_data;
long i, pos = 0, pagesize, pagestart;
RL_CALL(rl_list_get_element, RL_FOUND, db, list, &tmp, 0);
totalsize = *(long *)tmp;
if (totalsize == 0) {
*size = 0;
if (_data) {
*_data = NULL;
}
retval = RL_OK;
goto cleanup;
}
rl_normalize_string_range(totalsize, &start, &stop);
if (stop < start) {
*size = 0;
if (_data) {
*_data = NULL;
}
retval = RL_OK;
goto cleanup;
}
*size = stop - start + 1;
if (!_data) {
retval = RL_OK;
goto cleanup;
}
RL_MALLOC(data, sizeof(unsigned char) * (*size + 1));
i = start / db->page_size;
pagestart = start % db->page_size;
// pos = i * db->page_size + pagestart;
// the first element in the list is the length of the array, skip to the second
for (i++; i < list->size; i++) {
RL_CALL(rl_list_get_element, RL_FOUND, db, list, &tmp, i);
RL_CALL(rl_string_get, RL_OK, db, &tmp_data, *(long *)tmp);
pagesize = db->page_size - pagestart;
if (pos + pagesize > *size) {
pagesize = *size - pos;
}
memcpy(&data[pos], &tmp_data[pagestart], sizeof(unsigned char) * pagesize);
pos += pagesize;
pagestart = 0;
}
data[*size] = 0;
*_data = data;
retval = RL_OK;
cleanup:
if (retval != RL_OK) {
rl_free(data);
}
if (list) {
rl_list_nocache_destroy(db, list);
}
if (node) {
rl_list_node_nocache_destroy(db, node);
}
return retval;
}
int rl_multi_string_setrange(struct rlite *db, long number, const unsigned char *data, long size, long offset, long *newlength)
{
long oldsize, newsize;
rl_list *list = NULL;
void *_list, *tmp;
int retval;
RL_CALL(rl_read, RL_FOUND, db, &rl_data_type_list_long, number, &rl_list_type_long, &_list, 1);
list = _list;
unsigned char *tmp_data;
long i, pagesize, pagestart, page;
if (offset < 0) {
retval = RL_INVALID_PARAMETERS;
goto cleanup;
}
RL_CALL(rl_list_get_element, RL_FOUND, db, list, &tmp, 0);
oldsize = *(long *)tmp;
if (oldsize > offset) {
i = offset / db->page_size;
pagestart = offset % db->page_size;
newsize = offset + size;
if (newsize > (list->size - 1) * db->page_size) {
newsize = (list->size - 1) * db->page_size;
}
if (newsize > oldsize) {
RL_CALL(rl_list_remove_element, RL_OK, db, list, number, 0);
RL_MALLOC(tmp, sizeof(long));
*(long *)tmp = newsize;
RL_CALL(rl_list_add_element, RL_OK, db, list, number, tmp, 0);
}
for (i++; size > 0 && i < list->size; i++) {
RL_CALL(rl_list_get_element, RL_FOUND, db, list, &tmp, i);
page = *(long *)tmp;
RL_CALL(rl_string_get, RL_OK, db, &tmp_data, page);
pagesize = db->page_size - pagestart;
if (pagesize > size) {
pagesize = size;
}
memcpy(&tmp_data[pagestart], data, sizeof(char) * pagesize);
RL_CALL(rl_write, RL_OK, db, &rl_data_type_string, page, tmp_data);
// if there's more bytes that did not enter in this page it will be caught with an append
if (oldsize < pagesize + pagestart) {
oldsize = pagesize + pagestart;
}
data += pagesize;
offset += pagesize;
size -= pagesize;
pagestart = 0;
}
if (newlength) {
*newlength = oldsize;
}
}
if (oldsize < offset) {
RL_MALLOC(tmp_data, sizeof(char) * (size + offset - oldsize));
memset(tmp_data, 0, offset - oldsize);
memcpy(&tmp_data[offset - oldsize], data, sizeof(char) * size);
RL_CALL(rl_multi_string_append, RL_OK, db, number, tmp_data, size + offset - oldsize, newlength);
rl_free(tmp_data);
}
else if (oldsize == offset && size > 0) {
RL_CALL(rl_multi_string_append, RL_OK, db, number, data, size, newlength);
}
retval = RL_OK;
cleanup:
return retval;
}
int rl_read_signal(const char *signal_name, struct timeval *timeout, char **_data, size_t *_datalen)
{
char header[FIFO_HEADER_SIZE];
uint64_t crc;
size_t readbytes;
size_t datalen;
int fd;
int retval;
char *data = NULL;
fd_set rfds;
int oflag = O_RDONLY;
if (timeout) {
// select will block, we don't want open to block
oflag |= O_NONBLOCK;
}
fd = open(signal_name, oflag);
if (fd == -1) {
retval = RL_UNEXPECTED;
goto cleanup;
}
if (timeout) {
FD_ZERO(&rfds);
FD_SET(fd, &rfds);
retval = select(fd + 1, &rfds, NULL, NULL, timeout);
if (retval == -1) {
retval = RL_UNEXPECTED;
goto cleanup;
} else if (retval != 0) {
retval = RL_TIMEOUT;
goto cleanup;
}
}
readbytes = read(fd, header, FIFO_HEADER_SIZE);
if (readbytes != FIFO_HEADER_SIZE) {
retval = RL_UNEXPECTED;
goto cleanup;
}
datalen = (size_t)get_4bytes((unsigned char *)header);
RL_MALLOC(data, sizeof(char) * datalen);
readbytes = read(fd, data, datalen);
if (readbytes != datalen) {
retval = RL_UNEXPECTED;
goto cleanup;
}
crc = rl_crc64(0, (unsigned char *)data, datalen);
memrev64ifbe(&crc);
memcpy(&header[4], &crc, 8);
if (memcmp(&crc, &header[4], 8) != 0) {
retval = RL_UNEXPECTED;
goto cleanup;
}
if (_data) {
*_data = data;
}
if (_datalen) {
*_datalen = datalen;
}
retval = RL_OK;
cleanup:
if (fd >= 0) {
close(fd);
}
if (retval != RL_OK || _data == NULL) {
rl_free(data);
}
return retval;
}