// Maps tablet data to a next action summation data structure.
//
// worker - The worker.
// tablet - The tablet to work against.
// ret - A pointer to where the summation data structure should be returned.
//
// Returns 0 if successful, otherwise returns -1.
int sky_lua_aggregate_message_worker_map(sky_worker *worker,
sky_tablet *tablet,
void **ret)
{
int rc;
lua_State *L = NULL;
bstring msgpack_ret = NULL;
sky_data_descriptor *descriptor = NULL;
assert(worker != NULL);
assert(tablet != NULL);
assert(ret != NULL);
sky_lua_aggregate_message *message = (sky_lua_aggregate_message*)worker->data;
// Initialize the path iterator.
sky_path_iterator iterator;
sky_path_iterator_init(&iterator);
// Compile Lua script.
descriptor = sky_data_descriptor_create(); check_mem(descriptor);
rc = sky_lua_initscript_with_table(message->source, tablet->table, descriptor, &L);
check(rc == 0, "Unable to initialize script");
iterator.cursor.data_descriptor = descriptor;
iterator.cursor.data = calloc(1, descriptor->data_sz); check_mem(iterator.cursor.data);
// Assign the data file to iterate over.
rc = sky_path_iterator_set_tablet(&iterator, tablet);
check(rc == 0, "Unable to initialize path iterator");
// Execute function.
lua_getglobal(L, "sky_aggregate");
lua_pushlightuserdata(L, &iterator);
rc = lua_pcall(L, 1, 1, 0);
check(rc == 0, "Unable to execute Lua script: %s", lua_tostring(L, -1));
// Execute the script and return a msgpack variable.
rc = sky_lua_msgpack_pack(L, &msgpack_ret);
check(rc == 0, "Unable to execute Lua script");
// Close Lua.
lua_close(L);
// Return msgpack encoded response.
*ret = (void*)msgpack_ret;
free(iterator.cursor.data);
sky_data_descriptor_free(descriptor);
sky_path_iterator_uninit(&iterator);
return 0;
error:
*ret = NULL;
bdestroy(msgpack_ret);
if(L) lua_close(L);
free(iterator.cursor.data);
sky_data_descriptor_free(descriptor);
sky_path_iterator_uninit(&iterator);
return -1;
}
// Creates a data descriptor with a given number of properties.
//
// Returns a reference to the new descriptor.
sky_data_descriptor *sky_data_descriptor_create()
{
sky_data_descriptor *descriptor = NULL;
// Determine the total number of properties to track.
uint32_t property_count = SKY_PROPERTY_ID_COUNT + 1;
// Allocate memory for the descriptor and child descriptors in one block.
size_t sz = sizeof(sky_data_descriptor) + (sizeof(sky_data_property_descriptor) * property_count);
descriptor = calloc(sz, 1);
check_mem(descriptor);
descriptor->property_descriptors = (sky_data_property_descriptor*)(((void*)descriptor) + sizeof(sky_data_descriptor));
descriptor->property_count = property_count;
descriptor->property_zero_descriptor = NULL;
// Initialize all property descriptors to noop.
uint32_t i;
for(i=0; i<property_count; i++) {
sky_property_id_t property_id = SKY_PROPERTY_ID_MIN + (sky_property_id_t)i;
descriptor->property_descriptors[i].property_id = property_id;
descriptor->property_descriptors[i].set_func = sky_data_descriptor_set_noop;
// Save a pointer to the descriptor that points to property zero.
if(property_id == 0) {
descriptor->property_zero_descriptor = &descriptor->property_descriptors[i];
}
}
// Make sure the descriptor got set.
check(descriptor->property_zero_descriptor != NULL, "Property zero descriptor not initialized");
return descriptor;
error:
sky_data_descriptor_free(descriptor);
return NULL;
}
// Adds an event to the tablet.
//
// tablet - The tablet.
// event - The event to add.
//
// Returns 0 if successful, otherwise returns -1.
int sky_tablet_add_event(sky_tablet *tablet, sky_event *event)
{
int rc;
char *errptr = NULL;
void *new_data = NULL;
void *data = NULL;
sky_data_object *data_object = NULL;
sky_data_descriptor *descriptor = NULL;
sky_cursor cursor; memset(&cursor, 0, sizeof(cursor));
assert(tablet != NULL);
assert(event != NULL);
// Make sure that this event is being added to the correct tablet.
sky_tablet *target_tablet = NULL;
rc = sky_table_get_target_tablet(tablet->table, event->object_id, &target_tablet);
check(rc == 0, "Unable to determine target tablet");
check(tablet == target_tablet, "Event added to invalid tablet; IDX:%d of %d, OID:%d", tablet->index, tablet->table->tablet_count, event->object_id);
// Retrieve the existing value.
size_t data_length;
data = (void*)leveldb_get(tablet->leveldb_db, tablet->readoptions, (const char*)&event->object_id, sizeof(event->object_id), &data_length, &errptr);
check(errptr == NULL, "LevelDB get error: %s", errptr);
// Find the insertion point on the path.
size_t insert_offset = 0;
size_t event_length;
// If the object doesn't exist yet then just set the single event. Easy peasy.
if(data == NULL) {
event_length = sky_event_sizeof(event);
new_data = calloc(1, event_length); check_mem(new_data);
insert_offset = 0;
}
// If the object does exist, we need to find where to insert the event data.
// Also, we need to strip off any state which is redundant at the point of
// insertion.
else {
void *insert_ptr = NULL;
// Initialize data descriptor.
descriptor = sky_data_descriptor_create(); check_mem(descriptor);
rc = sky_data_descriptor_init_with_event(descriptor, event);
check(rc == 0, "Unable to initialize data descriptor for event insert");
// Initialize data object.
data_object = calloc(1, descriptor->data_sz); check_mem(data);
// Attach data & descriptor to the cursor.
cursor.data_descriptor = descriptor;
cursor.data = (void*)data_object;
// Initialize the cursor.
rc = sky_cursor_set_ptr(&cursor, data, data_length);
check(rc == 0, "Unable to set pointer on cursor");
// Loop over cursor until we reach the event insertion point.
while(!cursor.eof) {
// Retrieve event insertion pointer once the timestamp is reached.
if(data_object->ts >= event->timestamp) {
insert_ptr = cursor.ptr;
break;
}
// Move to next event.
check(sky_cursor_next_event(&cursor) == 0, "Unable to move to next event");
}
// If no insertion point was found then append the event to the
// end of the path.
if(insert_ptr == NULL) {
insert_ptr = data + data_length;
}
insert_offset = insert_ptr - data;
// Clear off any object data on the event that matches
// what is the current state of the event in the database.
uint32_t i;
for(i=0; i<event->data_count; i++) {
// Ignore any action properties.
if(event->data[i]->key > 0) {
sky_data_property_descriptor *property_descriptor = &descriptor->property_zero_descriptor[event->data[i]->key];
// If the values match then splice this from the array.
// Compare strings.
void *a = &event->data[i]->value;
void *b = ((void*)data_object)+property_descriptor->offset;
size_t n = sky_data_type_sizeof(event->data[i]->data_type);
bool is_equal = false;
if(event->data[i]->data_type == SKY_DATA_TYPE_STRING) {
is_equal = sky_string_bequals((sky_string*)b, event->data[i]->string_value);
}
// Compare other types.
else if(memcmp(a, b, n) == 0) {
is_equal = true;
}
// If the data is equal then remove it.
if(is_equal) {
sky_event_data_free(event->data[i]);
if(i < event->data_count - 1) {
memmove(&event->data[i], &event->data[i+1], (event->data_count-i-1) * sizeof(*event->data));
}
i--;
event->data_count--;
}
}
}
// Determine the serialized size of the event. If the event is
// completely redundant (e.g. it is a data-only event and the event
// matches the current object state) then don't allocate space for a
// new path value.
event_length = sky_event_sizeof(event);
if(event_length > 0) {
// Allocate space for the existing data plus the new data.
new_data = calloc(1, data_length + event_length); check_mem(new_data);
// Copy in data before event.
if(insert_offset > 0) {
memmove(new_data, data, insert_ptr-data);
}
// Copy in data after event.
if(insert_offset < data_length) {
event_length = sky_event_sizeof(event);
memmove(new_data+insert_offset+event_length, data+insert_offset, data_length-insert_offset);
}
}
}
// If no space was allocated then it means the event is redundant and
// should be ignored.
if(new_data != NULL) {
// If the object doesn't exist then just set the event as the data.
size_t event_sz;
rc = sky_event_pack(event, new_data + insert_offset, &event_sz);
check(rc == 0, "Unable to pack event");
check(event_sz == event_length, "Expected event size (%ld) does not match actual event size (%ld)", event_length, event_sz);
leveldb_put(tablet->leveldb_db, tablet->writeoptions, (const char*)&event->object_id, sizeof(event->object_id), new_data, data_length + event_length, &errptr);
check(errptr == NULL, "LevelDB put error: %s", errptr);
}
free(data_object);
sky_data_descriptor_free(descriptor);
free(data);
free(new_data);
return 0;
error:
if(errptr) leveldb_free(errptr);
sky_data_descriptor_free(descriptor);
if(data) free(data);
if(new_data) free(new_data);
if(data_object) free(data_object);
return -1;
}