int test_sky_table_open() {
struct tagbstring lock_file_path = bsStatic("tmp/.skylock");
struct tagbstring data_file_path = bsStatic("tmp/0/data");
struct tagbstring header_file_path = bsStatic("tmp/0/header");
cleantmp();
int rc;
sky_table *table = sky_table_create();
table->path = bfromcstr("tmp");
// Open table.
rc = sky_table_open(table);
mu_assert_int_equals(rc, 0);
mu_assert(sky_file_exists(&lock_file_path), "");
mu_assert(sky_file_exists(&data_file_path), "");
mu_assert(sky_file_exists(&header_file_path), "");
// Close table.
rc = sky_table_close(table);
mu_assert_int_equals(rc, 0);
mu_assert(!sky_file_exists(&lock_file_path), "");
mu_assert(sky_file_exists(&data_file_path), "");
mu_assert(sky_file_exists(&header_file_path), "");
sky_table_free(table);
return 0;
}
int test() {
pthread_t thread;
importtmp_n("tests/functional/fixtures/add_event/0/data.json", 4);
start_server(1, &thread);
send_msg("tests/functional/fixtures/add_event/0/input");
pthread_join(thread, NULL);
mu_assert_msg("tests/functional/fixtures/add_event/0/output");
void *data;
size_t data_length;
sky_table *table = sky_table_create();
table->path = bfromcstr("tmp");
sky_table_open(table);
sky_tablet_get_path(table->tablets[2], 10, &data, &data_length);
mu_assert_mem(data,
"\x03\xe8\x03\x00\x00\x00\x00\x00\x00\x05\x00\x1f\x00\x00\x00\xff"
"\xa6\x7a\x7a\x7a\x7a\x7a\x7a\xfe\x0a\x01\xa3\x78\x79\x7a\x02\xd1"
"\x00\xc8\x03\xcb\x40\x59\x0c\xcc\xcc\xcc\xcc\xcd\x04\xc3\x01\x00"
"\x00\x10\x00\x00\x00\x00\x00\x01\x00\x01\x00\x00\x20\x00\x00\x00"
"\x00\x00\x02\x00\x01\x00\x00\x30\x00\x00\x00\x00\x00\x03\x00",
data_length
);
sky_table_free(table);
free(data);
return 0;
}
int sky_importer_process_event(sky_importer *importer, bstring source,
jsmntok_t *tokens, uint32_t *index)
{
int rc;
check(importer != NULL, "Importer required");
check(source != NULL, "Source required");
check(tokens != NULL, "Tokens required");
check(index != NULL, "Token index required");
jsmntok_t *event_token = &tokens[*index];
(*index)++;
// Open table if it hasn't been already.
if(!importer->table->opened) {
check(sky_table_open(importer->table) == 0, "Unable to open table");
}
// Create the event object.
sky_event *event = sky_event_create(0, 0, 0); check_mem(event);
// Process over child tokens.
int32_t i;
for(i=0; i<(event_token->size/2); i++) {
jsmntok_t *token = &tokens[*index];
(*index)++;
if(sky_importer_tokstr_equal(source, token, "timestamp")) {
bstring timestamp = sky_importer_token_parse_bstring(source, &tokens[(*index)++]);
rc = sky_timestamp_parse(timestamp, &event->timestamp);
check(rc == 0, "Unable to parse timestamp");
bdestroy(timestamp);
}
else if(sky_importer_tokstr_equal(source, token, "objectId")) {
event->object_id = (sky_object_id_t)sky_importer_token_parse_int(source, &tokens[(*index)++]);
}
else if(sky_importer_tokstr_equal(source, token, "action")) {
sky_action *action = NULL;
bstring action_name = sky_importer_token_parse_bstring(source, &tokens[(*index)++]);
rc = sky_action_file_find_action_by_name(importer->table->action_file, action_name, &action);
check(rc == 0, "Unable to find action: %s", bdata(action_name));
event->action_id = action->id;
}
else if(sky_importer_tokstr_equal(source, token, "data")) {
rc = sky_importer_process_event_data(importer, event, source, tokens, index);
check(rc == 0, "Unable to import event data");
}
else {
sentinel("Invalid token at char %d", tokens[*index].start);
}
}
// Add event.
rc = sky_table_add_event(importer->table, event);
check(rc == 0, "Unable to add event");
return 0;
error:
return -1;
}
int test_sky_lua_generate_header() {
importtmp("tests/fixtures/sky_lua/0/data.json");
sky_table *table = sky_table_create();
table->path = bfromcstr("tmp");
sky_table_open(table);
struct tagbstring source = bsStatic(
"function aggregate(event)\n"
" label = event:first_name() .. ' ' .. event:last_name()\n"
" return event.x + event.y\n"
"end\n"
);
bstring event_decl = NULL;
bstring event_metatype = NULL;
bstring init_descriptor_func = NULL;
int rc = sky_lua_generate_event_info(&source, table->property_file, &event_decl, &event_metatype, &init_descriptor_func);
mu_assert_int_equals(rc, 0);
mu_assert_bstring(event_decl,
"typedef struct {\n"
" int64_t ts;\n"
" uint32_t timestamp;\n"
" uint16_t action_id;\n"
" sky_string_t _first_name;\n"
" sky_string_t _last_name;\n"
" int32_t x;\n"
" int32_t y;\n"
"} sky_lua_event_t;"
);
mu_assert_bstring(event_metatype,
"ffi.metatype('sky_lua_event_t', {\n"
" __index = {\n"
" first_name = function(event) return ffi.string(event._first_name.data, event._first_name.length) end,\n"
" last_name = function(event) return ffi.string(event._last_name.data, event._last_name.length) end,\n"
" x = function(event) return event.x end,\n"
" y = function(event) return event.y end,\n"
" }\n"
"})\n"
);
mu_assert_bstring(init_descriptor_func,
"function sky_init_descriptor(_descriptor)\n"
" descriptor = ffi.cast('sky_data_descriptor_t*', _descriptor)\n"
" descriptor:set_data_sz(ffi.sizeof('sky_lua_event_t'));\n"
" descriptor:set_ts_offset(ffi.offsetof('sky_lua_event_t', 'ts'));\n"
" descriptor:set_timestamp_offset(ffi.offsetof('sky_lua_event_t', 'timestamp'));\n"
" descriptor:set_action_id_offset(ffi.offsetof('sky_lua_event_t', 'action_id'));\n"
" descriptor:set_property(4, ffi.offsetof('sky_lua_event_t', '_first_name'), 1);\n"
" descriptor:set_property(5, ffi.offsetof('sky_lua_event_t', '_last_name'), 1);\n"
" descriptor:set_property(2, ffi.offsetof('sky_lua_event_t', 'x'), 2);\n"
" descriptor:set_property(3, ffi.offsetof('sky_lua_event_t', 'y'), 2);\n"
"end\n"
);
bdestroy(event_decl);
bdestroy(event_metatype);
bdestroy(init_descriptor_func);
sky_table_free(table);
return 0;
}
int test_sky_aggregate() {
importtmp("tests/fixtures/sky_lua/1/data.json");
sky_table *table = sky_table_create();
table->path = bfromcstr("tmp");
sky_table_open(table);
sky_data_descriptor *descriptor = sky_data_descriptor_create();
// Initialize the path iterator.
sky_path_iterator iterator;
sky_path_iterator_init(&iterator);
iterator.cursor.data_descriptor = descriptor;
sky_path_iterator_set_tablet(&iterator, table->tablets[0]);
struct tagbstring source = bsStatic(
"function aggregate(cursor, data)\n"
" data.path_count = (data.path_count or 0) + 1\n"
" while cursor:next() do\n"
" data.event_count = (data.event_count or 0) + 1\n"
" data.z = (data.z or 0) + cursor.event.x + cursor.event.y\n"
" end\n"
"end\n"
);
lua_State *L = NULL;
int rc = sky_lua_initscript_with_table(&source, table, descriptor, &L);
mu_assert_int_equals(rc, 0);
// Allocate data.
mu_assert_int_equals(descriptor->data_sz, 24);
iterator.cursor.data = calloc(1, descriptor->data_sz);
// Start benchmark.
struct timeval tv;
gettimeofday(&tv, NULL);
int64_t t0 = (tv.tv_sec*1000) + (tv.tv_usec/1000);
// Call sky_aggregate() function.
uint32_t i;
for(i=0; i<1; i++) {
//sky_path_iterator_set_tablet(&iterator, table->tablets[0]);
lua_getglobal(L, "sky_aggregate");
lua_pushlightuserdata(L, &iterator);
lua_call(L, 1, 0);
}
// End benchmark.
gettimeofday(&tv, NULL);
int64_t t1 = (tv.tv_sec*1000) + (tv.tv_usec/1000);
printf("[lua] t=%.3fs\n", ((float)(t1-t0))/1000);
sky_path_iterator_uninit(&iterator);
sky_table_free(table);
free(iterator.cursor.data);
return 0;
}
int sky_importer_process_property(sky_importer *importer, bstring source,
jsmntok_t *tokens, uint32_t *index)
{
int rc;
assert(importer != NULL);
assert(source != NULL);
assert(tokens != NULL);
assert(index != NULL);
jsmntok_t *property_token = &tokens[*index];
(*index)++;
// Create the property object.
sky_property *property = sky_property_create(); check_mem(property);
// Process over child tokens.
int32_t i;
for(i=0; i<(property_token->size/2); i++) {
jsmntok_t *token = &tokens[*index];
(*index)++;
if(sky_importer_tokstr_equal(source, token, "type")) {
bstring type = sky_importer_token_parse_bstring(source, &tokens[*index]);
property->type = biseqcstr(type, "action") == 1 ? SKY_PROPERTY_TYPE_ACTION : SKY_PROPERTY_TYPE_OBJECT;
bdestroy(type);
}
else if(sky_importer_tokstr_equal(source, token, "dataType")) {
bstring data_type_str = sky_importer_token_parse_bstring(source, &tokens[*index]);
property->data_type = sky_data_type_to_enum(data_type_str);
bdestroy(data_type_str);
}
else if(sky_importer_tokstr_equal(source, token, "name")) {
property->name = sky_importer_token_parse_bstring(source, &tokens[*index]);
}
else {
sentinel("Invalid token at char %d", tokens[*index].start);
}
(*index)++;
}
// Add property.
if(!importer->table->opened) {
check(sky_table_open(importer->table) == 0, "Unable to open table");
}
rc = sky_property_file_add_property(importer->table->property_file, property);
check(rc == 0, "Unable to add property: %s", bdata(property->name));
return 0;
error:
return -1;
}
// Opens a table and attaches it to the server.
//
// server - The server.
// name - The table name.
// path - The table path.
// ret - A pointer to where the table reference should be returned.
//
// Returns 0 if successful, otherwise returns -1.
int sky_server_open_table(sky_server *server, bstring name, bstring path,
sky_table **ret)
{
int rc;
sky_table *table = NULL;
assert(server != NULL);
assert(ret != NULL);
check(blength(path) > 0, "Table path required");
// Initialize return values.
*ret = NULL;
// Only open the table if it exists already.
if(sky_file_exists(path)) {
// Create the table.
table = sky_table_create(); check_mem(table);
table->name = bstrcpy(name); check_mem(table->name);
rc = sky_table_set_path(table, path);
check(rc == 0, "Unable to set table path");
// Open the table.
rc = sky_table_open(table);
check(rc == 0, "Unable to open table");
// Append the table to the list of open tables.
server->table_count++;
server->tables = realloc(server->tables, server->table_count * sizeof(*server->tables));
check_mem(server->tables);
server->tables[server->table_count-1] = table;
// Open servlets for the table's tablets.
rc = sky_server_create_servlets(server, table);
check(rc == 0, "Unable to create servlets for table");
// Return table.
*ret = table;
}
return 0;
error:
sky_table_free(table);
if(ret) *ret = NULL;
return -1;
}
int test_sky_get_property_message_process() {
loadtmp("tests/fixtures/get_property_message/1/table");
sky_server *server = sky_server_create(NULL);
sky_message_header *header = sky_message_header_create();
sky_table *table = sky_table_create();
table->path = bfromcstr("tmp");
sky_table_open(table);
FILE *input = fopen("tests/fixtures/get_property_message/1/input", "r");
FILE *output = fopen("tmp/output", "w");
int rc = sky_get_property_message_process(server, header, table, input, output);
mu_assert_int_equals(rc, 0);
mu_assert_file("tmp/output", "tests/fixtures/get_property_message/1/output");
sky_table_free(table);
sky_message_header_free(header);
sky_server_free(server);
return 0;
}
int sky_importer_process_action(sky_importer *importer, bstring source,
jsmntok_t *tokens, uint32_t *index)
{
int rc;
assert(importer != NULL);
assert(source != NULL);
assert(tokens != NULL);
assert(index != NULL);
jsmntok_t *action_token = &tokens[*index];
(*index)++;
// Create the action object.
sky_action *action = sky_action_create(); check_mem(action);
// Process over child tokens.
int32_t i;
for(i=0; i<(action_token->size/2); i++) {
jsmntok_t *token = &tokens[*index];
(*index)++;
if(sky_importer_tokstr_equal(source, token, "name")) {
action->name = sky_importer_token_parse_bstring(source, &tokens[*index]);
}
else {
sentinel("Invalid token at char %d", tokens[*index].start);
}
(*index)++;
}
// Add action.
if(!importer->table->opened) {
check(sky_table_open(importer->table) == 0, "Unable to open table");
}
rc = sky_action_file_add_action(importer->table->action_file, action);
check(rc == 0, "Unable to add action: %s", bdata(action->name));
return 0;
error:
return -1;
}
int test_sky_add_event_message_worker_map() {
importtmp("tests/fixtures/add_event_message/1/import.json");
sky_table *table = sky_table_create();
table->path = bfromcstr("tmp");
table->default_tablet_count = 1;
sky_table_open(table);
struct tagbstring XYZ_STR = bsStatic("xyz");
sky_add_event_message *message = sky_add_event_message_create();
message->event = sky_event_create(10, 1000L, 20);
message->event->data_count = 4;
message->event->data = calloc(message->event->data_count, sizeof(*message->event->data));
message->event->data[0] = sky_event_data_create_string(1, &XYZ_STR);
message->event->data[1] = sky_event_data_create_int(2, 200);
message->event->data[2] = sky_event_data_create_double(3, 100.2);
message->event->data[3] = sky_event_data_create_boolean(4, true);
sky_worker *worker = sky_worker_create();
worker->data = (void*)message;
void *null = NULL;
int rc = sky_add_event_message_worker_map(worker, table->tablets[0], &null);
mu_assert_int_equals(rc, 0);
void *data;
size_t data_length;
sky_tablet_get_path(table->tablets[0], 10, &data, &data_length);
mu_assert_int_equals(rc, 0);
mu_assert_mem(
data,
"\x03\xE8\x03\x00\x00\x00\x00\x00\x00\x14\x00\x15\x00\x00\x00\x01"
"\xA3\x78\x79\x7A\x02\xD1\x00\xC8\x03\xCB\x40\x59\x0C\xCC\xCC\xCC"
"\xCC\xCD\x04\xC3",
data_length
);
free(data);
sky_add_event_message_free(message);
sky_worker_free(worker);
sky_table_free(table);
return 0;
}
int test_sky_lua_initscript_with_table() {
int rc;
importtmp("tests/fixtures/sky_lua/0/data.json");
sky_table *table = sky_table_create();
table->path = bfromcstr("tmp");
sky_table_open(table);
sky_data_descriptor *descriptor = sky_data_descriptor_create();
sky_lua_event_0_t lua_event = {0, 0, 1, 20, 30};
struct tagbstring source = bsStatic(
"function aggregate(_event)\n"
" event = ffi.cast('sky_lua_event_t*', _event)\n"
" return event.x + event.y\n"
"end\n"
);
lua_State *L = NULL;
rc = sky_lua_initscript_with_table(&source, table, descriptor, &L);
mu_assert_int_equals(rc, 0);
mu_assert_int_equals(descriptor->timestamp_descriptor.ts_offset, 0);
mu_assert_int_equals(descriptor->timestamp_descriptor.timestamp_offset, 8);
mu_assert_int_equals(descriptor->action_descriptor.offset, 12);
mu_assert_int_equals(descriptor->property_zero_descriptor[1].offset, 0);
mu_assert_int_equals(descriptor->property_zero_descriptor[2].offset, (int)offsetof(sky_lua_event_0_t, x));
mu_assert_int_equals(descriptor->property_zero_descriptor[3].offset, (int)offsetof(sky_lua_event_0_t, y));
mu_assert_int_equals(descriptor->property_zero_descriptor[4].offset, 0);
// Call aggregate() function with the event pointer.
lua_getglobal(L, "aggregate");
lua_pushlightuserdata(L, &lua_event);
lua_call(L, 1, 1);
mu_assert_int_equals(rc, 0);
mu_assert_long_equals(lua_tointeger(L, -1), 50L);
sky_table_free(table);
return 0;
}
int test_sky_peach_message_process() {
importtmp("tests/fixtures/peach_message/1/import.json");
sky_table *table = sky_table_create();
table->path = bfromcstr("tmp");
sky_table_open(table);
// NOTE: The table contains two properties: foo (String) and this_is_a_really...(Int)
sky_peach_message *message = sky_peach_message_create();
message->query = bfromcstr(
"[Hashable(\"id\")]\n"
"[Serializable]\n"
"class Result {\n"
" public Int id;\n"
" public Int count;\n"
" public Int objectTotal;\n"
" public Int actionTotal;\n"
"}\n"
"Cursor cursor = path.events();\n"
"for each (Event event in cursor) {\n"
" String dynamic_prop2 = event.this_is_a_really_long_property_name_woohoo;\n"
" Result item = data.get(event.actionId);\n"
" item.count = item.count + 1;\n"
" item.objectTotal = item.objectTotal + event.object_prop;\n"
" item.actionTotal = item.actionTotal + event.action_prop;\n"
"}\n"
"return;"
);
FILE *output = fopen("tmp/output", "w");
mu_assert(sky_peach_message_process(message, table, output) == 0, "");
fclose(output);
mu_assert_file("tmp/output", "tests/fixtures/peach_message/1/output");
sky_peach_message_free(message);
sky_table_free(table);
return 0;
}
// Executes the benchmark to count the number of times an action occurred
// using the QIP language.
//
// options - A list of options to use.
void benchmark_count_with_qip(Options *options)
{
int rc;
uint32_t path_count = 0;
// Initialize the query.
qip_ast_node *type_ref, *var_decl;
uint32_t arg_count = 2;
qip_ast_node *args[arg_count];
// Path arg.
struct tagbstring path_str = bsStatic("path");
type_ref = qip_ast_type_ref_create_cstr("Path");
var_decl = qip_ast_var_decl_create(type_ref, &path_str, NULL);
args[0] = qip_ast_farg_create(var_decl);
// Map arg.
struct tagbstring data_str = bsStatic("data");
type_ref = qip_ast_type_ref_create_cstr("Map");
qip_ast_type_ref_add_subtype(type_ref, qip_ast_type_ref_create_cstr("Int"));
qip_ast_type_ref_add_subtype(type_ref, qip_ast_type_ref_create_cstr("Result"));
var_decl = qip_ast_var_decl_create(type_ref, &data_str, NULL);
args[1] = qip_ast_farg_create(var_decl);
// Initialize query.
struct tagbstring query = bsStatic(
"[Hashable(\"id\")]\n"
"class Result {\n"
" public Int id;\n"
" public Int count;\n"
"}\n"
"Cursor cursor = path.events();\n"
"for each (Event event in cursor) {\n"
" Result item = data.get(event.actionId);\n"
" item.count = item.count + 1;\n"
"}\n"
"return;"
);
struct tagbstring module_name = bsStatic("qip");
struct tagbstring core_class_path = bsStatic("lib/core");
struct tagbstring sky_class_path = bsStatic("lib/sky");
qip_compiler *compiler = qip_compiler_create();
qip_compiler_add_class_path(compiler, &core_class_path);
qip_compiler_add_class_path(compiler, &sky_class_path);
qip_module *module = qip_module_create(&module_name, compiler);
rc = qip_compiler_compile(compiler, module, &query, args, arg_count);
check(rc == 0, "Unable to compile");
if(module->error_count > 0) fprintf(stderr, "Parse error [line %d] %s\n", module->errors[0]->line_no, bdata(module->errors[0]->message));
check(module->error_count == 0, "Compile error");
qip_compiler_free(compiler);
// Retrieve pointer to function.
sky_qip_path_map_func process_path = NULL;
rc = qip_module_get_main_function(module, (void*)(&process_path));
check(rc == 0, "Unable to retrieve main function");
// Initialize table.
sky_table *table = sky_table_create(); check_mem(table);
rc = sky_table_set_path(table, options->path);
check(rc == 0, "Unable to set path on table");
// Open table
rc = sky_table_open(table);
check(rc == 0, "Unable to open table");
// Loop for desired number of iterations.
int i;
for(i=0; i<options->iterations; i++) {
sky_path_iterator iterator;
sky_path_iterator_init(&iterator);
// Attach data file.
rc = sky_path_iterator_set_data_file(&iterator, table->data_file);
check(rc == 0, "Unable to initialze path iterator");
// Initialize QIP args.
sky_qip_path *path = sky_qip_path_create();
qip_map *map = qip_map_create();
// Iterate over each path.
while(!iterator.eof) {
// Retrieve the path pointer.
rc = sky_path_iterator_get_ptr(&iterator, &path->path_ptr);
check(rc == 0, "Unable to retrieve the path iterator pointer");
// Execute query.
process_path(path, map);
// Move to next path.
rc = sky_path_iterator_next(&iterator);
check(rc == 0, "Unable to find next path");
path_count++;
}
// Clean up iteration.
qip_map_free(map);
}
// Clean up
rc = sky_table_close(table);
check(rc == 0, "Unable to close table");
sky_table_free(table);
// Show stats.
printf("Total paths processed: %d\n", path_count);
return;
error:
sky_table_free(table);
}
// Executes the benchmark over the database to compute step counts in order to
// generate a directed acyclic graph (DAG).
//
// options - A list of options to use.
void benchmark_dag(Options *options)
{
int rc;
sky_event *event = NULL;
uint32_t event_count = 0;
int32_t action_count = 100; // TODO: Retrieve action count from actions file.
// Initialize table.
sky_table *table = sky_table_create(); check_mem(table);
rc = sky_table_set_path(table, options->path);
check(rc == 0, "Unable to set path on table");
// Open table
rc = sky_table_open(table);
check(rc == 0, "Unable to open table");
// Loop for desired number of iterations.
int i;
for(i=0; i<options->iterations; i++) {
sky_path_iterator iterator;
sky_path_iterator_init(&iterator);
// Attach data file.
rc = sky_path_iterator_set_data_file(&iterator, table->data_file);
check(rc == 0, "Unable to initialze path iterator");
// Create a square matrix of structs.
Step *steps = calloc(action_count*action_count, sizeof(Step));
// Iterate over each path.
while(!iterator.eof) {
sky_action_id_t action_id, prev_action_id;
// Retrieve the path pointer.
void *path_ptr;
rc = sky_path_iterator_get_ptr(&iterator, &path_ptr);
check(rc == 0, "Unable to retrieve the path iterator pointer");
// Initialize the cursor.
sky_cursor cursor;
sky_cursor_init(&cursor);
rc = sky_cursor_set_path(&cursor, path_ptr);
check(rc == 0, "Unable to set cursor path");
// Increment total event count.
event_count++;
// Initialize the previous action.
rc = sky_cursor_get_action_id(&cursor, &prev_action_id);
check(rc == 0, "Unable to retrieve first action");
// Find next event.
rc = sky_cursor_next(&cursor);
check(rc == 0, "Unable to find next event");
// Loop over each event in the path.
while(!cursor.eof) {
// Increment total event count.
event_count++;
// Retrieve action.
rc = sky_cursor_get_action_id(&cursor, &action_id);
check(rc == 0, "Unable to retrieve first action");
// Aggregate step information.
int32_t index = ((prev_action_id-1)*action_count) + (action_id-1);
steps[index].count++;
// Assign current action as previous action.
prev_action_id = action_id;
// Find next event.
rc = sky_cursor_next(&cursor);
check(rc == 0, "Unable to find next event");
}
rc = sky_path_iterator_next(&iterator);
check(rc == 0, "Unable to find next path");
}
// Show DAG data.
//int x;
//int total=0;
//for(x=0; x<action_count*action_count; x++) {
// printf("%06d %d\n", x, steps[x].count);
// total += steps[x].count;
//}
//printf("total: %d\n", total);
}
// Clean up
rc = sky_table_close(table);
check(rc == 0, "Unable to close table");
sky_table_free(table);
// Show stats.
printf("Total events processed: %d\n", event_count);
return;
error:
sky_event_free(event);
sky_table_free(table);
}