// Maps tablet data to a next action summation data structure.
//
// worker - The worker.
// tablet - The tablet to work against.
// ret - A pointer to where the summation data structure should be returned.
//
// Returns 0 if successful, otherwise returns -1.
int sky_lua_aggregate_message_worker_map(sky_worker *worker,
sky_tablet *tablet,
void **ret)
{
int rc;
lua_State *L = NULL;
bstring msgpack_ret = NULL;
sky_data_descriptor *descriptor = NULL;
assert(worker != NULL);
assert(tablet != NULL);
assert(ret != NULL);
sky_lua_aggregate_message *message = (sky_lua_aggregate_message*)worker->data;
// Initialize the path iterator.
sky_path_iterator iterator;
sky_path_iterator_init(&iterator);
// Compile Lua script.
descriptor = sky_data_descriptor_create(); check_mem(descriptor);
rc = sky_lua_initscript_with_table(message->source, tablet->table, descriptor, &L);
check(rc == 0, "Unable to initialize script");
iterator.cursor.data_descriptor = descriptor;
iterator.cursor.data = calloc(1, descriptor->data_sz); check_mem(iterator.cursor.data);
// Assign the data file to iterate over.
rc = sky_path_iterator_set_tablet(&iterator, tablet);
check(rc == 0, "Unable to initialize path iterator");
// Execute function.
lua_getglobal(L, "sky_aggregate");
lua_pushlightuserdata(L, &iterator);
rc = lua_pcall(L, 1, 1, 0);
check(rc == 0, "Unable to execute Lua script: %s", lua_tostring(L, -1));
// Execute the script and return a msgpack variable.
rc = sky_lua_msgpack_pack(L, &msgpack_ret);
check(rc == 0, "Unable to execute Lua script");
// Close Lua.
lua_close(L);
// Return msgpack encoded response.
*ret = (void*)msgpack_ret;
free(iterator.cursor.data);
sky_data_descriptor_free(descriptor);
sky_path_iterator_uninit(&iterator);
return 0;
error:
*ret = NULL;
bdestroy(msgpack_ret);
if(L) lua_close(L);
free(iterator.cursor.data);
sky_data_descriptor_free(descriptor);
sky_path_iterator_uninit(&iterator);
return -1;
}
int test_sky_aggregate() {
importtmp("tests/fixtures/sky_lua/1/data.json");
sky_table *table = sky_table_create();
table->path = bfromcstr("tmp");
sky_table_open(table);
sky_data_descriptor *descriptor = sky_data_descriptor_create();
// Initialize the path iterator.
sky_path_iterator iterator;
sky_path_iterator_init(&iterator);
iterator.cursor.data_descriptor = descriptor;
sky_path_iterator_set_tablet(&iterator, table->tablets[0]);
struct tagbstring source = bsStatic(
"function aggregate(cursor, data)\n"
" data.path_count = (data.path_count or 0) + 1\n"
" while cursor:next() do\n"
" data.event_count = (data.event_count or 0) + 1\n"
" data.z = (data.z or 0) + cursor.event.x + cursor.event.y\n"
" end\n"
"end\n"
);
lua_State *L = NULL;
int rc = sky_lua_initscript_with_table(&source, table, descriptor, &L);
mu_assert_int_equals(rc, 0);
// Allocate data.
mu_assert_int_equals(descriptor->data_sz, 24);
iterator.cursor.data = calloc(1, descriptor->data_sz);
// Start benchmark.
struct timeval tv;
gettimeofday(&tv, NULL);
int64_t t0 = (tv.tv_sec*1000) + (tv.tv_usec/1000);
// Call sky_aggregate() function.
uint32_t i;
for(i=0; i<1; i++) {
//sky_path_iterator_set_tablet(&iterator, table->tablets[0]);
lua_getglobal(L, "sky_aggregate");
lua_pushlightuserdata(L, &iterator);
lua_call(L, 1, 0);
}
// End benchmark.
gettimeofday(&tv, NULL);
int64_t t1 = (tv.tv_sec*1000) + (tv.tv_usec/1000);
printf("[lua] t=%.3fs\n", ((float)(t1-t0))/1000);
sky_path_iterator_uninit(&iterator);
sky_table_free(table);
free(iterator.cursor.data);
return 0;
}
// Executes the benchmark over the database to compute step counts in order to
// generate a directed acyclic graph (DAG).
//
// options - A list of options to use.
void benchmark_dag(Options *options)
{
int rc;
sky_event *event = NULL;
uint32_t event_count = 0;
int32_t action_count = 100; // TODO: Retrieve action count from actions file.
// Initialize table.
sky_table *table = sky_table_create(); check_mem(table);
rc = sky_table_set_path(table, options->path);
check(rc == 0, "Unable to set path on table");
// Open table
rc = sky_table_open(table);
check(rc == 0, "Unable to open table");
// Loop for desired number of iterations.
int i;
for(i=0; i<options->iterations; i++) {
sky_path_iterator iterator;
sky_path_iterator_init(&iterator);
// Attach data file.
rc = sky_path_iterator_set_data_file(&iterator, table->data_file);
check(rc == 0, "Unable to initialze path iterator");
// Create a square matrix of structs.
Step *steps = calloc(action_count*action_count, sizeof(Step));
// Iterate over each path.
while(!iterator.eof) {
sky_action_id_t action_id, prev_action_id;
// Retrieve the path pointer.
void *path_ptr;
rc = sky_path_iterator_get_ptr(&iterator, &path_ptr);
check(rc == 0, "Unable to retrieve the path iterator pointer");
// Initialize the cursor.
sky_cursor cursor;
sky_cursor_init(&cursor);
rc = sky_cursor_set_path(&cursor, path_ptr);
check(rc == 0, "Unable to set cursor path");
// Increment total event count.
event_count++;
// Initialize the previous action.
rc = sky_cursor_get_action_id(&cursor, &prev_action_id);
check(rc == 0, "Unable to retrieve first action");
// Find next event.
rc = sky_cursor_next(&cursor);
check(rc == 0, "Unable to find next event");
// Loop over each event in the path.
while(!cursor.eof) {
// Increment total event count.
event_count++;
// Retrieve action.
rc = sky_cursor_get_action_id(&cursor, &action_id);
check(rc == 0, "Unable to retrieve first action");
// Aggregate step information.
int32_t index = ((prev_action_id-1)*action_count) + (action_id-1);
steps[index].count++;
// Assign current action as previous action.
prev_action_id = action_id;
// Find next event.
rc = sky_cursor_next(&cursor);
check(rc == 0, "Unable to find next event");
}
rc = sky_path_iterator_next(&iterator);
check(rc == 0, "Unable to find next path");
}
// Show DAG data.
//int x;
//int total=0;
//for(x=0; x<action_count*action_count; x++) {
// printf("%06d %d\n", x, steps[x].count);
// total += steps[x].count;
//}
//printf("total: %d\n", total);
}
// Clean up
rc = sky_table_close(table);
check(rc == 0, "Unable to close table");
sky_table_free(table);
// Show stats.
printf("Total events processed: %d\n", event_count);
return;
error:
sky_event_free(event);
sky_table_free(table);
}
// Executes the benchmark to count the number of times an action occurred
// using the QIP language.
//
// options - A list of options to use.
void benchmark_count_with_qip(Options *options)
{
int rc;
uint32_t path_count = 0;
// Initialize the query.
qip_ast_node *type_ref, *var_decl;
uint32_t arg_count = 2;
qip_ast_node *args[arg_count];
// Path arg.
struct tagbstring path_str = bsStatic("path");
type_ref = qip_ast_type_ref_create_cstr("Path");
var_decl = qip_ast_var_decl_create(type_ref, &path_str, NULL);
args[0] = qip_ast_farg_create(var_decl);
// Map arg.
struct tagbstring data_str = bsStatic("data");
type_ref = qip_ast_type_ref_create_cstr("Map");
qip_ast_type_ref_add_subtype(type_ref, qip_ast_type_ref_create_cstr("Int"));
qip_ast_type_ref_add_subtype(type_ref, qip_ast_type_ref_create_cstr("Result"));
var_decl = qip_ast_var_decl_create(type_ref, &data_str, NULL);
args[1] = qip_ast_farg_create(var_decl);
// Initialize query.
struct tagbstring query = bsStatic(
"[Hashable(\"id\")]\n"
"class Result {\n"
" public Int id;\n"
" public Int count;\n"
"}\n"
"Cursor cursor = path.events();\n"
"for each (Event event in cursor) {\n"
" Result item = data.get(event.actionId);\n"
" item.count = item.count + 1;\n"
"}\n"
"return;"
);
struct tagbstring module_name = bsStatic("qip");
struct tagbstring core_class_path = bsStatic("lib/core");
struct tagbstring sky_class_path = bsStatic("lib/sky");
qip_compiler *compiler = qip_compiler_create();
qip_compiler_add_class_path(compiler, &core_class_path);
qip_compiler_add_class_path(compiler, &sky_class_path);
qip_module *module = qip_module_create(&module_name, compiler);
rc = qip_compiler_compile(compiler, module, &query, args, arg_count);
check(rc == 0, "Unable to compile");
if(module->error_count > 0) fprintf(stderr, "Parse error [line %d] %s\n", module->errors[0]->line_no, bdata(module->errors[0]->message));
check(module->error_count == 0, "Compile error");
qip_compiler_free(compiler);
// Retrieve pointer to function.
sky_qip_path_map_func process_path = NULL;
rc = qip_module_get_main_function(module, (void*)(&process_path));
check(rc == 0, "Unable to retrieve main function");
// Initialize table.
sky_table *table = sky_table_create(); check_mem(table);
rc = sky_table_set_path(table, options->path);
check(rc == 0, "Unable to set path on table");
// Open table
rc = sky_table_open(table);
check(rc == 0, "Unable to open table");
// Loop for desired number of iterations.
int i;
for(i=0; i<options->iterations; i++) {
sky_path_iterator iterator;
sky_path_iterator_init(&iterator);
// Attach data file.
rc = sky_path_iterator_set_data_file(&iterator, table->data_file);
check(rc == 0, "Unable to initialze path iterator");
// Initialize QIP args.
sky_qip_path *path = sky_qip_path_create();
qip_map *map = qip_map_create();
// Iterate over each path.
while(!iterator.eof) {
// Retrieve the path pointer.
rc = sky_path_iterator_get_ptr(&iterator, &path->path_ptr);
check(rc == 0, "Unable to retrieve the path iterator pointer");
// Execute query.
process_path(path, map);
// Move to next path.
rc = sky_path_iterator_next(&iterator);
check(rc == 0, "Unable to find next path");
path_count++;
}
// Clean up iteration.
qip_map_free(map);
}
// Clean up
rc = sky_table_close(table);
check(rc == 0, "Unable to close table");
sky_table_free(table);
// Show stats.
printf("Total paths processed: %d\n", path_count);
return;
error:
sky_table_free(table);
}
// Runs a PEACH query against a table.
//
// message - The message.
// table - The table to run the query against.
// output - The output stream to write to.
//
// Returns 0 if successful, otherwise returns -1.
int sky_peach_message_process(sky_peach_message *message, sky_table *table,
FILE *output)
{
int rc;
check(message != NULL, "Message required");
check(table != NULL, "Table required");
check(output != NULL, "Output stream required");
// Compile.
sky_qip_module *module = sky_qip_module_create(); check_mem(module);
module->table = table;
rc = sky_qip_module_compile(module, message->query);
check(rc == 0, "Unable to compile query");
sky_qip_path_map_func main_function = (sky_qip_path_map_func)module->main_function;
// Initialize the path iterator.
sky_path_iterator iterator;
sky_path_iterator_init(&iterator);
rc = sky_path_iterator_set_data_file(&iterator, table->data_file);
check(rc == 0, "Unable to initialze path iterator");
// Initialize QIP args.
sky_qip_path *path = sky_qip_path_create();
qip_map *map = qip_map_create();
// Iterate over each path.
uint32_t path_count = 0;
while(!iterator.eof) {
// Retrieve the path pointer.
rc = sky_path_iterator_get_ptr(&iterator, &path->path_ptr);
check(rc == 0, "Unable to retrieve the path iterator pointer");
// Execute query.
main_function(path, map);
// Move to next path.
rc = sky_path_iterator_next(&iterator);
check(rc == 0, "Unable to find next path");
path_count++;
}
//debug("Paths processed: %d", path_count);
// Retrieve Result serialization function.
struct tagbstring result_str = bsStatic("Result");
struct tagbstring serialize_str = bsStatic("serialize");
sky_qip_result_serialize_func result_serialize = NULL;
rc = qip_module_get_class_method(module->_qip_module, &result_str, &serialize_str, (void*)(&result_serialize));
check(rc == 0 && result_serialize != NULL, "Unable to find serialize() method on class 'Result'");
// Serialize.
qip_serializer *serializer = qip_serializer_create();
qip_serializer_pack_map(module->_qip_module, serializer, map->count);
int64_t i;
for(i=0; i<map->count; i++) {
result_serialize(map->elements[i], serializer);
}
// Send response to output stream.
rc = fwrite(serializer->data, serializer->length, 1, output);
check(rc == 1, "Unable to write serialized data to stream");
qip_map_free(map);
sky_qip_module_free(module);
return 0;
error:
sky_qip_module_free(module);
return -1;
}
