// Creates a worker.
//
// id - The worker identifier.
//
// Returns a reference to the worker.
sky_worker *sky_worker_create()
{
sky_worker *worker = calloc(1, sizeof(sky_worker)); check_mem(worker);
worker->id = next_worker_id++;
return worker;
error:
sky_worker_free(worker);
return NULL;
}
// Delegates processing of the message to a worker.
//
// server - The server.
// header - The message header.
// table - The table the message is working against
// input - The input file stream.
// output - The output file stream.
//
// Returns 0 if successful, otherwise returns -1.
int sky_lua_aggregate_message_process(sky_server *server,
sky_message_header *header,
sky_table *table,
FILE *input, FILE *output)
{
int rc = 0;
sky_lua_aggregate_message *message = NULL;
assert(table != NULL);
assert(header != NULL);
assert(input != NULL);
assert(output != NULL);
// Create worker.
sky_worker *worker = sky_worker_create(); check_mem(worker);
worker->context = server->context;
worker->map = sky_lua_aggregate_message_worker_map;
worker->map_free = sky_lua_aggregate_message_worker_map_free;
worker->reduce = sky_lua_aggregate_message_worker_reduce;
worker->write = sky_lua_aggregate_message_worker_write;
worker->free = sky_lua_aggregate_message_worker_free;
worker->input = input;
worker->output = output;
// Attach servlets.
rc = sky_server_get_table_servlets(server, table, &worker->servlets, &worker->servlet_count);
check(rc == 0, "Unable to copy servlets to worker");
// Create and parse message object.
message = sky_lua_aggregate_message_create(); check_mem(message);
message->results = bfromcstr("\x80"); check_mem(message->results);
rc = sky_lua_aggregate_message_unpack(message, input);
check(rc == 0, "Unable to unpack 'lua::aggregate' message");
check(message->source != NULL, "Lua source required");
// Compile Lua script.
rc = sky_lua_initscript_with_table(message->source, table, NULL, &message->L);
check(rc == 0, "Unable to initialize script");
// Attach message to worker.
worker->data = (sky_lua_aggregate_message*)message;
// Start worker.
rc = sky_worker_start(worker);
check(rc == 0, "Unable to start worker");
return 0;
error:
sky_lua_aggregate_message_free(message);
sky_worker_free(worker);
return -1;
}
int test_sky_add_event_message_worker_write() {
sky_add_event_message *message = sky_add_event_message_create();
sky_worker *worker = sky_worker_create();
worker->data = (void*)message;
worker->output = fopen("tmp/output", "w");
int rc = sky_add_event_message_worker_write(worker, worker->output);
mu_assert_int_equals(rc, 0);
sky_add_event_message_free(message);
sky_worker_free(worker);
mu_assert_file("tmp/output", "tests/fixtures/add_event_message/1/output");
return 0;
}
// Delegates processing of the 'Next Actions' message to a worker.
//
// server - The server.
// header - The message header.
// table - The table the message is working against
// input - The input file stream.
// output - The output file stream.
//
// Returns 0 if successful, otherwise returns -1.
int sky_lua_map_reduce_message_process(sky_server *server,
sky_message_header *header,
sky_table *table,
FILE *input, FILE *output)
{
int rc = 0;
sky_lua_map_reduce_message *message = NULL;
assert(table != NULL);
assert(header != NULL);
assert(input != NULL);
assert(output != NULL);
// Create worker.
sky_worker *worker = sky_worker_create(); check_mem(worker);
worker->context = server->context;
worker->read = sky_lua_map_reduce_message_worker_read;
worker->map = sky_lua_map_reduce_message_worker_map;
worker->map_free = sky_lua_map_reduce_message_worker_map_free;
worker->reduce = sky_lua_map_reduce_message_worker_reduce;
worker->write = sky_lua_map_reduce_message_worker_write;
worker->free = sky_lua_map_reduce_message_worker_free;
worker->input = input;
worker->output = output;
// Attach servlets.
rc = sky_server_get_table_servlets(server, table, &worker->servlets, &worker->servlet_count);
check(rc == 0, "Unable to copy servlets to worker");
// Create a message object.
message = sky_lua_map_reduce_message_create(); check_mem(message);
check_mem(message->results);
// Attach message to worker.
worker->data = (sky_lua_map_reduce_message*)message;
// Start worker.
rc = sky_worker_start(worker);
check(rc == 0, "Unable to start worker");
return 0;
error:
sky_lua_map_reduce_message_free(message);
sky_worker_free(worker);
return -1;
}
int test_sky_add_event_message_worker_map() {
importtmp("tests/fixtures/add_event_message/1/import.json");
sky_table *table = sky_table_create();
table->path = bfromcstr("tmp");
table->default_tablet_count = 1;
sky_table_open(table);
struct tagbstring XYZ_STR = bsStatic("xyz");
sky_add_event_message *message = sky_add_event_message_create();
message->event = sky_event_create(10, 1000L, 20);
message->event->data_count = 4;
message->event->data = calloc(message->event->data_count, sizeof(*message->event->data));
message->event->data[0] = sky_event_data_create_string(1, &XYZ_STR);
message->event->data[1] = sky_event_data_create_int(2, 200);
message->event->data[2] = sky_event_data_create_double(3, 100.2);
message->event->data[3] = sky_event_data_create_boolean(4, true);
sky_worker *worker = sky_worker_create();
worker->data = (void*)message;
void *null = NULL;
int rc = sky_add_event_message_worker_map(worker, table->tablets[0], &null);
mu_assert_int_equals(rc, 0);
void *data;
size_t data_length;
sky_tablet_get_path(table->tablets[0], 10, &data, &data_length);
mu_assert_int_equals(rc, 0);
mu_assert_mem(
data,
"\x03\xE8\x03\x00\x00\x00\x00\x00\x00\x14\x00\x15\x00\x00\x00\x01"
"\xA3\x78\x79\x7A\x02\xD1\x00\xC8\x03\xCB\x40\x59\x0C\xCC\xCC\xCC"
"\xCC\xCD\x04\xC3",
data_length
);
free(data);
sky_add_event_message_free(message);
sky_worker_free(worker);
sky_table_free(table);
return 0;
}
// The worker thread function.
//
// worker - The worker.
//
// Returns 0 if successful.
void *sky_worker_run(void *_worker)
{
int rc;
uint32_t i;
sky_worker *worker = (sky_worker*)_worker;
sky_worklet *worklet = NULL;
check(worker != NULL, "Worker required");
// Start benchmark.
struct timeval tv;
gettimeofday(&tv, NULL);
int64_t t0 = (tv.tv_sec*1000) + (tv.tv_usec/1000);
// Read data from stream.
if(worker->read != NULL) {
rc = worker->read(worker, worker->input);
check(rc == 0, "Worker unable to read from stream");
}
// Push a message to each servlet.
for(i=0; i<worker->push_socket_count; i++) {
worklet = sky_worklet_create(worker); check_mem(worklet);
rc = sky_zmq_send_ptr(worker->push_sockets[i], &worklet);
check(rc == 0, "Worker unable to send worklet");
}
// Read in one pull message for every push message sent.
for(i=0; i<worker->push_socket_count; i++) {
// Receive worker back from servlet.
rc = sky_zmq_recv_ptr(worker->pull_socket, (void**)&worklet);
check(rc == 0 && worklet != NULL, "Worker unable to receive worklet");
// Reduce worklet.
if(worker->reduce != NULL) {
rc = worker->reduce(worker, worklet->data);
check(rc == 0, "Worker unable to reduce");
}
// Free worklet.
if(worker->map_free) worker->map_free(worklet->data);
worklet->data = NULL;
sky_worklet_free(worklet);
worklet = NULL;
}
// Output data to stream.
if(worker->write != NULL) {
rc = worker->write(worker, worker->output);
check(rc == 0, "Worker unable to write output");
}
// End benchmark.
gettimeofday(&tv, NULL);
int64_t t1 = (tv.tv_sec*1000) + (tv.tv_usec/1000);
printf("[worker] t=%.3fs\n", ((float)(t1-t0))/1000);
// Clean up worker.
worker->free(worker);
sky_worker_free(worker);
return NULL;
error:
sky_worker_free(worker);
return NULL;
}
