Ejemplo n.º 1
0
// Creates a worker.
//
// id - The worker identifier.
//
// Returns a reference to the worker.
sky_worker *sky_worker_create()
{
    sky_worker *worker = calloc(1, sizeof(sky_worker)); check_mem(worker);
    worker->id = next_worker_id++;
    return worker;

error:
    sky_worker_free(worker);
    return NULL;
}
Ejemplo n.º 2
0
// Delegates processing of the message to a worker.
//
// server  - The server.
// header  - The message header.
// table   - The table the message is working against
// input   - The input file stream.
// output  - The output file stream.
//
// Returns 0 if successful, otherwise returns -1.
int sky_lua_aggregate_message_process(sky_server *server,
                                       sky_message_header *header,
                                       sky_table *table,
                                       FILE *input, FILE *output)
{
    int rc = 0;
    sky_lua_aggregate_message *message = NULL;
    assert(table != NULL);
    assert(header != NULL);
    assert(input != NULL);
    assert(output != NULL);
    
    // Create worker.
    sky_worker *worker = sky_worker_create(); check_mem(worker);
    worker->context = server->context;
    worker->map = sky_lua_aggregate_message_worker_map;
    worker->map_free = sky_lua_aggregate_message_worker_map_free;
    worker->reduce = sky_lua_aggregate_message_worker_reduce;
    worker->write = sky_lua_aggregate_message_worker_write;
    worker->free = sky_lua_aggregate_message_worker_free;
    worker->input = input;
    worker->output = output;

    // Attach servlets.
    rc = sky_server_get_table_servlets(server, table, &worker->servlets, &worker->servlet_count);
    check(rc == 0, "Unable to copy servlets to worker");

    // Create and parse message object.
    message = sky_lua_aggregate_message_create(); check_mem(message);
    message->results = bfromcstr("\x80"); check_mem(message->results);
    rc = sky_lua_aggregate_message_unpack(message, input);
    check(rc == 0, "Unable to unpack 'lua::aggregate' message");
    check(message->source != NULL, "Lua source required");

    // Compile Lua script.
    rc = sky_lua_initscript_with_table(message->source, table, NULL, &message->L);
    check(rc == 0, "Unable to initialize script");

    // Attach message to worker.
    worker->data = (sky_lua_aggregate_message*)message;
    
    // Start worker.
    rc = sky_worker_start(worker);
    check(rc == 0, "Unable to start worker");
    
    return 0;

error:
    sky_lua_aggregate_message_free(message);
    sky_worker_free(worker);
    return -1;
}
Ejemplo n.º 3
0
int test_sky_add_event_message_worker_write() {
    sky_add_event_message *message = sky_add_event_message_create();
    sky_worker *worker = sky_worker_create();
    worker->data = (void*)message;
    worker->output = fopen("tmp/output", "w");

    int rc = sky_add_event_message_worker_write(worker, worker->output);
    mu_assert_int_equals(rc, 0);
    sky_add_event_message_free(message);
    sky_worker_free(worker);

    mu_assert_file("tmp/output", "tests/fixtures/add_event_message/1/output");

    return 0;
}
Ejemplo n.º 4
0
// Delegates processing of the 'Next Actions' message to a worker.
//
// server  - The server.
// header  - The message header.
// table   - The table the message is working against
// input   - The input file stream.
// output  - The output file stream.
//
// Returns 0 if successful, otherwise returns -1.
int sky_lua_map_reduce_message_process(sky_server *server,
                                       sky_message_header *header,
                                       sky_table *table,
                                       FILE *input, FILE *output)
{
    int rc = 0;
    sky_lua_map_reduce_message *message = NULL;
    assert(table != NULL);
    assert(header != NULL);
    assert(input != NULL);
    assert(output != NULL);
    
    // Create worker.
    sky_worker *worker = sky_worker_create(); check_mem(worker);
    worker->context = server->context;
    worker->read = sky_lua_map_reduce_message_worker_read;
    worker->map = sky_lua_map_reduce_message_worker_map;
    worker->map_free = sky_lua_map_reduce_message_worker_map_free;
    worker->reduce = sky_lua_map_reduce_message_worker_reduce;
    worker->write = sky_lua_map_reduce_message_worker_write;
    worker->free = sky_lua_map_reduce_message_worker_free;
    worker->input = input;
    worker->output = output;
    
    // Attach servlets.
    rc = sky_server_get_table_servlets(server, table, &worker->servlets, &worker->servlet_count);
    check(rc == 0, "Unable to copy servlets to worker");

    // Create a message object.
    message = sky_lua_map_reduce_message_create(); check_mem(message);
    check_mem(message->results);

    // Attach message to worker.
    worker->data = (sky_lua_map_reduce_message*)message;
    
    // Start worker.
    rc = sky_worker_start(worker);
    check(rc == 0, "Unable to start worker");
    
    return 0;

error:
    sky_lua_map_reduce_message_free(message);
    sky_worker_free(worker);
    return -1;
}
Ejemplo n.º 5
0
int test_sky_add_event_message_worker_map() {
    importtmp("tests/fixtures/add_event_message/1/import.json");
    sky_table *table = sky_table_create();
    table->path = bfromcstr("tmp");
    table->default_tablet_count = 1;
    sky_table_open(table);

    struct tagbstring XYZ_STR = bsStatic("xyz");
    sky_add_event_message *message = sky_add_event_message_create();
    message->event = sky_event_create(10, 1000L, 20);
    message->event->data_count = 4;
    message->event->data = calloc(message->event->data_count, sizeof(*message->event->data));
    message->event->data[0] = sky_event_data_create_string(1, &XYZ_STR);
    message->event->data[1] = sky_event_data_create_int(2, 200);
    message->event->data[2] = sky_event_data_create_double(3, 100.2);
    message->event->data[3] = sky_event_data_create_boolean(4, true);
    sky_worker *worker = sky_worker_create();
    worker->data = (void*)message;

    void *null = NULL;
    int rc = sky_add_event_message_worker_map(worker, table->tablets[0], &null);
    mu_assert_int_equals(rc, 0);

    void *data;
    size_t data_length;
    sky_tablet_get_path(table->tablets[0], 10, &data, &data_length);
    mu_assert_int_equals(rc, 0);
    mu_assert_mem(
        data, 
        "\x03\xE8\x03\x00\x00\x00\x00\x00\x00\x14\x00\x15\x00\x00\x00\x01"
        "\xA3\x78\x79\x7A\x02\xD1\x00\xC8\x03\xCB\x40\x59\x0C\xCC\xCC\xCC"
        "\xCC\xCD\x04\xC3",
        data_length
    );

    free(data);
    sky_add_event_message_free(message);
    sky_worker_free(worker);
    sky_table_free(table);
    return 0;
}
Ejemplo n.º 6
0
// The worker thread function.
//
// worker - The worker.
//
// Returns 0 if successful.
void *sky_worker_run(void *_worker)
{
    int rc;
    uint32_t i;
    sky_worker *worker = (sky_worker*)_worker;
    sky_worklet *worklet = NULL;
    check(worker != NULL, "Worker required");
    
    // Start benchmark.
    struct timeval tv;
    gettimeofday(&tv, NULL);
    int64_t t0 = (tv.tv_sec*1000) + (tv.tv_usec/1000);

    // Read data from stream.
    if(worker->read != NULL) {
        rc = worker->read(worker, worker->input);
        check(rc == 0, "Worker unable to read from stream");
    }

    // Push a message to each servlet.
    for(i=0; i<worker->push_socket_count; i++) {
        worklet = sky_worklet_create(worker); check_mem(worklet);
        rc = sky_zmq_send_ptr(worker->push_sockets[i], &worklet);
        check(rc == 0, "Worker unable to send worklet");
    }
    
    // Read in one pull message for every push message sent.
    for(i=0; i<worker->push_socket_count; i++) {
        // Receive worker back from servlet.
        rc = sky_zmq_recv_ptr(worker->pull_socket, (void**)&worklet);
        check(rc == 0 && worklet != NULL, "Worker unable to receive worklet");
        
        // Reduce worklet.
        if(worker->reduce != NULL) {
            rc = worker->reduce(worker, worklet->data);
            check(rc == 0, "Worker unable to reduce");
        }

        // Free worklet.
        if(worker->map_free) worker->map_free(worklet->data);
        worklet->data = NULL;
        sky_worklet_free(worklet);
        worklet = NULL;
    }
    
    // Output data to stream.
    if(worker->write != NULL) {
        rc = worker->write(worker, worker->output);
        check(rc == 0, "Worker unable to write output");
    }
    
    // End benchmark.
    gettimeofday(&tv, NULL);
    int64_t t1 = (tv.tv_sec*1000) + (tv.tv_usec/1000);
    printf("[worker] t=%.3fs\n", ((float)(t1-t0))/1000);
    
    // Clean up worker.
    worker->free(worker);
    sky_worker_free(worker);
    
    return NULL;

error:
    sky_worker_free(worker);
    return NULL;
}