/* Given a series of tasks, serialize into a series of array objects
* using messagepack and return pointer to the buffer containing the
* serialized data
*/
msgpack_sbuffer *
oolong_serialize(msgpack_sbuffer *sbuf, oolong_task **tasks, int size) {
int i;
oolong_task *task;
msgpack_packer *pk = msgpack_packer_new(sbuf, msgpack_sbuffer_write);
for(i = 0; i < size; i++) {
task = tasks[i];
/* Begin array packing */
msgpack_pack_array(pk, 3);
/* Pack description */
msgpack_pack_raw(pk, sizeof(task->description) + 1);
msgpack_pack_raw_body(pk, task->description, sizeof(task->description) + 1);
/* Pack completed */
msgpack_pack_int(pk, task->completed);
/* Pack due */
msgpack_pack_int(pk, task->due);
}
msgpack_packer_free(pk);
return sbuf;
}
void test()
{
uint64_t test_u64 = 0xFFF0000000000001LL;
size_t size = 10000000;
msgpack_sbuffer buf;
msgpack_sbuffer_init(&buf);
msgpack_packer * pk = msgpack_packer_new(&buf, msgpack_sbuffer_write);
msgpack_pack_array(pk, size);
{
int idx = 0;
for (; idx < size; ++idx)
msgpack_pack_uint64(pk, test_u64);
}
msgpack_packer_free(pk);
size_t upk_pos = 0;
msgpack_unpacked msg;
msgpack_unpacked_init(&msg);
while (msgpack_unpack_next(&msg, buf.data, buf.size, &upk_pos)) {
}
msgpack_sbuffer_destroy(&buf);
}
TEST(fixint, size)
{
msgpack_sbuffer* sbuf = msgpack_sbuffer_new();
msgpack_packer* pk = msgpack_packer_new(sbuf, msgpack_sbuffer_write);
size_t sum = 0;
EXPECT_EQ(0, msgpack_pack_fix_int8(pk, 0));
EXPECT_EQ(sum+=2, sbuf->size);
EXPECT_EQ(0, msgpack_pack_fix_int16(pk, 0));
EXPECT_EQ(sum+=3, sbuf->size);
EXPECT_EQ(0, msgpack_pack_fix_int32(pk, 0));
EXPECT_EQ(sum+=5, sbuf->size);
EXPECT_EQ(0, msgpack_pack_fix_int64(pk, 0));
EXPECT_EQ(sum+=9, sbuf->size);
EXPECT_EQ(0, msgpack_pack_fix_uint8(pk, 0));
EXPECT_EQ(sum+=2, sbuf->size);
EXPECT_EQ(0, msgpack_pack_fix_uint16(pk, 0));
EXPECT_EQ(sum+=3, sbuf->size);
EXPECT_EQ(0, msgpack_pack_fix_uint32(pk, 0));
EXPECT_EQ(sum+=5, sbuf->size);
EXPECT_EQ(0, msgpack_pack_fix_uint64(pk, 0));
EXPECT_EQ(sum+=9, sbuf->size);
msgpack_sbuffer_free(sbuf);
msgpack_packer_free(pk);
}
int main(void) {
/* creates buffer and serializer instance. */
msgpack_sbuffer* buffer = msgpack_sbuffer_new();
msgpack_packer* pk = msgpack_packer_new(buffer, msgpack_sbuffer_write);
/* serializes ["Hello", "MessagePack"]. */
msgpack_pack_array(pk, 2);
msgpack_pack_raw(pk, 5);
msgpack_pack_raw_body(pk, "Hello", 5);
msgpack_pack_raw(pk, 11);
msgpack_pack_raw_body(pk, "MessagePack", 11);
/* deserializes it. */
msgpack_unpacked msg;
msgpack_unpacked_init(&msg);
bool success = msgpack_unpack_next(&msg, buffer->data, buffer->size, NULL);
/* prints the deserialized object. */
msgpack_object obj = msg.data;
msgpack_object_print(stdout, obj); /*=> ["Hello", "MessagePack"] */
puts("");
// msgpack_pack_object/pk
/* cleaning */
msgpack_sbuffer_free(buffer);
msgpack_packer_free(pk);
}
static void msgpack_flushfree(CTX ctx, kpackAPI_t *pk)
{
msgpack_sbuffer *sbuffer = pk->sbuffer;
kbytes_t t = {{sbuffer->data}, sbuffer->size};
knh_OutputStream_write(ctx, pk->w, t);
knh_OutputStream_flush(ctx, pk->w); /* TODO need flush? */
msgpack_sbuffer_free(pk->sbuffer);
msgpack_packer_free(pk->pk);
}
void
yakyak_destroy(struct yakyak *yy)
{
msgpack_packer_free(yy->pk);
msgpack_sbuffer_free(yy->buf);
yy->pk = NULL;
yy->buf = NULL;
}
void monitor_send_queue(void)
{
if(draw_pk) {
monitor_pack_draw_queue();
monitor_method_send(draw_sbuf);
msgpack_packer_free(draw_pk);
draw_pk = NULL;
}
}
void monitor_disconnect(void)
{
msgpack_packer *pk;
pk = monitor_method_new("DISCONNECT", sbuf);
msgpack_pack_nil(pk);
monitor_method_send(sbuf);
msgpack_packer_free(pk);
}
TEST(pack, num)
{
msgpack_sbuffer* sbuf = msgpack_sbuffer_new();
msgpack_packer* pk = msgpack_packer_new(sbuf, msgpack_sbuffer_write);
EXPECT_EQ(0, msgpack_pack_int(pk, 1));
msgpack_sbuffer_free(sbuf);
msgpack_packer_free(pk);
}
void
json_string_to_msgpack(const char *json_str, msgpack_sbuffer *buf)
{
PackState state;
JsonLexContext *lex = makeJsonLexContext(cstring_to_text(json_str), true);
JsonSemAction *sem;
msgpack_packer *pk;
JsonContainer top_level;
JsonValue top_level_value;
/* initialize packer */
pk = msgpack_packer_new(buf, msgpack_sbuffer_write);
/* initialize top level object */
top_level_value = palloc(sizeof(JsonValueData));
top_level = palloc(sizeof(JsonContainerData));
top_level->type = JSON_TOP_LEVEL;
top_level->parent = NULL;
top_level->via.top_level.value = top_level_value;
/* initialize state object */
state = palloc(sizeof(PackStateData));
state->lex = lex;
state->current_container = top_level;
state->pk = pk;
state->buf = buf;
/* initialize sem action */
sem = palloc(sizeof(JsonSemAction));
sem->semstate = (void *) state;
sem->object_start = sem_object_start;
sem->object_end = sem_object_end;
sem->array_start = sem_array_start;
sem->array_end = sem_array_end;
sem->object_field_start = sem_object_field_start;
sem->object_field_end = NULL;
sem->array_element_start = sem_array_element_start;
sem->array_element_end = NULL;
sem->scalar = sem_scalar;
/* run parser */
pg_parse_json(lex, sem);
/* pack top level value */
pack_value(pk, top_level_value);
/* destroy packer */
msgpack_packer_free(pk);
/* destroy top level object */
destroy_container(top_level);
}
static void queue_cpu_sample_for_sending(rbkit_cpu_sample *sample) {
msgpack_sbuffer* buffer = msgpack_sbuffer_new();
msgpack_packer* packer = msgpack_packer_new(buffer, msgpack_sbuffer_write);
rbkit_cpu_sample_event *event = new_rbkit_cpu_sample_event(sample);
pack_event((rbkit_event_header *)event, packer);
free(event);
send_message(buffer);
msgpack_sbuffer_free(buffer);
msgpack_packer_free(packer);
}
static zmsg_t *create_call(const char *actionid, const char *module, const char *function, const char *data)
{
zmsg_t *msg = zmsg_new();
msgpack_sbuffer* pIntBuffer = msgpack_sbuffer_new();
msgpack_packer* pk = msgpack_packer_new( pIntBuffer, msgpack_sbuffer_write );
if (data && actionid && module)
{
msgpack_pack_map( pk, 3 );
{
m_pack_raw(pk, "command");
m_pack_raw(pk, "call");
m_pack_raw(pk, "body");
msgpack_pack_map( pk, 5 );
{
m_pack_raw(pk, "module");
m_pack_raw(pk, module);
m_pack_raw(pk, "function");
m_pack_raw(pk, function);
m_pack_raw(pk, "version");
msgpack_pack_uint64(pk, 1);
m_pack_raw(pk, "parameters");
msgpack_pack_map( pk, 1 );
{
m_pack_raw(pk, function);
m_pack_raw(pk, data);
}
m_pack_raw(pk, "seqno");
msgpack_pack_uint64(pk, 0);
}
m_pack_raw(pk, "actionid");
m_pack_raw(pk, actionid);
}
} else {
return NULL;
}
zmsg_pushmem(msg, pIntBuffer->data, pIntBuffer->size);
msgpack_sbuffer_free( pIntBuffer );
msgpack_packer_free( pk );
return msg;
}
static VALUE send_handshake_response(VALUE self) {
msgpack_sbuffer *buffer = msgpack_sbuffer_new();
msgpack_packer *packer = msgpack_packer_new(buffer, msgpack_sbuffer_write);
rbkit_hash_event *event = new_rbkit_hash_event(handshake, rbkit_status_as_hash(self));
pack_event((rbkit_event_header *)event, packer);
free(event);
if(buffer && buffer->size > 0)
respond_with_message(buffer->data, buffer->size);
msgpack_sbuffer_free(buffer);
msgpack_packer_free(packer);
return Qnil;
}
void publish_output(process_t *process, struct proc_data *procdata,
const char *ioname, const char *data, ssize_t length) {
/* TODO(sissel): move this to a separate file for 'event' streaming */
zmq_msg_t event;
int rc;
size_t msgsize;
program_t *program = pn_proc_program(process);
procnanny_t *pn = program->data;
fprintf(stdout, "%s[%d]: (%d bytes) %.*s\n", pn_proc_program(process)->name,
pn_proc_instance(process), length, length, data);
/* Fields:
* - data (the string read)
* - program name
* - process instance
* - stdout or stderr
*/
msgpack_sbuffer *buffer = msgpack_sbuffer_new();
msgpack_packer *output_msg = msgpack_packer_new(buffer, msgpack_sbuffer_write);
msgpack_pack_map(output_msg, 5);
/* "event" => "data" */
msgpack_pack_string(output_msg, "event", -1);
msgpack_pack_string(output_msg, "data", -1);
msgpack_pack_string(output_msg, "program", -1);
msgpack_pack_string(output_msg, program->name, program->name_len);
msgpack_pack_string(output_msg, "instance", -1);
msgpack_pack_int(output_msg, process->instance);
msgpack_pack_string(output_msg, "source", -1);
msgpack_pack_string(output_msg, ioname, -1);
msgpack_pack_string(output_msg, "data", -1);
msgpack_pack_string(output_msg, data, length);
zmq_msg_init_data(&event, buffer->data, buffer->size, free_msgpack_buffer, buffer);
zmq_send(pn->eventsocket, &event, 0);
zmq_msg_close(&event);
msgpack_packer_free(output_msg);
} /* publish_output */
static VALUE stop_stat_server() {
if (logger->enabled == Qtrue)
stop_stat_tracing();
// Destroy the list which aggregates messages
message_list_destroy();
// Clear object_table which holds object allocation info
st_foreach(logger->object_table, free_values_i, 0);
st_clear(logger->object_table);
st_foreach(logger->str_table, free_keys_i, 0);
st_clear(logger->str_table);
msgpack_sbuffer_free(logger->sbuf);
msgpack_packer_free(logger->msgpacker);
zmq_close(zmq_publisher);
zmq_close(zmq_response_socket);
zmq_ctx_destroy(zmq_context);
free(logger);
logger = 0;
return Qnil;
}
static VALUE send_hash_as_event(int argc, VALUE *argv, VALUE self) {
VALUE hash_object;
VALUE event_type;
msgpack_sbuffer *buffer;
msgpack_packer *packer;
rbkit_hash_event *event;
rb_scan_args(argc, argv, "20", &hash_object, &event_type);
buffer = msgpack_sbuffer_new();
packer = msgpack_packer_new(buffer, msgpack_sbuffer_write);
event = new_rbkit_hash_event(FIX2INT(event_type), hash_object);
pack_event((rbkit_event_header *)event, packer);
free(event);
send_message(buffer);
msgpack_sbuffer_free(buffer);
msgpack_packer_free(packer);
return Qnil;
}
static char *
pack_data(int *size)
{
msgpack_sbuffer *buffer = NULL;
msgpack_packer *packer = NULL;
char *data;
buffer = msgpack_sbuffer_new();
msgpack_sbuffer_init(buffer);
packer = msgpack_packer_new((void *)buffer, msgpack_sbuffer_write);
pack_node(packer, &node);
data = (char *)malloc(buffer->size + 1);
memcpy(data, &buffer->data[0], buffer->size);
data[buffer->size] = '\0';
*size = buffer->size;
msgpack_packer_free(packer);
msgpack_sbuffer_free(buffer);
return data;
}
TEST(pack, insufficient)
{
msgpack_sbuffer* sbuf = msgpack_sbuffer_new();
msgpack_packer* pk = msgpack_packer_new(sbuf, msgpack_sbuffer_write);
EXPECT_EQ(0, msgpack_pack_int(pk, 255)); // uint8 (2bytes)
msgpack_unpack_return success;
size_t offset = 0;
msgpack_unpacked msg;
msgpack_unpacked_init(&msg);
success = msgpack_unpack_next(&msg, sbuf->data, 1, &offset);
EXPECT_EQ(MSGPACK_UNPACK_CONTINUE, success);
EXPECT_EQ(0u, offset);
msgpack_unpacked_destroy(&msg);
msgpack_sbuffer_free(sbuf);
msgpack_packer_free(pk);
}
void
save_data(char *filename, save_data_function pack_function)
{
msgpack_sbuffer *buffer = NULL;
msgpack_packer *packer = NULL;
char *save_file_name;
FILE *save_file;
if ((save_file_name = get_save_file_path(filename)) == NULL) {
red_log(REDD_WARNING, "get_save_file_path returned NULL");
return;
}
buffer = msgpack_sbuffer_new();
msgpack_sbuffer_init(buffer);
packer = msgpack_packer_new((void *)buffer, msgpack_sbuffer_write);
msgpack_pack_map(packer, 1);
pack_function(packer);
if ((save_file = fopen(save_file_name, "w")) != NULL) {
size_t written;
written = fwrite((void *)&buffer->data[0], 1, buffer->size, save_file);
fclose(save_file);
if (written != buffer->size) {
red_log(REDD_WARNING, "Failed to write to %s", save_file_name);
}
} else {
red_log(REDD_WARNING, "Failed to open %s", save_file_name);
}
free(save_file_name);
save_file_name = NULL;
msgpack_packer_free(packer);
packer = NULL;
msgpack_sbuffer_free(buffer);
buffer = NULL;
}
static VALUE send_objectspace_dump(VALUE self) {
msgpack_sbuffer* buffer = msgpack_sbuffer_new();
msgpack_packer* pk = msgpack_packer_new(buffer, msgpack_sbuffer_write);
rbkit_object_dump * dump = get_object_dump(logger->object_table);
rbkit_object_space_dump_event *event = new_rbkit_object_space_dump_event(dump);
// Object space dump can span across messages.
// So we keep creating and queueing the messages
// until we've packed all objects.
while(event->packed_objects < event->object_count) {
pack_event((rbkit_event_header *)event, pk);
send_message(buffer);
}
free(event->current_page);
free(event);
free(dump);
msgpack_sbuffer_free(buffer);
msgpack_packer_free(pk);
return Qnil;
}
TEST(unpack, sequence)
{
msgpack_sbuffer* sbuf = msgpack_sbuffer_new();
msgpack_packer* pk = msgpack_packer_new(sbuf, msgpack_sbuffer_write);
EXPECT_EQ(0, msgpack_pack_int(pk, 1));
EXPECT_EQ(0, msgpack_pack_int(pk, 2));
EXPECT_EQ(0, msgpack_pack_int(pk, 3));
msgpack_packer_free(pk);
msgpack_unpack_return success;
size_t offset = 0;
msgpack_unpacked msg;
msgpack_unpacked_init(&msg);
success = msgpack_unpack_next(&msg, sbuf->data, sbuf->size, &offset);
EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, success);
EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, msg.data.type);
EXPECT_EQ(1u, msg.data.via.u64);
success = msgpack_unpack_next(&msg, sbuf->data, sbuf->size, &offset);
EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, success);
EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, msg.data.type);
EXPECT_EQ(2u, msg.data.via.u64);
success = msgpack_unpack_next(&msg, sbuf->data, sbuf->size, &offset);
EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, success);
EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, msg.data.type);
EXPECT_EQ(3u, msg.data.via.u64);
success = msgpack_unpack_next(&msg, sbuf->data, sbuf->size, &offset);
EXPECT_EQ(MSGPACK_UNPACK_CONTINUE, success);
msgpack_sbuffer_free(sbuf);
msgpack_unpacked_destroy(&msg);
}
int main(void)
{
msgpack_sbuffer* buffer = msgpack_sbuffer_new();
msgpack_packer* pk = msgpack_packer_new(buffer, msgpack_sbuffer_write);
msgpack_pack_array(pk, 2);
msgpack_pack_raw(pk, 5);
msgpack_pack_raw_body(pk, "hello", 5);
msgpack_pack_raw(pk, 11);
msgpack_pack_raw_body(pk, "messagepack", 11);
msgpack_unpacked msg;
msgpack_unpacked_init(&msg);
bool success = msgpack_unpack_next(&msg, buffer->data, buffer->size, NULL);
msgpack_object obj = msg.data;
msgpack_object_print(stdout, obj);
printf("\n");
msgpack_sbuffer_free(buffer);
msgpack_packer_free(pk);
}
static void *client_thread(void *arg) {
ssize_t nwritten;
pthread_detach(pthread_self());
int client_fd = (int) (long) arg;
int yes = 1, tcp_keepalive_probes = 3, tcp_keepalive_time = 5, tcp_keepalive_intvl = 2;
int nodelay = 1;
if (setsockopt(client_fd, SOL_SOCKET, SO_KEEPALIVE, &yes, sizeof(yes)) == -1) {
log_error("setsockopt failed: %s", strerror(errno));
close(client_fd);
return NULL;
}
if (setsockopt(client_fd, SOL_TCP, TCP_KEEPCNT, &tcp_keepalive_probes, sizeof(tcp_keepalive_probes)) == -1) {
log_error("setsockopt failed: %s", strerror(errno));
close(client_fd);
return NULL;
}
if (setsockopt(client_fd, SOL_TCP, TCP_KEEPIDLE, &tcp_keepalive_time, sizeof(tcp_keepalive_time)) == -1) {
log_error("setsockopt failed: %s", strerror(errno));
close(client_fd);
return NULL;
}
if (setsockopt(client_fd, SOL_TCP, TCP_KEEPINTVL, &tcp_keepalive_intvl, sizeof(tcp_keepalive_intvl)) == -1) {
log_error("setsockopt failed: %s", strerror(errno));
close(client_fd);
return NULL;
}
if (setsockopt(client_fd, IPPROTO_TCP, TCP_NODELAY, &nodelay, sizeof(nodelay)) == -1) {
log_error("setsockopt failed: %s", strerror(errno));
close(client_fd);
return NULL;
}
size_t request_size = 8192;
struct request *request_back = NULL;
struct request *request = (struct request *) malloc(request_size);
if (!request) {
log_error("malloc failed: %s", strerror(errno));
close(client_fd);
return NULL;
}
msgpack_unpacked msg;
msgpack_unpacked_init(&msg);
bool enable_gzip = false;
while (1) {
struct response_header response_header;
ssize_t recvsize = readn(client_fd, request, sizeof(struct request));
if (!recvsize) {
log_warn("peer closed connection");
break;
}
if (recvsize < (ssize_t) sizeof(struct request)) {
log_warn("error while receiving header, received %zd", recvsize);
break;
}
if (request->payload_size > 256 * 1024 * 1024) {
log_warn("payload size %"PRIu32" too large", request->payload_size);
break;
}
if (sizeof(struct request) + request->payload_size > request_size) {
request_size = sizeof(struct request) + request->payload_size;
request_back = request;
request = realloc(request, request_size);
if (!request) {
log_error("realloc failed: %s", strerror(errno));
free(request_back);
break;
}
}
recvsize = readn(client_fd, request->payload, request->payload_size);
if (!recvsize) {
log_warn("peer closed connection");
break;
}
if (recvsize < request->payload_size) {
log_warn("error while receiving payload, received %zd (should be %"PRIu32")", recvsize, request->payload_size);
break;
}
if (request->flags & REQUEST_FLAG_GZIP) {
enable_gzip = true;
uLongf destLen = 16;
Bytef *dest = malloc(destLen);
if (!dest) {
log_error("malloc failed: %s", strerror(errno));
break;
}
int ret;
keep_malloc:
ret = uncompress(dest, &destLen, (Bytef *) request->payload, request->payload_size);
if (ret == Z_BUF_ERROR) {
destLen = destLen * 2;
free(dest);
dest = malloc(destLen);
if (!dest) {
log_error("malloc failed: %s", strerror(errno));
break;
}
goto keep_malloc;
}
if (ret != Z_OK) {
free(dest);
break;
}
request->flags &= ~REQUEST_FLAG_GZIP;
if (sizeof(struct request) + destLen > request_size) {
request_size = sizeof(struct request) + destLen;
request_back = request;
request = realloc(request, request_size);
if (!request) {
log_error("realloc failed: %s", strerror(errno));
free(request_back);
free(dest);
break;
}
}
memcpy(request->payload, dest, destLen);
request->payload_size = destLen;
free(dest);
}
bool success = msgpack_unpack_next(&msg, request->payload, request->payload_size, NULL);
if (!success) {
log_warn("error while parsing payload");
break;
}
msgpack_sbuffer* buffer = msgpack_sbuffer_new();
msgpack_packer* pk = msgpack_packer_new(buffer, msgpack_sbuffer_write);
response_header.seq = request->seq;
__sync_add_and_fetch(&pending_commands, 1);
if (server->terminated) {
__sync_add_and_fetch(&pending_commands, -1);
break;
}
if (server->redirection) {
response_header.status = RESPONSE_STATUS_REDIRECTION;
msgpack_pack_map(pk, 2);
mp_pack_string(pk, "host");
mp_pack_string(pk, server->redirection->host);
mp_pack_string(pk, "port");
msgpack_pack_uint16(pk, server->redirection->port);
}
else {
response_header.status = execute_command(request, request->command, msg.data, pk);
}
__sync_add_and_fetch(&pending_commands, -1);
msgpack_packer_free(pk);
if (!(request->flags & REQUEST_FLAG_NO_REPLY)) {
if (!(buffer->size > 0)) {
response_header.payload_size = 0;
nwritten = writen(client_fd, &response_header, sizeof(response_header));
if (!nwritten) {
log_error("writen failed: %s", strerror(errno));
msgpack_sbuffer_free(buffer);
break;
}
}
else {
if (!enable_gzip) {
response_header.payload_size = buffer->size;
nwritten = writen(client_fd, &response_header, sizeof(response_header));
if (!nwritten) {
log_error("writen failed: %s", strerror(errno));
msgpack_sbuffer_free(buffer);
break;
}
nwritten = writen(client_fd, buffer->data, buffer->size);
if (!nwritten) {
log_error("writen failed: %s", strerror(errno));
msgpack_sbuffer_free(buffer);
break;
}
}
else {
uLongf destLen = buffer->size * 1.00101 + 13;
Bytef *dest = malloc(destLen);
if (!dest) {
log_error("malloc failed: %s", strerror(errno));
msgpack_sbuffer_free(buffer);
break;
}
int ret = compress(dest, &destLen, (Bytef *) buffer->data, buffer->size);
while (ret == Z_BUF_ERROR) {
destLen = destLen * 2 + 16;
free(dest);
dest = malloc(destLen);
if (!dest) {
log_error("malloc failed: %s", strerror(errno));
msgpack_sbuffer_free(buffer);
break;
}
ret = compress(dest, &destLen, (Bytef *) buffer->data, buffer->size);
}
if (ret != Z_OK) {
log_error("error while compressing response: %d", ret);
if (dest)
free(dest);
break;
}
response_header.payload_size = destLen;
nwritten = writen(client_fd, &response_header, sizeof(response_header));
if (!nwritten) {
log_warn("peer closed connection");
msgpack_sbuffer_free(buffer);
free(dest);
break;
}
if (nwritten < 0) {
log_error("send response header failed: %s", strerror(errno));
msgpack_sbuffer_free(buffer);
free(dest);
break;
}
if (nwritten < (ssize_t)sizeof(response_header)) {
log_warn("error while sending response header, sent %zd (should be %"PRIu64")", nwritten, sizeof(response_header));
msgpack_sbuffer_free(buffer);
free(dest);
break;
}
nwritten = writen(client_fd, dest, destLen);
if (!nwritten) {
log_warn("peer closed connection");
msgpack_sbuffer_free(buffer);
free(dest);
break;
}
if (nwritten < 0) {
log_error("send response failed: %s", strerror(errno));
msgpack_sbuffer_free(buffer);
free(dest);
break;
}
if (nwritten < (ssize_t)destLen) {
log_warn("error while sending response, sent %zd (should be %"PRIu64")", nwritten, destLen);
msgpack_sbuffer_free(buffer);
free(dest);
break;
}
free(dest);
}
}
}
msgpack_sbuffer_free(buffer);
if (request_size > 65536) {
request_size = 8192;
request_back = request;
request = realloc(request, request_size);
if (!request) {
log_error("realloc failed: %s", strerror(errno));
free(request_back);
break;
}
}
}
close(client_fd);
free(request);
msgpack_unpacked_destroy(&msg);
return NULL;
}
static void
msgpack_wrap_redis_reply(const struct cmd *cmd, struct msg_out *out, const redisReply *r) {
unsigned int i;
msgpack_packer* pk = msgpack_packer_new(out, on_msgpack_write);
/* copy verb, as jansson only takes a char* but not its length. */
char *verb = "";
size_t verb_sz = 0;
if(cmd->count) {
verb_sz = cmd->argv_len[0];
verb = cmd->argv[0];
}
/* Create map object */
msgpack_pack_map(pk, 1);
/* The single element is the verb */
msgpack_pack_raw(pk, verb_sz);
msgpack_pack_raw_body(pk, verb, verb_sz);
switch(r->type) {
case REDIS_REPLY_STATUS:
case REDIS_REPLY_ERROR:
msgpack_pack_array(pk, 2);
/* first element: book */
if(r->type == REDIS_REPLY_ERROR)
msgpack_pack_false(pk);
else
msgpack_pack_true(pk);
/* second element: message */
msgpack_pack_raw(pk, r->len);
msgpack_pack_raw_body(pk, r->str, r->len);
break;
case REDIS_REPLY_STRING:
if(verb_sz ==4 && strncasecmp(verb, "INFO", 4) == 0) {
msg_info_reply(pk, r->str, r->len);
} else {
msgpack_pack_raw(pk, r->len);
msgpack_pack_raw_body(pk, r->str, r->len);
}
break;
case REDIS_REPLY_INTEGER:
msgpack_pack_int(pk, r->integer);
break;
case REDIS_REPLY_ARRAY:
if(verb_sz == 7 && strncasecmp(verb, "HGETALL", 7) == 0) {
msg_hgetall_reply(pk, r);
break;
}
msgpack_pack_array(pk, r->elements);
for(i = 0; i < r->elements; ++i) {
redisReply *e = r->element[i];
switch(e->type) {
case REDIS_REPLY_STRING:
msgpack_pack_raw(pk, e->len);
msgpack_pack_raw_body(pk, e->str, e->len);
break;
case REDIS_REPLY_INTEGER:
msgpack_pack_int(pk, e->integer);
break;
default:
msgpack_pack_nil(pk);
break;
}
}
break;
default:
msgpack_pack_nil(pk);
break;
}
msgpack_packer_free(pk);
}
void publish_proc_event(process_t *process, const char *name) {
/* TODO(sissel): move this to a separate file for 'event' streaming */
zmq_msg_t event;
int rc;
size_t msgsize;
program_t *program = pn_proc_program(process);
procnanny_t *pn = program->data;
/* Fields:
* - program name
* - process instance
* - exit status
* - duration ?
*/
msgpack_sbuffer *buffer = msgpack_sbuffer_new();
msgpack_packer *output_msg = msgpack_packer_new(buffer, msgpack_sbuffer_write);
msgpack_pack_map(output_msg, 5);
msgpack_pack_string(output_msg, "event", -1);
msgpack_pack_string(output_msg, name, -1);
msgpack_pack_string(output_msg, "program", -1);
msgpack_pack_string(output_msg, program->name, program->name_len);
msgpack_pack_string(output_msg, "instance", -1);
msgpack_pack_int(output_msg, process->instance);
msgpack_pack_string(output_msg, "state", -1);
switch (process->state) {
case PROCESS_STATE_STARTING: msgpack_pack_string(output_msg, "starting", -1); break;
case PROCESS_STATE_RUNNING: msgpack_pack_string(output_msg, "running", -1); break;
case PROCESS_STATE_STOPPING: msgpack_pack_string(output_msg, "stopping", -1); break;
case PROCESS_STATE_EXITED: msgpack_pack_string(output_msg, "exited", -1); break;
case PROCESS_STATE_BACKOFF: msgpack_pack_string(output_msg, "backoff", -1); break;
case PROCESS_STATE_NEW: msgpack_pack_string(output_msg, "new", -1); break;
default: msgpack_pack_string(output_msg, "unknown", -1); break;
}
if (process->state == PROCESS_STATE_EXITED
|| process->state == PROCESS_STATE_BACKOFF) {
msgpack_pack_string(output_msg, "code", -1);
msgpack_pack_nil(output_msg);
} else if (process->exit_signal == 0) {
msgpack_pack_string(output_msg, "code", -1);
msgpack_pack_int(output_msg, process->exit_status);
} else {
msgpack_pack_string(output_msg, "signal", -1);
/* lol. */
switch (process->exit_signal) {
case SIGHUP: msgpack_pack_string(output_msg, "HUP", 3); break;
case SIGINT: msgpack_pack_string(output_msg, "INT", 3); break;
case SIGQUIT: msgpack_pack_string(output_msg, "QUIT", 4); break;
case SIGILL: msgpack_pack_string(output_msg, "ILL", 3); break;
case SIGTRAP: msgpack_pack_string(output_msg, "TRAP", 4); break;
case SIGABRT: msgpack_pack_string(output_msg, "ABRT", 4); break;
case SIGBUS: msgpack_pack_string(output_msg, "BUS", 3); break;
case SIGFPE: msgpack_pack_string(output_msg, "FPE", 3); break;
case SIGKILL: msgpack_pack_string(output_msg, "KILL", 4); break;
case SIGUSR1: msgpack_pack_string(output_msg, "USR1", 4); break;
case SIGSEGV: msgpack_pack_string(output_msg, "SEGV", 4); break;
case SIGUSR2: msgpack_pack_string(output_msg, "USR2", 4); break;
case SIGPIPE: msgpack_pack_string(output_msg, "PIPE", 4); break;
case SIGALRM: msgpack_pack_string(output_msg, "ALRM", 4); break;
case SIGTERM: msgpack_pack_string(output_msg, "TERM", 4); break;
case SIGSTKFLT: msgpack_pack_string(output_msg, "STKFLT", 6); break;
case SIGCHLD: msgpack_pack_string(output_msg, "CHLD", 4); break;
case SIGCONT: msgpack_pack_string(output_msg, "CONT", 4); break;
case SIGSTOP: msgpack_pack_string(output_msg, "STOP", 4); break;
case SIGTSTP: msgpack_pack_string(output_msg, "TSTP", 4); break;
case SIGTTIN: msgpack_pack_string(output_msg, "TTIN", 4); break;
case SIGTTOU: msgpack_pack_string(output_msg, "TTOU", 4); break;
case SIGURG: msgpack_pack_string(output_msg, "URG", 3); break;
case SIGXCPU: msgpack_pack_string(output_msg, "XCPU", 4); break;
case SIGXFSZ: msgpack_pack_string(output_msg, "XFSZ", 4); break;
case SIGVTALRM: msgpack_pack_string(output_msg, "VTALRM", 5); break;
case SIGPROF: msgpack_pack_string(output_msg, "PROF", 4); break;
case SIGWINCH: msgpack_pack_string(output_msg, "WINCH", 5); break;
case SIGPOLL: msgpack_pack_string(output_msg, "POLL", 4); break;
case SIGPWR: msgpack_pack_string(output_msg, "PWR", 3); break;
case SIGSYS: msgpack_pack_string(output_msg, "SYS", 3); break;
default: msgpack_pack_string(output_msg, "unknown", -1); break;
} /* switch process->exit_signal */
} /* if process was killed by a signal */
zmq_msg_init_data(&event, buffer->data, buffer->size, free_msgpack_buffer, buffer);
zmq_send(pn->eventsocket, &event, 0);
zmq_msg_close(&event);
msgpack_packer_free(output_msg);
} /* publish_output */
void session_report(struct session * s, char type){
if(s->need_report==0){
return;
}
s->need_report = 0;
struct msg_report rp;
rp.buffer = msgpack_sbuffer_new();
rp.pk = msgpack_packer_new(rp.buffer, msgpack_sbuffer_write);
rp.count = 0;
// 15 < keys < 65535
// https://github.com/msgpack/msgpack/blob/master/spec.md#formats-map
// +--------+--------+--------+~~~~~~~~~~~~~~~~~+
// | 0xde |YYYYYYYY|YYYYYYYY| N*2 objects |
// +--------+--------+--------+~~~~~~~~~~~~~~~~~+
msgpack_pack_map(rp.pk, 20);
report_add_pair(&rp, "@class", "http-scope");
report_add_pair_int(&rp, "@time", s->start_time);
report_add_pair(&rp, "method", http_method_str(s->method));
report_add_pair(&rp, "host", s->host);
report_add_pair(&rp, "host", s->host);
report_add_pair(&rp, "path", s->path);
report_add_pair(&rp, "node", config.node);
report_add_pair_int(&rp, "code", s->status_code);
//print_data(rp.buffer->data, rp.buffer->size);
report_add_pair(&rp, "server", int_ntoa(s->tcp->addr.daddr));
report_add_pair_int(&rp, "server-port", s->tcp->addr.dest);
report_add_pair(&rp, "client", int_ntoa(s->tcp->addr.saddr));
// report_add_pair_int(&rp, "loat_packet", s->lost_packets);
// report_add_pair_int(&rp, "total_packets", s->packets);
// /* report_add_packet_int(&rp, "packets", "%d/%d",s->lost_packets, s->packets);*/
report_add_pair(&rp, "status", close_status(type));
report_add_pair_int(&rp, "req-bytes", s->tcp->server.count);
report_add_pair_int(&rp, "rep-bytes", s->tcp->client.count);
report_add_pair_float(&rp, "net-req-time", s->req_time/1000000.0);
report_add_pair_float(&rp, "server-time", s->server_time/1000000.0);
report_add_pair_float(&rp, "net-rep-time", s->rep_time/1000000.0);
if(s->referer){
report_add_pair(&rp, "referer", s->referer);
char * referer_host = strstr(s->referer, "//");
if(referer_host){
referer_host+=2;
char *p = strstr(referer_host, "/");
if(p) *p = 0;
report_add_pair(&rp, "referer-host", referer_host);
}
}
struct kv *p = s->proplist;
while(p){
urldecode(p->value);
report_add_pair(&rp, kv_type_string(p->type, p->key), p->value);
p = p->next;
}
if(s->is_catch_response_body && s->response_body_size<65535){
report_add_pair_int(&rp, "body-captured-size", s->response_body_size);
report_add_key(&rp, "body-captured");
msgpack_pack_raw(rp.pk, s->response_body_size);
struct body_buf *b = s->response_body_first;
int size_pushed = 0;
while(b){
msgpack_pack_raw_body(rp.pk, b->data, b->size);
size_pushed += b->size;
b = b->next;
}
}
//print_data(rp.buffer->data, rp.buffer->size);
char *cp = rp.buffer->data;
cp = cp + 1;
*cp = rp.count >> 8;
cp = cp + 1;
*cp = rp.count % 256;
//printf("rp.buffer->size=%d\n", rp.count);
rp.routing_key = malloc(ROUTING_KEY_BUF_SIZE);
bzero(rp.routing_key, ROUTING_KEY_BUF_SIZE);
snprintf(rp.routing_key, ROUTING_KEY_BUF_SIZE,
"http-scope.%s.%dxx.%d", s->host, s->status_code / 100 ,s->status_code);
send_report(&rp);
//print_data(rp.buffer->data, rp.buffer->size);
free(rp.routing_key);
msgpack_sbuffer_free(rp.buffer);
msgpack_packer_free(rp.pk);
session_log(s);
session_clean(s);
session_start(s);
}
void rpc_service_handle(rpc_service_t *service, zmq_msg_t *request) {
/* Parse the msgpack */
zmq_msg_t response;
int rc;
msgpack_unpacked request_msg;
msgpack_unpacked_init(&request_msg);
rc = msgpack_unpack_next(&request_msg, zmq_msg_data(request),
zmq_msg_size(request), NULL);
insist_return(rc, (void)(0), "Failed to unpack message '%.*s'",
(int)zmq_msg_size(request), (char *)zmq_msg_data(request));
msgpack_object request_obj = request_msg.data;
/* Find the method name */
char *method = NULL;
size_t method_len = -1;
rc = obj_get(&request_obj, "method", MSGPACK_OBJECT_RAW, &method, &method_len);
msgpack_sbuffer *response_buffer = msgpack_sbuffer_new();
msgpack_sbuffer *result_buffer = msgpack_sbuffer_new();
msgpack_sbuffer *error_buffer = msgpack_sbuffer_new();
msgpack_packer *response_msg = msgpack_packer_new(response_buffer, msgpack_sbuffer_write);
msgpack_packer *result = msgpack_packer_new(result_buffer, msgpack_sbuffer_write);
msgpack_packer *error = msgpack_packer_new(error_buffer, msgpack_sbuffer_write);
//printf("Method: %.*s\n", method_len, method);
void *clock = zmq_stopwatch_start();
double duration;
if (rc != 0) { /* method not found */
msgpack_pack_nil(result); /* result is nil on error */
msgpack_pack_map(error, 2);
msgpack_pack_string(error, "error", -1);
msgpack_pack_string(error, "Message had no 'method' field", -1);
msgpack_pack_string(error, "request", -1);
msgpack_pack_object(error, request_obj);
} else { /* valid method, keep going */
//printf("The method is: '%.*s'\n", (int)method_len, method);
rpc_name name;
name.name = method;
name.len = method_len;
rpc_method *rpcmethod = g_tree_lookup(service->methods, &name);
/* if we found a valid rpc method and the args check passed ... */
if (rpcmethod != NULL) {
/* the callback is responsible for filling in the 'result' and 'error'
* objects. */
rpcmethod->callback(NULL, &request_obj, result, error, rpcmethod->data);
} else {
msgpack_pack_nil(result); /* result is nil on error */
/* TODO(sissel): allow methods to register themselves */
//fprintf(stderr, "Invalid request '%.*s' (unknown method): ",
//method_len, method);
//msgpack_object_print(stderr, request_obj);
//fprintf(stderr, "\n");
msgpack_pack_map(error, 2);
msgpack_pack_string(error, "error", -1);
msgpack_pack_string(error, "No such method requested", -1);
msgpack_pack_string(error, "request", -1);
msgpack_pack_object(error, request_obj);
}
} /* valid/invalid method handling */
duration = zmq_stopwatch_stop(clock) / 1000000.;
//printf("method '%.*s' took %lf seconds\n", (int)method_len, method);
msgpack_unpacked result_unpacked;
msgpack_unpacked error_unpacked;
msgpack_unpacked response_unpacked;
msgpack_unpacked_init(&result_unpacked);
msgpack_unpacked_init(&error_unpacked);
msgpack_unpacked_init(&response_unpacked);
/* TODO(sissel): If this unpack test fails, we should return an error to the calling
* client indicating that some internal error has occurred */
//fprintf(stderr, "Result payload: '%.*s'\n", result_buffer->size,
//result_buffer->data);
rc = msgpack_unpack_next(&result_unpacked, result_buffer->data,
result_buffer->size, NULL);
insist(rc == true, "msgpack_unpack_next failed on 'result' buffer"
" of request '%.*s'", (int)method_len, method);
rc = msgpack_unpack_next(&error_unpacked, error_buffer->data,
error_buffer->size, NULL);
insist(rc == true, "msgpack_unpack_next failed on 'error' buffer"
" of request '%.*s'", (int)method_len, method);
msgpack_pack_map(response_msg, 3); /* result, error, duration */
msgpack_pack_string(response_msg, "result", 6);
msgpack_pack_object(response_msg, result_unpacked.data);
msgpack_pack_string(response_msg, "error", 5);
msgpack_pack_object(response_msg, error_unpacked.data);
msgpack_pack_string(response_msg, "duration", 8);
msgpack_pack_double(response_msg, duration);
rc = msgpack_unpack_next(&response_unpacked, response_buffer->data,
response_buffer->size, NULL);
insist(rc == true, "msgpack_unpack_next failed on full response buffer"
" of request '%.*s'", (int)method_len, method);
//printf("request: ");
//msgpack_object_print(stdout, request_obj);
//printf("\n");
//printf("response: ");
//msgpack_object_print(stdout, response_unpacked.data);
//printf("\n");
zmq_msg_init_data(&response, response_buffer->data, response_buffer->size,
free_msgpack_buffer, response_buffer);
zmq_send(service->socket, &response, 0);
zmq_msg_close(&response);
msgpack_packer_free(error);
msgpack_packer_free(result);
msgpack_sbuffer_free(error_buffer);
msgpack_sbuffer_free(result_buffer);
msgpack_packer_free(response_msg);
msgpack_unpacked_destroy(&request_msg);
} /* rpc_service_handle */
void msgpack_packer_free_wrap(msgpack_packer* pk)
{
msgpack_packer_free(pk);
}
void write_raw_msgpack(FILE *out, conjugrad_float_t *x, int ncol, void *meta) {
int nsingle = ncol * (N_ALPHA - 1);
int nsingle_padded = nsingle + N_ALPHA_PAD - (nsingle % N_ALPHA_PAD);
conjugrad_float_t *x1 = x;
conjugrad_float_t *x2 = &x[nsingle_padded];
(void)x2;
msgpack_sbuffer* buffer = msgpack_sbuffer_new();
msgpack_packer* pk = msgpack_packer_new(buffer, msgpack_sbuffer_write);
#ifdef JANSSON
if(meta != NULL) {
msgpack_pack_map(pk, 5);
meta_write_msgpack(pk, (json_t *)meta);
} else {
msgpack_pack_map(pk, 4);
}
#else
msgpack_pack_map(pk, 4);
#endif
msgpack_pack_str(pk, 6);
msgpack_pack_str_body(pk, "format", 6);
msgpack_pack_str(pk, 5);
msgpack_pack_str_body(pk, "ccm-1", 5);
msgpack_pack_str(pk, 4);
msgpack_pack_str_body(pk, "ncol", 4);
msgpack_pack_int32(pk, ncol);
msgpack_pack_str(pk, 8);
msgpack_pack_str_body(pk, "x_single", 8);
msgpack_pack_array(pk, ncol * (N_ALPHA - 1));
for(int i = 0; i < ncol; i++) {
for(int a = 0; a < N_ALPHA - 1; a++) {
#if CONJUGRAD_FLOAT == 32
msgpack_pack_float(pk, V(i, a));
#elif CONJUGRAD_FLOAT == 64
msgpack_pack_double(pk, V(i, a));
#endif
}
}
msgpack_pack_str(pk, 6);
msgpack_pack_str_body(pk, "x_pair", 6);
int nedge = ncol * (ncol - 1) / 2;
msgpack_pack_map(pk, nedge);
char sbuf[8192];
for(int i = 0; i < ncol; i++) {
for(int j = i + 1; j < ncol; j++) {
int nchar = snprintf(sbuf, 8192, "%d/%d", i, j);
msgpack_pack_str(pk, nchar);
msgpack_pack_str_body(pk, sbuf, nchar);
msgpack_pack_map(pk, 3);
msgpack_pack_str(pk, 1);
msgpack_pack_str_body(pk, "i", 1);
msgpack_pack_int32(pk, i);
msgpack_pack_str(pk, 1);
msgpack_pack_str_body(pk, "j", 1);
msgpack_pack_int32(pk, j);
msgpack_pack_str(pk, 1);
msgpack_pack_str_body(pk, "x", 1);
msgpack_pack_array(pk, N_ALPHA * N_ALPHA);
for(int a = 0; a < N_ALPHA; a++) {
for(int b = 0; b < N_ALPHA; b++) {
#if CONJUGRAD_FLOAT == 32
msgpack_pack_float(pk, W(b, j, a, i));
#elif CONJUGRAD_FLOAT == 64
msgpack_pack_double(pk, W(b, j, a, i));
#endif
}
}
}
}
fwrite(buffer->data, buffer->size, 1, out);
msgpack_sbuffer_free(buffer);
msgpack_packer_free(pk);
}
void rpc_handle(rpc_io *rpcio, zmq_msg_t *request) {
/* Parse the msgpack */
zmq_msg_t response;
int rc;
msgpack_unpacked request_msg;
msgpack_unpacked_init(&request_msg);
rc = msgpack_unpack_next(&request_msg, zmq_msg_data(request),
zmq_msg_size(request), NULL);
insist_return(rc, (void)(0), "Failed to unpack message '%.*s'",
zmq_msg_size(request), zmq_msg_data(request));
msgpack_object request_obj = request_msg.data;
printf("Object: ");
msgpack_object_print(stdout, request_obj); /*=> ["Hello", "MessagePack"] */
printf("\n");
/* Find the method name */
char *method = NULL;
size_t method_len = -1;
rc = obj_get(&request_obj, "request", MSGPACK_OBJECT_RAW, &method, &method_len);
msgpack_sbuffer *buffer = msgpack_sbuffer_new();
msgpack_packer *response_msg = msgpack_packer_new(buffer, msgpack_sbuffer_write);
printf("Method: %.*s\n", method_len, method);
if (rc != 0) {
fprintf(stderr, "Message had no 'request' field. Ignoring: ");
msgpack_object_print(stderr, request_obj);
msgpack_pack_map(response_msg, 2);
msgpack_pack_string(response_msg, "error", -1);
msgpack_pack_string(response_msg, "Message had no 'request' field", -1);
msgpack_pack_string(response_msg, "request", -1);
msgpack_pack_object(response_msg, request_obj);
} else {
/* Found request */
printf("The method is: '%.*s'\n", (int)method_len, method);
//msgpack_pack_map(response_msg, 2);
//msgpack_pack_string(response_msg, "results", 7);
void *clock = zmq_stopwatch_start();
/* TODO(sissel): Use gperf here or allow methods to register themselves */
if (!strncmp("dance", method, method_len)) {
api_request_dance(rpcio, &request_obj, response_msg);
} else if (!strncmp("restart", method, method_len)) {
api_request_restart(rpcio, &request_obj, response_msg);
} else if (!strncmp("status", method, method_len)) {
api_request_status(rpcio, &request_obj, response_msg);
} else if (!strncmp("create", method, method_len)) {
api_request_create(rpcio, &request_obj, response_msg);
} else {
fprintf(stderr, "Invalid request '%.*s' (unknown method): ", method_len, method);
msgpack_object_print(stderr, request_obj);
msgpack_pack_map(response_msg, 2);
msgpack_pack_string(response_msg, "error", -1);
msgpack_pack_string(response_msg, "No such method requested", -1);
msgpack_pack_string(response_msg, "request", -1);
msgpack_pack_object(response_msg, request_obj);
}
double duration = zmq_stopwatch_stop(clock) / 1000000.;
printf("method '%.*s' took %lf seconds\n", (int)method_len, method);
//msgpack_pack_string(response_msg, "stats", 5);
//msgpack_pack_map(response_msg, 1),
//msgpack_pack_string(response_msg, "duration", 8);
//msgpack_pack_double(response_msg, duration);
}
zmq_msg_init_data(&response, buffer->data, buffer->size, free_msgpack_buffer, buffer);
zmq_send(rpcio->socket, &response, 0);
zmq_msg_close(&response);
//msgpack_sbuffer_free(buffer);
msgpack_packer_free(response_msg);
msgpack_unpacked_destroy(&request_msg);
} /* rpc_handle */
