void unpack(char const* buf, size_t len) {
/* buf is allocated by client. */
msgpack_unpacked result;
size_t off = 0;
msgpack_unpack_return ret;
int i = 0;
msgpack_unpacked_init(&result);
ret = msgpack_unpack_next(&result, buf, len, &off);
while (ret == MSGPACK_UNPACK_SUCCESS) {
msgpack_object obj = result.data;
/* Use obj. */
printf("Object no %d:\n", ++i);
msgpack_object_print(stdout, obj);
printf("\n");
/* If you want to allocate something on the zone, you can use zone. */
/* msgpack_zone* zone = result.zone; */
/* The lifetime of the obj and the zone, */
ret = msgpack_unpack_next(&result, buf, len, &off);
}
msgpack_unpacked_destroy(&result);
if (ret == MSGPACK_UNPACK_CONTINUE) {
printf("All msgpack_object in the buffer is consumed.\n");
}
else if (ret == MSGPACK_UNPACK_PARSE_ERROR) {
printf("The data in the buf is invalid format.\n");
}
}
struct mail_body* unpack(char * buf, size_t len) {
msgpack_unpack_return ret;
msgpack_unpacked result;
int i = 0;
size_t off = 0;
struct mail_body *mb;
msgpack_unpacked_init(&result);
ret = msgpack_unpack_next(&result, buf, len, &off);
// add judge the result.type whether it is the required.
// That is to say to identify the data type.
judge(result.data.via.u64);
mb->type = (unsigned long)result.data.via.u64;
ret = msgpack_unpack_next(&result, buf, len, &off);
strncpy(mb->username, result.data.via.str.ptr, result.data.via.str.size);
ret = msgpack_unpack_next(&result, buf, len, &off);
strncpy(mb->password, result.data.via.str.ptr, result.data.via.str.size);
return mb;
}
void test()
{
uint64_t test_u64 = 0xFFF0000000000001LL;
size_t size = 10000000;
msgpack_sbuffer buf;
msgpack_sbuffer_init(&buf);
msgpack_packer * pk = msgpack_packer_new(&buf, msgpack_sbuffer_write);
msgpack_pack_array(pk, size);
{
int idx = 0;
for (; idx < size; ++idx)
msgpack_pack_uint64(pk, test_u64);
}
msgpack_packer_free(pk);
size_t upk_pos = 0;
msgpack_unpacked msg;
msgpack_unpacked_init(&msg);
while (msgpack_unpack_next(&msg, buf.data, buf.size, &upk_pos)) {
}
msgpack_sbuffer_destroy(&buf);
}
int main(void) {
/* creates buffer and serializer instance. */
msgpack_sbuffer* buffer = msgpack_sbuffer_new();
msgpack_packer* pk = msgpack_packer_new(buffer, msgpack_sbuffer_write);
/* serializes ["Hello", "MessagePack"]. */
msgpack_pack_array(pk, 2);
msgpack_pack_raw(pk, 5);
msgpack_pack_raw_body(pk, "Hello", 5);
msgpack_pack_raw(pk, 11);
msgpack_pack_raw_body(pk, "MessagePack", 11);
/* deserializes it. */
msgpack_unpacked msg;
msgpack_unpacked_init(&msg);
bool success = msgpack_unpack_next(&msg, buffer->data, buffer->size, NULL);
/* prints the deserialized object. */
msgpack_object obj = msg.data;
msgpack_object_print(stdout, obj); /*=> ["Hello", "MessagePack"] */
puts("");
// msgpack_pack_object/pk
/* cleaning */
msgpack_sbuffer_free(buffer);
msgpack_packer_free(pk);
}
static int event(zloop_t *loop, zmq_pollitem_t *item, void *arg)
{
if (interrupt)
return -1;
zmsg_t *msg = zmsg_recv(dealer);
zframe_t *payload = zmsg_pop(msg);
zmsg_destroy(&msg);
msgpack_unpacked object;
msgpack_unpacked_init(&object);
if (msgpack_unpack_next(&object, (char*)zframe_data(payload), zframe_size(payload) , NULL))
{
//zclock_log("message");
//msgpack_object_print(stdout, object.data);
char *command = (char*)m_lookup(object.data, "command");
if (command) {
//zclock_log("command: %s", command);
if (streq(command, "exception")) {
failed++;
}
if (streq(command, "result")) {
success++;
}
free(command);
}
}
msgpack_unpacked_destroy(&object);
zframe_destroy(&payload);
return 0;
}
static inline int process_pack(struct tcp_conn *conn,
char *pack, size_t size)
{
size_t off = 0;
msgpack_unpacked result;
msgpack_object entry;
struct flb_in_tcp_config *ctx;
ctx = conn->ctx;
/* First pack the results, iterate concatenated messages */
msgpack_unpacked_init(&result);
while (msgpack_unpack_next(&result, pack, size, &off)) {
entry = result.data;
msgpack_pack_array(&ctx->mp_pck, 2);
msgpack_pack_uint64(&ctx->mp_pck, time(NULL));
msgpack_pack_map(&ctx->mp_pck, 1);
msgpack_pack_bin(&ctx->mp_pck, 3);
msgpack_pack_bin_body(&ctx->mp_pck, "msg", 3);
msgpack_pack_object(&ctx->mp_pck, entry);
ctx->buffer_id++;
}
msgpack_unpacked_destroy(&result);
return 0;
}
yar_unpackager * yar_unpack_init(char *data, uint len) /* {{{ */ {
yar_unpackager *unpk = malloc(sizeof(yar_unpackager));
msgpack_unpacked_init(&unpk->msg);
if (!msgpack_unpack_next(&unpk->msg, data, len, NULL)) {
yar_unpack_free(unpk);
return NULL;
}
return unpk;
}
void start_echo(char *arg)
{
struct timeval tv;
gettimeofday(&tv,NULL);
start = (uint64_t)tv.tv_sec;
//while(!interrupt)
//{
char *actionid = getActionid();
zmsg_t *msg = create_call(actionid, "worker", "sleep", arg);
if (msg)
{
zmsg_send(&msg, dealer);
counter ++;
zmsg_destroy(&msg);
zmsg_t *msg = zmsg_recv(dealer);
zframe_t *payload = zmsg_pop(msg);
zmsg_destroy(&msg);
msgpack_unpacked object;
msgpack_unpacked_init(&object);
if (msgpack_unpack_next(&object, (char*)zframe_data(payload), zframe_size(payload) , NULL))
{
//zclock_log("message");
//msgpack_object_print(stdout, object.data);
char *command = (char*)m_lookup(object.data, "command");
if (command) {
//zclock_log("command: %s", command);
if (streq(command, "exception")) {
failed++;
zclock_log("exception");
}
if (streq(command, "result")) {
success++;
zclock_log("result ok");
}
free(command);
}
}
msgpack_unpacked_destroy(&object);
zframe_destroy(&payload);
}
gettimeofday(&tv,NULL);
uint64_t end = (uint64_t)tv.tv_sec;
if ((end - start) == 1)
{
float speed = counter/(end-start);
zclock_log("speed %f m/s, failed %ld, success %ld", speed, failed, success);
counter = 0;
failed = 0;
success = 0;
start = end;
}
//} //end while
}
int handle_rep(zmq_msg_t * msg)
{
msgpack_object obj;
msgpack_unpacked pack;
char *errmsg;
msgpack_unpacked_init(&pack);
if (!msgpack_unpack_next
(&pack, zmq_msg_data(msg), zmq_msg_size(msg), NULL)) {
LOG_ERROR("handle_rep: msgpack_unpack_next failed\n");
return (-1);
}
obj = pack.data;
if (obj.type != MSGPACK_OBJECT_ARRAY) {
LOG_ERROR("handle_rep: not an array\n");
return (-2);
}
if (obj.via.array.size < 1) {
LOG_ERROR("handle_rep: empty array\n");
return (-3);
}
if (obj.via.array.ptr[0].type != MSGPACK_OBJECT_BOOLEAN) {
LOG_ERROR("handle_rep: first entry is nota boolean\n");
return (-4);
}
if (obj.via.array.ptr[0].via.boolean == false) {
if (obj.via.array.size > 1
&& obj.via.array.ptr[1].type == MSGPACK_OBJECT_RAW) {
errmsg = raw_to_string(&(obj.via.array.ptr[1].via.raw));
if (errmsg) {
LOG_SERV("%s\n", errmsg);
free(errmsg);
}
}
msgpack_unpacked_destroy(&pack);
return (1);
}
if (obj.via.array.size > 1) {
/* Technically speaking unspecified, but I feel lazy */
msgpack_object_print(stdout, obj.via.array.ptr[1]);
printf("\n");
}
return (0);
}
TEST(unpack, sequence)
{
msgpack_sbuffer* sbuf = msgpack_sbuffer_new();
msgpack_packer* pk = msgpack_packer_new(sbuf, msgpack_sbuffer_write);
EXPECT_EQ(0, msgpack_pack_int(pk, 1));
EXPECT_EQ(0, msgpack_pack_int(pk, 2));
EXPECT_EQ(0, msgpack_pack_int(pk, 3));
msgpack_packer_free(pk);
msgpack_unpack_return success;
size_t offset = 0;
msgpack_unpacked msg;
msgpack_unpacked_init(&msg);
success = msgpack_unpack_next(&msg, sbuf->data, sbuf->size, &offset);
EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, success);
EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, msg.data.type);
EXPECT_EQ(1u, msg.data.via.u64);
success = msgpack_unpack_next(&msg, sbuf->data, sbuf->size, &offset);
EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, success);
EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, msg.data.type);
EXPECT_EQ(2u, msg.data.via.u64);
success = msgpack_unpack_next(&msg, sbuf->data, sbuf->size, &offset);
EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, success);
EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, msg.data.type);
EXPECT_EQ(3u, msg.data.via.u64);
success = msgpack_unpack_next(&msg, sbuf->data, sbuf->size, &offset);
EXPECT_EQ(MSGPACK_UNPACK_CONTINUE, success);
msgpack_sbuffer_free(sbuf);
msgpack_unpacked_destroy(&msg);
}
void flb_pack_print(char *data, size_t bytes)
{
msgpack_unpacked result;
size_t off = 0, cnt = 0;
msgpack_unpacked_init(&result);
while (msgpack_unpack_next(&result, data, bytes, &off)) {
/* FIXME: lazy output */
printf("[%zd] ", cnt++);
msgpack_object_print(stdout, result.data);
printf("\n");
}
msgpack_unpacked_destroy(&result);
}
void rpc_call_handle_response(rpc_call_t *rpc, zmq_msg_t *response) {
int rc;
msgpack_unpacked response_msg;
msgpack_unpacked_init(&response_msg);
rc = msgpack_unpack_next(&response_msg, zmq_msg_data(response),
zmq_msg_size(response), NULL);
insist_return(rc, (void)(0), "Failed to unpack message '%.*s'",
(int)zmq_msg_size(response), (char *)zmq_msg_data(response));
msgpack_object response_obj = response_msg.data;
/* TODO(sissel): Call rpc->callback */
printf("rpc call response: ");
msgpack_object_print(stdout, response_obj);
printf("\n");
} /* rpc_service_handle */
static gboolean
receive_message(Session *session)
{
session->message_size =
read(session->socket_fd, session->message, MESSAGE_SIZE);
if (session->message_size == -1) {
perror("failed to read()");
return FALSE;
}
if (session->message_size == 0) {
return stop_session(session);
}
if (parse_data) {
if (session->message[0] == '{') {
JsonParser *parser;
parser = json_parser_new();
json_parser_load_from_data(parser,
session->message, session->message_size,
NULL);
g_object_unref(parser);
} else {
msgpack_unpacked unpacked;
size_t offset;
msgpack_unpacked_init(&unpacked);
msgpack_unpack_next(&unpacked,
session->message, session->message_size,
&offset);
msgpack_unpacked_destroy(&unpacked);
}
}
{
int epoll_fd = session->context->epoll_fd;
struct epoll_event event;
event.events = EPOLLOUT;
event.data.ptr = session;
if (epoll_ctl(epoll_fd, EPOLL_CTL_MOD, session->socket_fd, &event) == -1) {
perror("failed to epoll_ctl(EPOLL_CTL_MOD, client_socket_fd)");
return FALSE;
}
}
return TRUE;
}
TEST(pack, insufficient)
{
msgpack_sbuffer* sbuf = msgpack_sbuffer_new();
msgpack_packer* pk = msgpack_packer_new(sbuf, msgpack_sbuffer_write);
EXPECT_EQ(0, msgpack_pack_int(pk, 255)); // uint8 (2bytes)
msgpack_unpack_return success;
size_t offset = 0;
msgpack_unpacked msg;
msgpack_unpacked_init(&msg);
success = msgpack_unpack_next(&msg, sbuf->data, 1, &offset);
EXPECT_EQ(MSGPACK_UNPACK_CONTINUE, success);
EXPECT_EQ(0u, offset);
msgpack_unpacked_destroy(&msg);
msgpack_sbuffer_free(sbuf);
msgpack_packer_free(pk);
}
int main(void)
{
msgpack_sbuffer* buffer = msgpack_sbuffer_new();
msgpack_packer* pk = msgpack_packer_new(buffer, msgpack_sbuffer_write);
msgpack_pack_array(pk, 2);
msgpack_pack_raw(pk, 5);
msgpack_pack_raw_body(pk, "hello", 5);
msgpack_pack_raw(pk, 11);
msgpack_pack_raw_body(pk, "messagepack", 11);
msgpack_unpacked msg;
msgpack_unpacked_init(&msg);
bool success = msgpack_unpack_next(&msg, buffer->data, buffer->size, NULL);
msgpack_object obj = msg.data;
msgpack_object_print(stdout, obj);
printf("\n");
msgpack_sbuffer_free(buffer);
msgpack_packer_free(pk);
}
void init_unpack_array(msgpack_object_type type)
{
size_t off = 0;
msgpack_sbuffer sbuf;
msgpack_sbuffer_init(&sbuf);
msgpack_packer pk;
msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write);
msgpack_pack_array(&pk, 3);
if (type == MSGPACK_OBJECT_NIL) {
msgpack_pack_nil(&pk);
} else if (type == MSGPACK_OBJECT_POSITIVE_INTEGER) {
msgpack_pack_int(&pk, 1);
} else if (type == MSGPACK_OBJECT_BOOLEAN) {
msgpack_pack_true(&pk);
} else if (type == MSGPACK_OBJECT_MAP) {
msgpack_pack_map(&pk, 1);
} else if (type == MSGPACK_OBJECT_ARRAY) {
msgpack_pack_array(&pk, 3);
msgpack_pack_int(&pk, 1);
msgpack_pack_int(&pk, 1);
msgpack_pack_int(&pk, 1);
} else if (type == MSGPACK_OBJECT_FLOAT) {
msgpack_pack_double(&pk, 1.2);
}
msgpack_pack_true(&pk);
msgpack_pack_str(&pk, 7);
msgpack_pack_str_body(&pk, "example", 7);
msgpack_unpacked result;
msgpack_unpacked_init(&result);
msgpack_unpack_next(&result, sbuf.data, sbuf.size, &off);
msgpack_sbuffer_destroy(&sbuf);
deserialized = result.data;
}
static void scl_read_power(float *_voltage, float *_current)
{
char buffer[128];
int ret = scl_recv_static(scl_socket, buffer, sizeof(buffer));
if (ret > 0)
{
msgpack_unpacked msg;
msgpack_unpacked_init(&msg);
if (msgpack_unpack_next(&msg, buffer, ret, NULL))
{
msgpack_object root = msg.data;
assert (root.type == MSGPACK_OBJECT_ARRAY);
assert(root.via.array.size == 2);
*_voltage = root.via.array.ptr[0].via.dec;
*_current = root.via.array.ptr[1].via.dec;
}
msgpack_unpacked_destroy(&msg);
}
else
{
sleep(1);
LOG(LL_ERROR, "could not read voltage");
}
}
int cb_fluentd_flush(void *data, size_t bytes, void *out_context,
struct flb_config *config)
{
int fd;
int ret = -1;
int maps = 0;
size_t total;
char *buf = NULL;
msgpack_packer mp_pck;
msgpack_sbuffer mp_sbuf;
msgpack_unpacked mp_umsg;
size_t mp_upos = 0;
(void) out_context;
(void) config;
/*
* The incoming data comes in Fluent Bit format an array of objects, as we
* aim to send this information to Fluentd through it in_forward plugin, we
* need to transform the data. The Fluentd in_forward plugin allows one
* of the following formats:
*
* 1. [tag, time, record]
*
* or
*
* 2. [tag, [[time,record], [time,record], ...]]
*
* we use the format #2
*/
/* Initialize packager */
msgpack_sbuffer_init(&mp_sbuf);
msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write);
/*
* Count the number of map entries
*
* FIXME: Fluent Bit should expose the number of maps into the
* data, so we avoid this silly counting.
*/
msgpack_unpacked_init(&mp_umsg);
while (msgpack_unpack_next(&mp_umsg, data, bytes, &mp_upos)) {
maps++;
}
msgpack_unpacked_destroy(&mp_umsg);
/* Output: root array */
msgpack_pack_array(&mp_pck, 2);
msgpack_pack_bin(&mp_pck, sizeof(FLB_CONFIG_DEFAULT_TAG) - 1);
msgpack_pack_bin_body(&mp_pck,
FLB_CONFIG_DEFAULT_TAG,
sizeof(FLB_CONFIG_DEFAULT_TAG) - 1);
msgpack_pack_array(&mp_pck, maps);
/* Allocate a new buffer to merge data */
total = bytes + mp_sbuf.size;
buf = malloc(total);
if (!buf) {
perror("malloc");
return -1;
}
memcpy(buf, mp_sbuf.data, mp_sbuf.size);
memcpy(buf + mp_sbuf.size, data, bytes);
msgpack_sbuffer_destroy(&mp_sbuf);
fd = flb_net_tcp_connect(out_fluentd_plugin.host,
out_fluentd_plugin.port);
if (fd <= 0) {
free(buf);
return -1;
}
/* FIXME: plain TCP write */
ret = write(fd, buf, total);
close(fd);
free(buf);
return ret;
}
/*
* This plugin accepts following formats of MessagePack:
* { map => val, map => val, map => val }
* or [ time, { map => val, map => val, map => val } ]
*/
int in_xbee_rx_queue_msgpack(struct flb_in_xbee_config *ctx, const char *buf ,int len)
{
msgpack_unpacked record;
msgpack_unpacked field;
msgpack_unpacked_init(&record);
msgpack_unpacked_init(&field);
size_t off = 0;
size_t start = 0;
size_t off2;
size_t mp_offset;
int queued = 0;
uint64_t t;
pthread_mutex_lock(&ctx->mtx_mp);
while (msgpack_unpack_next(&record, buf, len, &off)) {
if (record.data.type == MSGPACK_OBJECT_ARRAY && record.data.via.array.size == 2) {
/* [ time, { map => val, map => val, map => val } ] */
msgpack_unpacked_destroy(&field);
msgpack_unpacked_init(&field);
off2 = 0;
if (! msgpack_unpack_next(&field, buf + 1, len - 1, &off2))
break;
if (field.data.type != MSGPACK_OBJECT_POSITIVE_INTEGER)
break;
t = field.data.via.u64;
mp_offset = off2;
if (! msgpack_unpack_next(&field, buf + 1, len - 1, &off2))
break;
if (field.data.type != MSGPACK_OBJECT_MAP)
break;
in_xbee_flush_if_needed(ctx);
ctx->buffer_id++;
msgpack_pack_array(&ctx->mp_pck, 2);
msgpack_pack_uint64(&ctx->mp_pck, t);
msgpack_pack_bin_body(&ctx->mp_pck, (char*) buf + 1 + mp_offset, off2 - mp_offset);
} else if (record.data.type == MSGPACK_OBJECT_MAP) {
/* { map => val, map => val, map => val } */
in_xbee_flush_if_needed(ctx);
ctx->buffer_id++;
msgpack_pack_array(&ctx->mp_pck, 2);
msgpack_pack_uint64(&ctx->mp_pck, time(NULL));
msgpack_pack_bin_body(&ctx->mp_pck, buf + start, off - start);
} else {
break;
}
start = off;
queued++;
}
msgpack_unpacked_destroy(&record);
msgpack_unpacked_destroy(&field);
pthread_mutex_unlock(&ctx->mtx_mp);
return queued;
}
static void *client_thread(void *arg) {
ssize_t nwritten;
pthread_detach(pthread_self());
int client_fd = (int) (long) arg;
int yes = 1, tcp_keepalive_probes = 3, tcp_keepalive_time = 5, tcp_keepalive_intvl = 2;
int nodelay = 1;
if (setsockopt(client_fd, SOL_SOCKET, SO_KEEPALIVE, &yes, sizeof(yes)) == -1) {
log_error("setsockopt failed: %s", strerror(errno));
close(client_fd);
return NULL;
}
if (setsockopt(client_fd, SOL_TCP, TCP_KEEPCNT, &tcp_keepalive_probes, sizeof(tcp_keepalive_probes)) == -1) {
log_error("setsockopt failed: %s", strerror(errno));
close(client_fd);
return NULL;
}
if (setsockopt(client_fd, SOL_TCP, TCP_KEEPIDLE, &tcp_keepalive_time, sizeof(tcp_keepalive_time)) == -1) {
log_error("setsockopt failed: %s", strerror(errno));
close(client_fd);
return NULL;
}
if (setsockopt(client_fd, SOL_TCP, TCP_KEEPINTVL, &tcp_keepalive_intvl, sizeof(tcp_keepalive_intvl)) == -1) {
log_error("setsockopt failed: %s", strerror(errno));
close(client_fd);
return NULL;
}
if (setsockopt(client_fd, IPPROTO_TCP, TCP_NODELAY, &nodelay, sizeof(nodelay)) == -1) {
log_error("setsockopt failed: %s", strerror(errno));
close(client_fd);
return NULL;
}
size_t request_size = 8192;
struct request *request_back = NULL;
struct request *request = (struct request *) malloc(request_size);
if (!request) {
log_error("malloc failed: %s", strerror(errno));
close(client_fd);
return NULL;
}
msgpack_unpacked msg;
msgpack_unpacked_init(&msg);
bool enable_gzip = false;
while (1) {
struct response_header response_header;
ssize_t recvsize = readn(client_fd, request, sizeof(struct request));
if (!recvsize) {
log_warn("peer closed connection");
break;
}
if (recvsize < (ssize_t) sizeof(struct request)) {
log_warn("error while receiving header, received %zd", recvsize);
break;
}
if (request->payload_size > 256 * 1024 * 1024) {
log_warn("payload size %"PRIu32" too large", request->payload_size);
break;
}
if (sizeof(struct request) + request->payload_size > request_size) {
request_size = sizeof(struct request) + request->payload_size;
request_back = request;
request = realloc(request, request_size);
if (!request) {
log_error("realloc failed: %s", strerror(errno));
free(request_back);
break;
}
}
recvsize = readn(client_fd, request->payload, request->payload_size);
if (!recvsize) {
log_warn("peer closed connection");
break;
}
if (recvsize < request->payload_size) {
log_warn("error while receiving payload, received %zd (should be %"PRIu32")", recvsize, request->payload_size);
break;
}
if (request->flags & REQUEST_FLAG_GZIP) {
enable_gzip = true;
uLongf destLen = 16;
Bytef *dest = malloc(destLen);
if (!dest) {
log_error("malloc failed: %s", strerror(errno));
break;
}
int ret;
keep_malloc:
ret = uncompress(dest, &destLen, (Bytef *) request->payload, request->payload_size);
if (ret == Z_BUF_ERROR) {
destLen = destLen * 2;
free(dest);
dest = malloc(destLen);
if (!dest) {
log_error("malloc failed: %s", strerror(errno));
break;
}
goto keep_malloc;
}
if (ret != Z_OK) {
free(dest);
break;
}
request->flags &= ~REQUEST_FLAG_GZIP;
if (sizeof(struct request) + destLen > request_size) {
request_size = sizeof(struct request) + destLen;
request_back = request;
request = realloc(request, request_size);
if (!request) {
log_error("realloc failed: %s", strerror(errno));
free(request_back);
free(dest);
break;
}
}
memcpy(request->payload, dest, destLen);
request->payload_size = destLen;
free(dest);
}
bool success = msgpack_unpack_next(&msg, request->payload, request->payload_size, NULL);
if (!success) {
log_warn("error while parsing payload");
break;
}
msgpack_sbuffer* buffer = msgpack_sbuffer_new();
msgpack_packer* pk = msgpack_packer_new(buffer, msgpack_sbuffer_write);
response_header.seq = request->seq;
__sync_add_and_fetch(&pending_commands, 1);
if (server->terminated) {
__sync_add_and_fetch(&pending_commands, -1);
break;
}
if (server->redirection) {
response_header.status = RESPONSE_STATUS_REDIRECTION;
msgpack_pack_map(pk, 2);
mp_pack_string(pk, "host");
mp_pack_string(pk, server->redirection->host);
mp_pack_string(pk, "port");
msgpack_pack_uint16(pk, server->redirection->port);
}
else {
response_header.status = execute_command(request, request->command, msg.data, pk);
}
__sync_add_and_fetch(&pending_commands, -1);
msgpack_packer_free(pk);
if (!(request->flags & REQUEST_FLAG_NO_REPLY)) {
if (!(buffer->size > 0)) {
response_header.payload_size = 0;
nwritten = writen(client_fd, &response_header, sizeof(response_header));
if (!nwritten) {
log_error("writen failed: %s", strerror(errno));
msgpack_sbuffer_free(buffer);
break;
}
}
else {
if (!enable_gzip) {
response_header.payload_size = buffer->size;
nwritten = writen(client_fd, &response_header, sizeof(response_header));
if (!nwritten) {
log_error("writen failed: %s", strerror(errno));
msgpack_sbuffer_free(buffer);
break;
}
nwritten = writen(client_fd, buffer->data, buffer->size);
if (!nwritten) {
log_error("writen failed: %s", strerror(errno));
msgpack_sbuffer_free(buffer);
break;
}
}
else {
uLongf destLen = buffer->size * 1.00101 + 13;
Bytef *dest = malloc(destLen);
if (!dest) {
log_error("malloc failed: %s", strerror(errno));
msgpack_sbuffer_free(buffer);
break;
}
int ret = compress(dest, &destLen, (Bytef *) buffer->data, buffer->size);
while (ret == Z_BUF_ERROR) {
destLen = destLen * 2 + 16;
free(dest);
dest = malloc(destLen);
if (!dest) {
log_error("malloc failed: %s", strerror(errno));
msgpack_sbuffer_free(buffer);
break;
}
ret = compress(dest, &destLen, (Bytef *) buffer->data, buffer->size);
}
if (ret != Z_OK) {
log_error("error while compressing response: %d", ret);
if (dest)
free(dest);
break;
}
response_header.payload_size = destLen;
nwritten = writen(client_fd, &response_header, sizeof(response_header));
if (!nwritten) {
log_warn("peer closed connection");
msgpack_sbuffer_free(buffer);
free(dest);
break;
}
if (nwritten < 0) {
log_error("send response header failed: %s", strerror(errno));
msgpack_sbuffer_free(buffer);
free(dest);
break;
}
if (nwritten < (ssize_t)sizeof(response_header)) {
log_warn("error while sending response header, sent %zd (should be %"PRIu64")", nwritten, sizeof(response_header));
msgpack_sbuffer_free(buffer);
free(dest);
break;
}
nwritten = writen(client_fd, dest, destLen);
if (!nwritten) {
log_warn("peer closed connection");
msgpack_sbuffer_free(buffer);
free(dest);
break;
}
if (nwritten < 0) {
log_error("send response failed: %s", strerror(errno));
msgpack_sbuffer_free(buffer);
free(dest);
break;
}
if (nwritten < (ssize_t)destLen) {
log_warn("error while sending response, sent %zd (should be %"PRIu64")", nwritten, destLen);
msgpack_sbuffer_free(buffer);
free(dest);
break;
}
free(dest);
}
}
}
msgpack_sbuffer_free(buffer);
if (request_size > 65536) {
request_size = 8192;
request_back = request;
request = realloc(request, request_size);
if (!request) {
log_error("realloc failed: %s", strerror(errno));
free(request_back);
break;
}
}
}
close(client_fd);
free(request);
msgpack_unpacked_destroy(&msg);
return NULL;
}
/* EXPECTED ARGUMENTS:
* <endpoint> <identity>
*/
int main(int argc, char *argv[]) {
if (argc < 3) {
fprintf(stderr, "Usage: testdriver <endpoint> <identity>\n");
return 1;
}
void *context = zmq_ctx_new();
void *socket = zmq_socket(context, ZMQ_DEALER);
char *address = argv[1];
char *identity = argv[2];
fprintf(stdout, "Connecting worker %s to socket at %s\n", identity, address);
zmq_setsockopt(socket, ZMQ_IDENTITY, identity, strlen(identity));
zmq_connect(socket, address);
zmq_send_const(socket, "RDY", 3, 0);
char *outbuffer = NULL;
msgpack_unpacked unpacked;
for (;;) {
zmq_msg_t msg;
zmq_msg_init(&msg);
zmq_msg_recv(&msg, socket, 0);
size_t size = zmq_msg_size(&msg);
char *inbuffer = (char *) malloc(size + 1);
if (NULL == inbuffer) {
fprintf(stderr, "Out of Memory!\n");
return 1;
}
memcpy(inbuffer, zmq_msg_data(&msg), size);
zmq_msg_close(&msg);
inbuffer[size] = 0;
msgpack_unpacked_init(&unpacked);
size_t off = 0;
bool res = msgpack_unpack_next(&unpacked, inbuffer, size, &off);
if (!res) {
fprintf(stderr,"res == %d\n", res);
fprintf(stderr,"off == %zu\n", off);
return 1;
}
msgpack_object obj = unpacked.data;
uint32_t id = (uint32_t) obj.via.u64;
msgpack_unpacked_destroy(&unpacked);
res = msgpack_unpack_next(&unpacked, inbuffer, size, &off);
if (!res) {
fprintf(stderr, "res == %d\n", res);
fprintf(stderr, "off == %zu\n", off);
return 1;
}
obj = unpacked.data;
const char* path = obj.via.str.ptr;
PIN_SCORE_START(outbuffer);
// TODO call sut here with path
msgpack_unpacked_destroy(&unpacked);
free(inbuffer);
outbuffer = PIN_SCORE_END(id, socket);
}
}
void rpc_handle(rpc_io *rpcio, zmq_msg_t *request) {
/* Parse the msgpack */
zmq_msg_t response;
int rc;
msgpack_unpacked request_msg;
msgpack_unpacked_init(&request_msg);
rc = msgpack_unpack_next(&request_msg, zmq_msg_data(request),
zmq_msg_size(request), NULL);
insist_return(rc, (void)(0), "Failed to unpack message '%.*s'",
zmq_msg_size(request), zmq_msg_data(request));
msgpack_object request_obj = request_msg.data;
printf("Object: ");
msgpack_object_print(stdout, request_obj); /*=> ["Hello", "MessagePack"] */
printf("\n");
/* Find the method name */
char *method = NULL;
size_t method_len = -1;
rc = obj_get(&request_obj, "request", MSGPACK_OBJECT_RAW, &method, &method_len);
msgpack_sbuffer *buffer = msgpack_sbuffer_new();
msgpack_packer *response_msg = msgpack_packer_new(buffer, msgpack_sbuffer_write);
printf("Method: %.*s\n", method_len, method);
if (rc != 0) {
fprintf(stderr, "Message had no 'request' field. Ignoring: ");
msgpack_object_print(stderr, request_obj);
msgpack_pack_map(response_msg, 2);
msgpack_pack_string(response_msg, "error", -1);
msgpack_pack_string(response_msg, "Message had no 'request' field", -1);
msgpack_pack_string(response_msg, "request", -1);
msgpack_pack_object(response_msg, request_obj);
} else {
/* Found request */
printf("The method is: '%.*s'\n", (int)method_len, method);
//msgpack_pack_map(response_msg, 2);
//msgpack_pack_string(response_msg, "results", 7);
void *clock = zmq_stopwatch_start();
/* TODO(sissel): Use gperf here or allow methods to register themselves */
if (!strncmp("dance", method, method_len)) {
api_request_dance(rpcio, &request_obj, response_msg);
} else if (!strncmp("restart", method, method_len)) {
api_request_restart(rpcio, &request_obj, response_msg);
} else if (!strncmp("status", method, method_len)) {
api_request_status(rpcio, &request_obj, response_msg);
} else if (!strncmp("create", method, method_len)) {
api_request_create(rpcio, &request_obj, response_msg);
} else {
fprintf(stderr, "Invalid request '%.*s' (unknown method): ", method_len, method);
msgpack_object_print(stderr, request_obj);
msgpack_pack_map(response_msg, 2);
msgpack_pack_string(response_msg, "error", -1);
msgpack_pack_string(response_msg, "No such method requested", -1);
msgpack_pack_string(response_msg, "request", -1);
msgpack_pack_object(response_msg, request_obj);
}
double duration = zmq_stopwatch_stop(clock) / 1000000.;
printf("method '%.*s' took %lf seconds\n", (int)method_len, method);
//msgpack_pack_string(response_msg, "stats", 5);
//msgpack_pack_map(response_msg, 1),
//msgpack_pack_string(response_msg, "duration", 8);
//msgpack_pack_double(response_msg, duration);
}
zmq_msg_init_data(&response, buffer->data, buffer->size, free_msgpack_buffer, buffer);
zmq_send(rpcio->socket, &response, 0);
zmq_msg_close(&response);
//msgpack_sbuffer_free(buffer);
msgpack_packer_free(response_msg);
msgpack_unpacked_destroy(&request_msg);
} /* rpc_handle */
void cb_forward_flush(void *data, size_t bytes,
char *tag, int tag_len,
struct flb_input_instance *i_ins, void *out_context,
struct flb_config *config)
{
int ret = -1;
int entries = 0;
size_t off = 0;
size_t total;
size_t bytes_sent;
msgpack_packer mp_pck;
msgpack_sbuffer mp_sbuf;
msgpack_unpacked result;
struct flb_out_forward_config *ctx = out_context;
struct flb_upstream_conn *u_conn;
(void) i_ins;
(void) config;
flb_debug("[out_forward] request %lu bytes to flush", bytes);
/* Initialize packager */
msgpack_sbuffer_init(&mp_sbuf);
msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write);
/* Count number of entries, is there a better way to do this ? */
msgpack_unpacked_init(&result);
while (msgpack_unpack_next(&result, data, bytes, &off)) {
entries++;
}
flb_debug("[out_fw] %i entries tag='%s' tag_len=%i",
entries, tag, tag_len);
msgpack_unpacked_destroy(&result);
/* Output: root array */
msgpack_pack_array(&mp_pck, 2);
msgpack_pack_str(&mp_pck, tag_len);
msgpack_pack_str_body(&mp_pck, tag, tag_len);
msgpack_pack_array(&mp_pck, entries);
/* Get a TCP connection instance */
u_conn = flb_upstream_conn_get(ctx->u);
if (!u_conn) {
flb_error("[out_forward] no upstream connections available");
msgpack_sbuffer_destroy(&mp_sbuf);
FLB_OUTPUT_RETURN(FLB_RETRY);
}
/* Write message header */
ret = flb_io_net_write(u_conn, mp_sbuf.data, mp_sbuf.size, &bytes_sent);
if (ret == -1) {
flb_error("[out_forward] could not write chunk header");
msgpack_sbuffer_destroy(&mp_sbuf);
flb_upstream_conn_release(u_conn);
FLB_OUTPUT_RETURN(FLB_RETRY);
}
msgpack_sbuffer_destroy(&mp_sbuf);
total = ret;
/* Write body */
ret = flb_io_net_write(u_conn, data, bytes, &bytes_sent);
if (ret == -1) {
flb_error("[out_forward] error writing content body");
flb_upstream_conn_release(u_conn);
FLB_OUTPUT_RETURN(FLB_RETRY);
}
total += bytes_sent;
flb_upstream_conn_release(u_conn);
flb_trace("[out_forward] ended write()=%d bytes", total);
FLB_OUTPUT_RETURN(FLB_OK);
}
/*
* Convert the internal Fluent Bit data representation to the required
* one by Treasure Data cloud service.
*
* This function returns a new msgpack buffer and store the bytes length
* in the out_size variable.
*/
static char *td_format(void *data, size_t bytes, int *out_size)
{
int i;
int ret;
int n_size;
size_t off = 0;
time_t atime;
char *buf;
struct msgpack_sbuffer mp_sbuf;
struct msgpack_packer mp_pck;
msgpack_unpacked result;
msgpack_object root;
msgpack_object map;
msgpack_sbuffer *sbuf;
/* Initialize contexts for new output */
msgpack_sbuffer_init(&mp_sbuf);
msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write);
/* Iterate the original buffer and perform adjustments */
msgpack_unpacked_init(&result);
/* Perform some format validation */
ret = msgpack_unpack_next(&result, data, bytes, &off);
if (!ret) {
return NULL;
}
/* We 'should' get an array */
if (result.data.type != MSGPACK_OBJECT_ARRAY) {
/*
* If we got a different format, we assume the caller knows what he is
* doing, we just duplicate the content in a new buffer and cleanup.
*/
buf = malloc(bytes);
if (!buf) {
return NULL;
}
memcpy(buf, data, bytes);
*out_size = bytes;
return buf;
}
root = result.data;
if (root.via.array.size == 0) {
return NULL;
}
off = 0;
msgpack_unpacked_destroy(&result);
msgpack_unpacked_init(&result);
while (msgpack_unpack_next(&result, data, bytes, &off)) {
if (result.data.type != MSGPACK_OBJECT_ARRAY) {
continue;
}
/* Each array must have two entries: time and record */
root = result.data;
if (root.via.array.size != 2) {
continue;
}
atime = root.via.array.ptr[0].via.u64;
map = root.via.array.ptr[1];
n_size = map.via.map.size + 1;
msgpack_pack_map(&mp_pck, n_size);
msgpack_pack_bin(&mp_pck, 4);
msgpack_pack_bin_body(&mp_pck, "time", 4);
msgpack_pack_int32(&mp_pck, atime);
for (i = 0; i < n_size - 1; i++) {
msgpack_pack_object(&mp_pck, map.via.map.ptr[i].key);
msgpack_pack_object(&mp_pck, map.via.map.ptr[i].val);
}
}
msgpack_unpacked_destroy(&result);
/* Create new buffer */
sbuf = &mp_sbuf;
*out_size = sbuf->size;
buf = malloc(sbuf->size);
if (!buf) {
return NULL;
}
/* set a new buffer and re-initialize our MessagePack context */
memcpy(buf, sbuf->data, sbuf->size);
msgpack_sbuffer_destroy(&mp_sbuf);
return buf;
}
std::shared_ptr<Response> decodeResponse(std::string const &buf) {
msgpack_unpacked msg;
msgpack_unpacked_init(&msg);
ASSERTPP( msgpack_unpack_next(&msg, buf.data(), buf.size(), nullptr) );
msgpack_object obj = msg.data;
ASSERTPP(obj.type == MSGPACK_OBJECT_MAP);
std::shared_ptr<Response> r{new Response};
for(int i = 0; i < obj.via.map.size; ++i) {
msgpack_object key_obj = obj.via.map.ptr[i].key;
msgpack_object val = obj.via.map.ptr[i].val;
ASSERTPP(key_obj.type == MSGPACK_OBJECT_RAW);
msgpack_object_raw key = key_obj.via.raw;
if(key.size == 5 && 0 == strncmp("state", key.ptr, 5)) {
if(val.type == MSGPACK_OBJECT_NIL) {
// do nosing
}
else {
const char *ptr = key.ptr + key.size;
char *err = nullptr;
size_t max_len = buf.size() -
(reinterpret_cast<const char *>(ptr) - reinterpret_cast<const char *>(buf.data()));
void *void_ptr = const_cast<void *>(reinterpret_cast<const void *>(ptr));
size_t m_len = ::msgpackclen_buf_read(void_ptr, max_len, &err);
if(err)
throw std::runtime_error(std::string("msgpackclen_buf_read error: ") + err);
r->state.assign(ptr, m_len);
}
}
else if(key.size == 5 && 0 == strncmp("value", key.ptr, 5)) {
switch(val.type) {
case MSGPACK_OBJECT_NIL:
r->value = Value();
break;
case MSGPACK_OBJECT_BOOLEAN:
r->value = Value(Value::Type::Boolean, (bool)val.via.boolean);
break;
case MSGPACK_OBJECT_POSITIVE_INTEGER:
r->value = Value(Value::Type::Numeric, val.via.u64);
break;
case MSGPACK_OBJECT_NEGATIVE_INTEGER:
r->value = Value(Value::Type::Numeric, val.via.i64);
break;
case MSGPACK_OBJECT_DOUBLE:
r->value = Value(Value::Type::Numeric, val.via.dec);
break;
case MSGPACK_OBJECT_RAW:
r->value = Value(Value::Type::String, std::string(val.via.raw.ptr, val.via.raw.size));
break;
case MSGPACK_OBJECT_ARRAY:
case MSGPACK_OBJECT_MAP:
{
const char *ptr = key.ptr + key.size;
char *err = nullptr;
size_t max_len = buf.size() -
(reinterpret_cast<const char *>(ptr) - reinterpret_cast<const char *>(buf.data()));
void *void_ptr = const_cast<void *>(reinterpret_cast<const void *>(ptr));
size_t m_len = ::msgpackclen_buf_read(void_ptr, max_len, &err);
if(err)
throw std::runtime_error(std::string("msgpackclen_buf_read error: ") + err);
r->value = Value(Value::Type::Table, std::string(ptr, m_len));
}
break;
}
}
else if(key.size == 9 && 0 == strncmp("timestamp", key.ptr, 9)) {
switch(val.type) {
case MSGPACK_OBJECT_POSITIVE_INTEGER:
r->timestamp = val.via.u64;
break;
case MSGPACK_OBJECT_NEGATIVE_INTEGER:
r->timestamp = val.via.i64;
break;
case MSGPACK_OBJECT_DOUBLE:
r->timestamp = val.via.dec;
break;
default:
throw std::runtime_error("Invalid type for timestamp");
}
}
else if(key.size == 5 && 0 == strncmp("error", key.ptr, 5)) {
ASSERTPP(val.type == MSGPACK_OBJECT_RAW);
r->error.assign(val.via.raw.ptr, val.via.raw.size);
}
}
return r;
}
void rpc_service_handle(rpc_service_t *service, zmq_msg_t *request) {
/* Parse the msgpack */
zmq_msg_t response;
int rc;
msgpack_unpacked request_msg;
msgpack_unpacked_init(&request_msg);
rc = msgpack_unpack_next(&request_msg, zmq_msg_data(request),
zmq_msg_size(request), NULL);
insist_return(rc, (void)(0), "Failed to unpack message '%.*s'",
(int)zmq_msg_size(request), (char *)zmq_msg_data(request));
msgpack_object request_obj = request_msg.data;
/* Find the method name */
char *method = NULL;
size_t method_len = -1;
rc = obj_get(&request_obj, "method", MSGPACK_OBJECT_RAW, &method, &method_len);
msgpack_sbuffer *response_buffer = msgpack_sbuffer_new();
msgpack_sbuffer *result_buffer = msgpack_sbuffer_new();
msgpack_sbuffer *error_buffer = msgpack_sbuffer_new();
msgpack_packer *response_msg = msgpack_packer_new(response_buffer, msgpack_sbuffer_write);
msgpack_packer *result = msgpack_packer_new(result_buffer, msgpack_sbuffer_write);
msgpack_packer *error = msgpack_packer_new(error_buffer, msgpack_sbuffer_write);
//printf("Method: %.*s\n", method_len, method);
void *clock = zmq_stopwatch_start();
double duration;
if (rc != 0) { /* method not found */
msgpack_pack_nil(result); /* result is nil on error */
msgpack_pack_map(error, 2);
msgpack_pack_string(error, "error", -1);
msgpack_pack_string(error, "Message had no 'method' field", -1);
msgpack_pack_string(error, "request", -1);
msgpack_pack_object(error, request_obj);
} else { /* valid method, keep going */
//printf("The method is: '%.*s'\n", (int)method_len, method);
rpc_name name;
name.name = method;
name.len = method_len;
rpc_method *rpcmethod = g_tree_lookup(service->methods, &name);
/* if we found a valid rpc method and the args check passed ... */
if (rpcmethod != NULL) {
/* the callback is responsible for filling in the 'result' and 'error'
* objects. */
rpcmethod->callback(NULL, &request_obj, result, error, rpcmethod->data);
} else {
msgpack_pack_nil(result); /* result is nil on error */
/* TODO(sissel): allow methods to register themselves */
//fprintf(stderr, "Invalid request '%.*s' (unknown method): ",
//method_len, method);
//msgpack_object_print(stderr, request_obj);
//fprintf(stderr, "\n");
msgpack_pack_map(error, 2);
msgpack_pack_string(error, "error", -1);
msgpack_pack_string(error, "No such method requested", -1);
msgpack_pack_string(error, "request", -1);
msgpack_pack_object(error, request_obj);
}
} /* valid/invalid method handling */
duration = zmq_stopwatch_stop(clock) / 1000000.;
//printf("method '%.*s' took %lf seconds\n", (int)method_len, method);
msgpack_unpacked result_unpacked;
msgpack_unpacked error_unpacked;
msgpack_unpacked response_unpacked;
msgpack_unpacked_init(&result_unpacked);
msgpack_unpacked_init(&error_unpacked);
msgpack_unpacked_init(&response_unpacked);
/* TODO(sissel): If this unpack test fails, we should return an error to the calling
* client indicating that some internal error has occurred */
//fprintf(stderr, "Result payload: '%.*s'\n", result_buffer->size,
//result_buffer->data);
rc = msgpack_unpack_next(&result_unpacked, result_buffer->data,
result_buffer->size, NULL);
insist(rc == true, "msgpack_unpack_next failed on 'result' buffer"
" of request '%.*s'", (int)method_len, method);
rc = msgpack_unpack_next(&error_unpacked, error_buffer->data,
error_buffer->size, NULL);
insist(rc == true, "msgpack_unpack_next failed on 'error' buffer"
" of request '%.*s'", (int)method_len, method);
msgpack_pack_map(response_msg, 3); /* result, error, duration */
msgpack_pack_string(response_msg, "result", 6);
msgpack_pack_object(response_msg, result_unpacked.data);
msgpack_pack_string(response_msg, "error", 5);
msgpack_pack_object(response_msg, error_unpacked.data);
msgpack_pack_string(response_msg, "duration", 8);
msgpack_pack_double(response_msg, duration);
rc = msgpack_unpack_next(&response_unpacked, response_buffer->data,
response_buffer->size, NULL);
insist(rc == true, "msgpack_unpack_next failed on full response buffer"
" of request '%.*s'", (int)method_len, method);
//printf("request: ");
//msgpack_object_print(stdout, request_obj);
//printf("\n");
//printf("response: ");
//msgpack_object_print(stdout, response_unpacked.data);
//printf("\n");
zmq_msg_init_data(&response, response_buffer->data, response_buffer->size,
free_msgpack_buffer, response_buffer);
zmq_send(service->socket, &response, 0);
zmq_msg_close(&response);
msgpack_packer_free(error);
msgpack_packer_free(result);
msgpack_sbuffer_free(error_buffer);
msgpack_sbuffer_free(result_buffer);
msgpack_packer_free(response_msg);
msgpack_unpacked_destroy(&request_msg);
} /* rpc_service_handle */
int handle_req(void *socket, zmq_msg_t * msg)
{
int command_id;
char *query;
char *path = NULL;
msgpack_object obj;
msgpack_unpacked pack;
msgpack_unpacked_init(&pack);
if (!msgpack_unpack_next
(&pack, zmq_msg_data(msg), zmq_msg_size(msg), NULL)) {
LOG_ERROR("handle_req: msgpack_unpack_next failed\n");
return (-1);
}
obj = pack.data;
if (obj.type != MSGPACK_OBJECT_ARRAY) {
LOG_ERROR("handle_req: not an array\n");
pcma_send(socket, failed_packfn, "not an array");
return (-2);
}
const char *command = (const char *) obj.via.array.ptr[0].via.raw.ptr;
int command_size = obj.via.array.ptr[0].via.raw.size;
if (!command) {
LOG_ERROR("handle_req: no command\n");
pcma_send(socket, failed_packfn, "no command");
return (-3);
}
if (command_size == PING_COMMAND_SIZE &&
!bcmp(PING_COMMAND, command, PING_COMMAND_SIZE)) {
command_id = PING_COMMAND_ID;
} else if (command_size == LIST_COMMAND_SIZE &&
!bcmp(LIST_COMMAND, command, LIST_COMMAND_SIZE)) {
command_id = LIST_COMMAND_ID;
} else if (command_size == LOCK_COMMAND_SIZE &&
!bcmp(LOCK_COMMAND, command, LOCK_COMMAND_SIZE)) {
command_id = LOCK_COMMAND_ID;
} else if (command_size == UNLOCK_COMMAND_SIZE &&
!bcmp(UNLOCK_COMMAND, command, UNLOCK_COMMAND_SIZE)) {
command_id = UNLOCK_COMMAND_ID;
} else {
LOG_ERROR("handle_req: unknown command\n");
pcma_send(socket, failed_packfn, "unknown command");
return (-4);
}
switch (command_id) {
case PING_COMMAND_ID:
case LIST_COMMAND_ID:
if (obj.via.array.size != 1) {
LOG_ERROR("handle_req: no parameter expected\n");
pcma_send(socket, failed_packfn, "no parameter expected");
return (-5);
}
break;
case LOCK_COMMAND_ID:
case UNLOCK_COMMAND_ID:
if (obj.via.array.size != 2) {
LOG_ERROR("handle_req: 1 parameter expected\n");
pcma_send(socket, failed_packfn, "1 parameter expected");
return (-6);
}
if (obj.via.array.ptr[1].type != MSGPACK_OBJECT_RAW) {
LOG_ERROR("handle_req: RAW parameter expected\n");
pcma_send(socket, failed_packfn, "RAW parameter expected");
return (-7);
}
path = raw_to_string(&obj.via.array.ptr[1].via.raw);
if (!path) {
perror("raw_to_string");
pcma_send(socket, failed_packfn, "raw_to_string failure");
}
}
switch (command_id) {
case PING_COMMAND_ID:
handle_ping_request(socket);
break;
case LIST_COMMAND_ID:
handle_list_request(socket);
break;
case LOCK_COMMAND_ID:
handle_lock_request(socket, path);
break;
case UNLOCK_COMMAND_ID:
handle_unlock_request(socket, path);
break;
}
msgpack_unpacked_destroy(&pack);
if (path)
free(path);
return 0;
}
/*
* Convert the internal Fluent Bit data representation to the required
* one by Elasticsearch.
*
* 'Sadly' this process involves to convert from Msgpack to JSON.
*/
static char *es_format(void *data, size_t bytes, int *out_size,
struct flb_out_es_config *ctx)
{
int i;
int ret;
int n_size;
int index_len;
uint32_t psize;
size_t off = 0;
time_t atime;
char *buf;
char *ptr_key = NULL;
char *ptr_val = NULL;
char buf_key[256];
char buf_val[512];
msgpack_unpacked result;
msgpack_object root;
msgpack_object map;
char *j_entry;
char j_index[ES_BULK_HEADER];
json_t *j_map;
struct es_bulk *bulk;
/* Iterate the original buffer and perform adjustments */
msgpack_unpacked_init(&result);
/* Perform some format validation */
ret = msgpack_unpack_next(&result, data, bytes, &off);
if (!ret) {
return NULL;
}
/* We 'should' get an array */
if (result.data.type != MSGPACK_OBJECT_ARRAY) {
/*
* If we got a different format, we assume the caller knows what he is
* doing, we just duplicate the content in a new buffer and cleanup.
*/
return NULL;
}
root = result.data;
if (root.via.array.size == 0) {
return NULL;
}
/* Create the bulk composer */
bulk = es_bulk_create();
if (!bulk) {
return NULL;
}
/* Format the JSON header required by the ES Bulk API */
index_len = snprintf(j_index,
ES_BULK_HEADER,
ES_BULK_INDEX_FMT,
ctx->index, ctx->type);
off = 0;
msgpack_unpacked_destroy(&result);
msgpack_unpacked_init(&result);
while (msgpack_unpack_next(&result, data, bytes, &off)) {
if (result.data.type != MSGPACK_OBJECT_ARRAY) {
continue;
}
/* Each array must have two entries: time and record */
root = result.data;
if (root.via.array.size != 2) {
continue;
}
/* Create a map entry */
j_map = json_create_object();
atime = root.via.array.ptr[0].via.u64;
map = root.via.array.ptr[1];
n_size = map.via.map.size + 1;
json_add_to_object(j_map, "date", json_create_number(atime));
for (i = 0; i < n_size - 1; i++) {
msgpack_object *k = &map.via.map.ptr[i].key;
msgpack_object *v = &map.via.map.ptr[i].val;
if (k->type != MSGPACK_OBJECT_BIN && k->type != MSGPACK_OBJECT_STR) {
continue;
}
/* Store key */
psize = k->via.bin.size;
if (psize <= (sizeof(buf_key) - 1)) {
memcpy(buf_key, k->via.bin.ptr, psize);
buf_key[psize] = '\0';
ptr_key = buf_key;
}
else {
/* Long JSON map keys have a performance penalty */
ptr_key = flb_malloc(psize + 1);
memcpy(ptr_key, k->via.bin.ptr, psize);
ptr_key[psize] = '\0';
}
/*
* Sanitize key name, Elastic Search 2.x don't allow dots
* in field names:
*
* https://goo.gl/R5NMTr
*/
char *p = ptr_key;
char *end = ptr_key + psize;
while (p != end) {
if (*p == '.') *p = '_';
p++;
}
/* Store value */
if (v->type == MSGPACK_OBJECT_NIL) {
json_add_to_object(j_map, ptr_key, json_create_null());
}
else if (v->type == MSGPACK_OBJECT_BOOLEAN) {
json_add_to_object(j_map, ptr_key,
json_create_bool(v->via.boolean));
}
else if (v->type == MSGPACK_OBJECT_POSITIVE_INTEGER) {
json_add_to_object(j_map, ptr_key,
json_create_number(v->via.u64));
}
else if (v->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) {
json_add_to_object(j_map, ptr_key,
json_create_number(v->via.i64));
}
else if (v->type == MSGPACK_OBJECT_FLOAT) {
json_add_to_object(j_map, ptr_key,
json_create_number(v->via.f64));
}
else if (v->type == MSGPACK_OBJECT_STR) {
/* String value */
psize = v->via.str.size;
if (psize <= (sizeof(buf_val) - 1)) {
memcpy(buf_val, v->via.str.ptr, psize);
buf_val[psize] = '\0';
ptr_val = buf_val;
}
else {
ptr_val = flb_malloc(psize + 1);
memcpy(ptr_val, k->via.str.ptr, psize);
ptr_val[psize] = '\0';
}
json_add_to_object(j_map, ptr_key,
json_create_string(ptr_val));
}
else if (v->type == MSGPACK_OBJECT_BIN) {
/* Bin value */
psize = v->via.bin.size;
if (psize <= (sizeof(buf_val) - 1)) {
memcpy(buf_val, v->via.bin.ptr, psize);
buf_val[psize] = '\0';
ptr_val = buf_val;
}
else {
ptr_val = flb_malloc(psize + 1);
memcpy(ptr_val, k->via.bin.ptr, psize);
ptr_val[psize] = '\0';
}
json_add_to_object(j_map, ptr_key,
json_create_string(ptr_val));
}
if (ptr_key && ptr_key != buf_key) {
flb_free(ptr_key);
}
ptr_key = NULL;
if (ptr_val && ptr_val != buf_val) {
flb_free(ptr_val);
}
ptr_val = NULL;
}
/*
* At this point we have our JSON message, but in order to
* ingest this data into Elasticsearch we need to compose the
* Bulk API request, sadly it requires to prepend a JSON entry
* with details about the target 'index' and 'type' for EVERY
* message.
*/
j_entry = json_print_unformatted(j_map);
json_delete(j_map);
ret = es_bulk_append(bulk,
j_index, index_len,
j_entry, strlen(j_entry));
flb_free(j_entry);
if (ret == -1) {
/* We likely ran out of memory, abort here */
msgpack_unpacked_destroy(&result);
*out_size = 0;
es_bulk_destroy(bulk);
return NULL;
}
}
msgpack_unpacked_destroy(&result);
*out_size = bulk->len;
buf = bulk->ptr;
/*
* Note: we don't destroy the bulk as we need to keep the allocated
* buffer with the data. Instead we just release the bulk context and
* return the bulk->ptr buffer
*/
flb_free(bulk);
return buf;
}
