INLINE json_t prf_cmd_createkeyvalue_b(const char* key, int value)
{
json_t p = json_create_obj();
json_additem_toobj(p, "key", json_create_str(key));
json_additem_toobj(p, "value", json_create_bool(value));
return p;
}
/*************************************************************************************************
* inlines
*/
json_t prf_cmd_createkeyvalue_n(const char* key, fl64 value, int isgraph)
{
json_t p = json_create_obj();
json_additem_toobj(p, "key", json_create_str(key));
json_additem_toobj(p, "value", json_create_num(value));
if (isgraph)
json_additem_toobj(p, "isgraph", json_create_bool(TRUE));
return p;
}
/*
* Convert the internal Fluent Bit data representation to the required
* one by Elasticsearch.
*
* 'Sadly' this process involves to convert from Msgpack to JSON.
*/
static char *es_format(void *data, size_t bytes, int *out_size,
struct flb_out_es_config *ctx)
{
int i;
int ret;
int n_size;
int index_len;
uint32_t psize;
size_t off = 0;
time_t atime;
char *buf;
char *ptr_key = NULL;
char *ptr_val = NULL;
char buf_key[256];
char buf_val[512];
msgpack_unpacked result;
msgpack_object root;
msgpack_object map;
char *j_entry;
char j_index[ES_BULK_HEADER];
json_t *j_map;
struct es_bulk *bulk;
/* Iterate the original buffer and perform adjustments */
msgpack_unpacked_init(&result);
/* Perform some format validation */
ret = msgpack_unpack_next(&result, data, bytes, &off);
if (!ret) {
return NULL;
}
/* We 'should' get an array */
if (result.data.type != MSGPACK_OBJECT_ARRAY) {
/*
* If we got a different format, we assume the caller knows what he is
* doing, we just duplicate the content in a new buffer and cleanup.
*/
return NULL;
}
root = result.data;
if (root.via.array.size == 0) {
return NULL;
}
/* Create the bulk composer */
bulk = es_bulk_create();
if (!bulk) {
return NULL;
}
/* Format the JSON header required by the ES Bulk API */
index_len = snprintf(j_index,
ES_BULK_HEADER,
ES_BULK_INDEX_FMT,
ctx->index, ctx->type);
off = 0;
msgpack_unpacked_destroy(&result);
msgpack_unpacked_init(&result);
while (msgpack_unpack_next(&result, data, bytes, &off)) {
if (result.data.type != MSGPACK_OBJECT_ARRAY) {
continue;
}
/* Each array must have two entries: time and record */
root = result.data;
if (root.via.array.size != 2) {
continue;
}
/* Create a map entry */
j_map = json_create_object();
atime = root.via.array.ptr[0].via.u64;
map = root.via.array.ptr[1];
n_size = map.via.map.size + 1;
json_add_to_object(j_map, "date", json_create_number(atime));
for (i = 0; i < n_size - 1; i++) {
msgpack_object *k = &map.via.map.ptr[i].key;
msgpack_object *v = &map.via.map.ptr[i].val;
if (k->type != MSGPACK_OBJECT_BIN && k->type != MSGPACK_OBJECT_STR) {
continue;
}
/* Store key */
psize = k->via.bin.size;
if (psize <= (sizeof(buf_key) - 1)) {
memcpy(buf_key, k->via.bin.ptr, psize);
buf_key[psize] = '\0';
ptr_key = buf_key;
}
else {
/* Long JSON map keys have a performance penalty */
ptr_key = flb_malloc(psize + 1);
memcpy(ptr_key, k->via.bin.ptr, psize);
ptr_key[psize] = '\0';
}
/*
* Sanitize key name, Elastic Search 2.x don't allow dots
* in field names:
*
* https://goo.gl/R5NMTr
*/
char *p = ptr_key;
char *end = ptr_key + psize;
while (p != end) {
if (*p == '.') *p = '_';
p++;
}
/* Store value */
if (v->type == MSGPACK_OBJECT_NIL) {
json_add_to_object(j_map, ptr_key, json_create_null());
}
else if (v->type == MSGPACK_OBJECT_BOOLEAN) {
json_add_to_object(j_map, ptr_key,
json_create_bool(v->via.boolean));
}
else if (v->type == MSGPACK_OBJECT_POSITIVE_INTEGER) {
json_add_to_object(j_map, ptr_key,
json_create_number(v->via.u64));
}
else if (v->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) {
json_add_to_object(j_map, ptr_key,
json_create_number(v->via.i64));
}
else if (v->type == MSGPACK_OBJECT_FLOAT) {
json_add_to_object(j_map, ptr_key,
json_create_number(v->via.f64));
}
else if (v->type == MSGPACK_OBJECT_STR) {
/* String value */
psize = v->via.str.size;
if (psize <= (sizeof(buf_val) - 1)) {
memcpy(buf_val, v->via.str.ptr, psize);
buf_val[psize] = '\0';
ptr_val = buf_val;
}
else {
ptr_val = flb_malloc(psize + 1);
memcpy(ptr_val, k->via.str.ptr, psize);
ptr_val[psize] = '\0';
}
json_add_to_object(j_map, ptr_key,
json_create_string(ptr_val));
}
else if (v->type == MSGPACK_OBJECT_BIN) {
/* Bin value */
psize = v->via.bin.size;
if (psize <= (sizeof(buf_val) - 1)) {
memcpy(buf_val, v->via.bin.ptr, psize);
buf_val[psize] = '\0';
ptr_val = buf_val;
}
else {
ptr_val = flb_malloc(psize + 1);
memcpy(ptr_val, k->via.bin.ptr, psize);
ptr_val[psize] = '\0';
}
json_add_to_object(j_map, ptr_key,
json_create_string(ptr_val));
}
if (ptr_key && ptr_key != buf_key) {
flb_free(ptr_key);
}
ptr_key = NULL;
if (ptr_val && ptr_val != buf_val) {
flb_free(ptr_val);
}
ptr_val = NULL;
}
/*
* At this point we have our JSON message, but in order to
* ingest this data into Elasticsearch we need to compose the
* Bulk API request, sadly it requires to prepend a JSON entry
* with details about the target 'index' and 'type' for EVERY
* message.
*/
j_entry = json_print_unformatted(j_map);
json_delete(j_map);
ret = es_bulk_append(bulk,
j_index, index_len,
j_entry, strlen(j_entry));
flb_free(j_entry);
if (ret == -1) {
/* We likely ran out of memory, abort here */
msgpack_unpacked_destroy(&result);
*out_size = 0;
es_bulk_destroy(bulk);
return NULL;
}
}
msgpack_unpacked_destroy(&result);
*out_size = bulk->len;
buf = bulk->ptr;
/*
* Note: we don't destroy the bulk as we need to keep the allocated
* buffer with the data. Instead we just release the bulk context and
* return the bulk->ptr buffer
*/
flb_free(bulk);
return buf;
}
