struct json *json_create_double_array(double *numbers, int count)
{
int i;
struct json *n = 0, *p = 0, *a = json_create_array();
for (i = 0; a && i < count; i++) {
n = json_create_number(numbers[i]);
if (!i)
a->child = n;
else
suffix_object(p, n);
p = n;
}
return a;
}
static int invoke_procedure_id(struct jrpc_server *server, struct json *method,
struct jrpc_connection *conn, struct json *id,
struct json *params)
{
//We have to copy ID because using it on the reply and deleting the response Object will also delete ID
struct json *id_copy = NULL;
if (id != NULL)
id_copy = (id->type == JSON_T_STRING)
? json_create_string(id->valuestring)
: json_create_number(id->valueint);
if (server->debug_level)
dlog("Method Invoked: %s\n", method->valuestring);
return invoke_procedure(server, conn, method->valuestring, params,
id_copy);
}
struct json_val *json_parse_number(struct read_state *state)
{
char dbl_buf[30];
char *write = dbl_buf;
char ch = *state->read++;
if (ch == '-') {
*write++ = '-';
}
if (!read_state_get(state, &ch))
goto fail;
if (ch == '0') {
//pass
} else if (ch >= '1' && ch <= '9') {
*write++ = ch;
while (true) {
if (!read_state_get(state, &ch))
goto calc_value;
if (ch < '0' || ch > '9') {
read_state_put_back(state);
break;
}
*write++ = ch;
}
} else {
json_error_print(state, "Invalid character found parsing number\n");
goto fail;
}
if (read_state_get(state, &ch)) {
if (ch == '.') {
*write++ = '.';
if (!read_state_get(state, &ch)) {
json_error_print(state, "Expected digit after decimal\n");
goto fail;
}
if(ch >= '0' && ch <= '9') {
*write++ = ch;
} else {
json_error_print(state, "Invalid character found after decimal\n");
goto fail;
}
while (true) {
if (!read_state_get(state, &ch)) {
goto calc_value;
}
if(ch >= '0' && ch <= '9') {
*write++ = ch;
} else {
read_state_put_back(state);
break;
}
}
}
else {
read_state_put_back(state);
}
}
if (read_state_get(state, &ch)) {
if (ch == 'e' || ch == 'E') {
*write++ = 'e';
if (read_state_get(state, &ch)) {
if (ch == '+' || ch == '-') {
*write++ = ch;
} else {
read_state_put_back(state);
}
}
if (!read_state_get(state, &ch)) {
json_error_print(state, "Expected digit in exponent\n");
goto fail;
} else if (ch < '0' || ch > '9') {
json_error_print(state, "Invalid character in exponent\n");
} else {
*write++ = ch;
while (true) {
if (!read_state_get(state, &ch)) {
goto calc_value;
}
if(ch >= '0' && ch <= '9') {
*write++ = ch;
} else {
read_state_put_back(state);
break;
}
}
}
}
else {
read_state_put_back(state);
}
}
calc_value:
*write++ = '\0';
return json_create_number(strtod(dbl_buf, 0));
fail:
return 0;
}
/*
* Convert the internal Fluent Bit data representation to the required
* one by Elasticsearch.
*
* 'Sadly' this process involves to convert from Msgpack to JSON.
*/
static char *es_format(void *data, size_t bytes, int *out_size,
struct flb_out_es_config *ctx)
{
int i;
int ret;
int n_size;
int index_len;
uint32_t psize;
size_t off = 0;
time_t atime;
char *buf;
char *ptr_key = NULL;
char *ptr_val = NULL;
char buf_key[256];
char buf_val[512];
msgpack_unpacked result;
msgpack_object root;
msgpack_object map;
char *j_entry;
char j_index[ES_BULK_HEADER];
json_t *j_map;
struct es_bulk *bulk;
/* Iterate the original buffer and perform adjustments */
msgpack_unpacked_init(&result);
/* Perform some format validation */
ret = msgpack_unpack_next(&result, data, bytes, &off);
if (!ret) {
return NULL;
}
/* We 'should' get an array */
if (result.data.type != MSGPACK_OBJECT_ARRAY) {
/*
* If we got a different format, we assume the caller knows what he is
* doing, we just duplicate the content in a new buffer and cleanup.
*/
return NULL;
}
root = result.data;
if (root.via.array.size == 0) {
return NULL;
}
/* Create the bulk composer */
bulk = es_bulk_create();
if (!bulk) {
return NULL;
}
/* Format the JSON header required by the ES Bulk API */
index_len = snprintf(j_index,
ES_BULK_HEADER,
ES_BULK_INDEX_FMT,
ctx->index, ctx->type);
off = 0;
msgpack_unpacked_destroy(&result);
msgpack_unpacked_init(&result);
while (msgpack_unpack_next(&result, data, bytes, &off)) {
if (result.data.type != MSGPACK_OBJECT_ARRAY) {
continue;
}
/* Each array must have two entries: time and record */
root = result.data;
if (root.via.array.size != 2) {
continue;
}
/* Create a map entry */
j_map = json_create_object();
atime = root.via.array.ptr[0].via.u64;
map = root.via.array.ptr[1];
n_size = map.via.map.size + 1;
json_add_to_object(j_map, "date", json_create_number(atime));
for (i = 0; i < n_size - 1; i++) {
msgpack_object *k = &map.via.map.ptr[i].key;
msgpack_object *v = &map.via.map.ptr[i].val;
if (k->type != MSGPACK_OBJECT_BIN && k->type != MSGPACK_OBJECT_STR) {
continue;
}
/* Store key */
psize = k->via.bin.size;
if (psize <= (sizeof(buf_key) - 1)) {
memcpy(buf_key, k->via.bin.ptr, psize);
buf_key[psize] = '\0';
ptr_key = buf_key;
}
else {
/* Long JSON map keys have a performance penalty */
ptr_key = flb_malloc(psize + 1);
memcpy(ptr_key, k->via.bin.ptr, psize);
ptr_key[psize] = '\0';
}
/*
* Sanitize key name, Elastic Search 2.x don't allow dots
* in field names:
*
* https://goo.gl/R5NMTr
*/
char *p = ptr_key;
char *end = ptr_key + psize;
while (p != end) {
if (*p == '.') *p = '_';
p++;
}
/* Store value */
if (v->type == MSGPACK_OBJECT_NIL) {
json_add_to_object(j_map, ptr_key, json_create_null());
}
else if (v->type == MSGPACK_OBJECT_BOOLEAN) {
json_add_to_object(j_map, ptr_key,
json_create_bool(v->via.boolean));
}
else if (v->type == MSGPACK_OBJECT_POSITIVE_INTEGER) {
json_add_to_object(j_map, ptr_key,
json_create_number(v->via.u64));
}
else if (v->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) {
json_add_to_object(j_map, ptr_key,
json_create_number(v->via.i64));
}
else if (v->type == MSGPACK_OBJECT_FLOAT) {
json_add_to_object(j_map, ptr_key,
json_create_number(v->via.f64));
}
else if (v->type == MSGPACK_OBJECT_STR) {
/* String value */
psize = v->via.str.size;
if (psize <= (sizeof(buf_val) - 1)) {
memcpy(buf_val, v->via.str.ptr, psize);
buf_val[psize] = '\0';
ptr_val = buf_val;
}
else {
ptr_val = flb_malloc(psize + 1);
memcpy(ptr_val, k->via.str.ptr, psize);
ptr_val[psize] = '\0';
}
json_add_to_object(j_map, ptr_key,
json_create_string(ptr_val));
}
else if (v->type == MSGPACK_OBJECT_BIN) {
/* Bin value */
psize = v->via.bin.size;
if (psize <= (sizeof(buf_val) - 1)) {
memcpy(buf_val, v->via.bin.ptr, psize);
buf_val[psize] = '\0';
ptr_val = buf_val;
}
else {
ptr_val = flb_malloc(psize + 1);
memcpy(ptr_val, k->via.bin.ptr, psize);
ptr_val[psize] = '\0';
}
json_add_to_object(j_map, ptr_key,
json_create_string(ptr_val));
}
if (ptr_key && ptr_key != buf_key) {
flb_free(ptr_key);
}
ptr_key = NULL;
if (ptr_val && ptr_val != buf_val) {
flb_free(ptr_val);
}
ptr_val = NULL;
}
/*
* At this point we have our JSON message, but in order to
* ingest this data into Elasticsearch we need to compose the
* Bulk API request, sadly it requires to prepend a JSON entry
* with details about the target 'index' and 'type' for EVERY
* message.
*/
j_entry = json_print_unformatted(j_map);
json_delete(j_map);
ret = es_bulk_append(bulk,
j_index, index_len,
j_entry, strlen(j_entry));
flb_free(j_entry);
if (ret == -1) {
/* We likely ran out of memory, abort here */
msgpack_unpacked_destroy(&result);
*out_size = 0;
es_bulk_destroy(bulk);
return NULL;
}
}
msgpack_unpacked_destroy(&result);
*out_size = bulk->len;
buf = bulk->ptr;
/*
* Note: we don't destroy the bulk as we need to keep the allocated
* buffer with the data. Instead we just release the bulk context and
* return the bulk->ptr buffer
*/
flb_free(bulk);
return buf;
}
