/* Creates a new upstream context */
struct flb_io_upstream *flb_io_upstream_new(char *host, int port, int flags)
{
int fd;
struct flb_io_upstream *u;
u = malloc(sizeof(struct flb_io_upstream));
if (!u) {
perror("malloc");
return NULL;
}
/* Upon upstream creation, we always try to perform a connection */
flb_debug("[upstream] connecting to %s:%i", host, port);
fd = flb_net_tcp_connect(host, port);
if (fd == -1) {
flb_warn("[upstream] could not connect to %s:%i", host, port);
}
else {
flb_debug("[upstream] connected!");
}
u->fd = fd;
u->tcp_host = strdup(host);
u->tcp_port = port;
u->flags = flags;
return u;
}
/* Write data to an upstream connection/server */
int flb_io_net_write(struct flb_io_upstream *u, void *data,
size_t len, size_t *out_len)
{
int ret = -1;
struct flb_thread *th;
flb_debug("[io] trying to write %zd bytes", len);
th = pthread_getspecific(flb_thread_key);
if (u->flags & FLB_IO_TCP) {
ret = net_io_write(th, u, data, len, out_len);
}
#ifdef HAVE_TLS
else if (u->flags & FLB_IO_TLS) {
ret = net_io_tls_write(th, u, data, len, out_len);
}
#endif
if (ret == -1 && u->fd > 0) {
close(u->fd);
u->fd = -1;
}
flb_debug("[io] thread has finished");
return ret;
}
int cb_td_flush(void *data, size_t bytes, void *out_context,
struct flb_config *config)
{
int n;
int ret;
int bytes_out;
char *pack;
size_t bytes_sent;
char buf[1024];
size_t len;
char *request;
struct flb_out_td_config *ctx = out_context;
/* Convert format */
pack = td_format(data, bytes, &bytes_out);
if (!pack) {
return -1;
}
request = td_http_request(pack, bytes_out, &len, ctx, config);
ret = flb_io_net_write(ctx->u, request, len, &bytes_sent);
if (ret == -1) {
perror("write");
}
free(request);
free(pack);
n = flb_io_net_read(ctx->u, buf, sizeof(buf) - 1);
if (n > 0) {
buf[n] = '\0';
flb_debug("[TD] API server response:\n%s", buf);
}
return bytes_sent;
}
struct flb_in_serial_config *serial_config_read(struct flb_in_serial_config *config,
struct mk_rconf *conf)
{
char *file;
char *bitrate;
struct mk_rconf_section *section;
section = mk_rconf_section_get(conf, "serial");
if (!section) {
return NULL;
}
/* Validate serial section keys */
file = mk_rconf_section_get_key(section, "file", MK_RCONF_STR);
bitrate = mk_rconf_section_get_key(section, "bitrate", MK_RCONF_STR);
if (!file) {
flb_utils_error_c("[serial] error reading filename from "
"configuration");
}
if (!bitrate) {
flb_utils_error_c("[serial] error reading bitrate from "
"configuration");
}
config->fd = -1;
config->file = file;
config->bitrate = bitrate;
flb_debug("Serial / file='%s' bitrate='%s'",
config->file, config->bitrate);
return config;
}
int cb_td_flush(void *data, size_t bytes, void *out_context,
struct flb_config *config)
{
int n;
char buf[1024];
ssize_t w_bytes;
size_t out_len;
char *request;
struct flb_out_td_config *ctx = out_context;
request = td_http_request(data, bytes, &out_len, ctx, config);
w_bytes = write(ctx->fd, request, out_len);
if (w_bytes < 0) {
perror("write");
/* FIXME: handle connection timeout */
if (errno == EBADF) {
close(ctx->fd);
ctx->fd = flb_net_tcp_connect(out_td_plugin.host,
out_td_plugin.port);
}
}
free(request);
n = read(ctx->fd, buf, 4096);
buf[n] = '\0';
flb_debug("[TD] API server response:\n%s", buf);
return w_bytes;
}
/* Cleanup serial input */
int in_serial_exit(void *in_context, struct flb_config *config)
{
struct flb_in_serial_config *ctx = in_context;
flb_debug("[in_serial] Restoring original termios...");
tcsetattr(ctx->fd, TCSANOW, &ctx->tio_orig);
}
FLB_INLINE int net_io_tls_write(struct flb_thread *th, struct flb_io_upstream *u,
void *data, size_t len, size_t *out_len)
{
int ret;
size_t total = 0;
if (!u->tls_session) {
u->tls_session = flb_tls_session_new(u->tls->context);
if (!u->tls_session) {
flb_error("[io_tls] could not create tls session");
return -1;
}
ret = flb_io_net_tls_connect(u, th);
if (ret == -1) {
flb_error("[io_tls] could not connect/initiate TLS session");
return -1;
}
}
retry_write:
ret = mbedtls_ssl_write(&u->tls_session->ssl,
data + total,
len - total);
if (ret == MBEDTLS_ERR_SSL_WANT_WRITE) {
io_tls_event_switch(u, MK_EVENT_WRITE);
flb_thread_yield(th, FLB_FALSE);
goto retry_write;
}
else if (ret == MBEDTLS_ERR_SSL_WANT_READ) {
io_tls_event_switch(u, MK_EVENT_READ);
flb_thread_yield(th, FLB_FALSE);
goto retry_write;
}
else if (ret < 0) {
char err_buf[72];
mbedtls_strerror(ret, err_buf, sizeof(err_buf));
flb_debug("[tls] SSL error: %s", err_buf);
/* There was an error transmitting data */
mk_event_del(u->evl, &u->event);
tls_session_destroy(u->tls_session);
u->tls_session = NULL;
return -1;
}
/* Update statistics */
//flb_stats_update(out->stats_fd, ret, 0);
/* Update counter and check if we need to continue writing */
total += ret;
if (total < len) {
io_tls_event_switch(u, MK_EVENT_WRITE);
flb_thread_yield(th, FLB_FALSE);
goto retry_write;
}
mk_event_del(u->evl, &u->event);
return 0;
}
void in_xbee_cb(struct xbee *xbee, struct xbee_con *con,
struct xbee_pkt **pkt, void **data)
{
struct iovec *v;
struct flb_in_xbee_config *ctx;
if ((*pkt)->dataLen == 0) {
flb_debug("xbee data length too short, skip");
return;
}
ctx = *data;
if (ctx->buffer_len + 1 >= FLB_XBEE_BUFFER_SIZE) {
/* fixme: use flb_engine_flush() */
return;
}
/* Insert entry into the iovec */
v = &ctx->buffer[ctx->buffer_len];
v->iov_base = malloc((*pkt)->dataLen);
memcpy(v->iov_base, (*pkt)->data, (*pkt)->dataLen);
v->iov_len = (*pkt)->dataLen;
ctx->buffer_len++;
}
int in_mem_collect(struct flb_config *config, void *in_context)
{
struct sysinfo info;
(void) config;
struct flb_in_mem_config *ctx = in_context;
uint32_t totalram, freeram;
sysinfo(&info);
totalram = info.totalram / 1024;
freeram = info.freeram / 1024;
msgpack_pack_map(&ctx->pckr, 3);
msgpack_pack_raw(&ctx->pckr, 4);
msgpack_pack_raw_body(&ctx->pckr, "time", 4);
msgpack_pack_uint64(&ctx->pckr, time(NULL));
msgpack_pack_raw(&ctx->pckr, 5);
msgpack_pack_raw_body(&ctx->pckr, "total", 5);
msgpack_pack_uint32(&ctx->pckr, totalram);
msgpack_pack_raw(&ctx->pckr, 4);
msgpack_pack_raw_body(&ctx->pckr, "free", 4);
msgpack_pack_uint32(&ctx->pckr, freeram);
flb_debug("[in_mem] memory total %d kb, free %d kb (buffer=%i)",
info.totalram,
info.freeram,
ctx->idx);
++ctx->idx;
return 0;
}
static int json_tokenise(char *js, size_t len,
struct flb_pack_state *state)
{
int ret;
int n;
void *tmp;
ret = jsmn_parse(&state->parser, js, len,
state->tokens, state->tokens_size);
while (ret == JSMN_ERROR_NOMEM) {
n = state->tokens_size += 256;
tmp = realloc(state->tokens, sizeof(jsmntok_t) * n);
if (!tmp) {
perror("realloc");
return -1;
}
state->tokens = tmp;
state->tokens_size = n;
ret = jsmn_parse(&state->parser, js, len,
state->tokens, state->tokens_size);
}
if (ret == JSMN_ERROR_INVAL) {
return FLB_ERR_JSON_INVAL;
}
if (ret == JSMN_ERROR_PART) {
/* This is a partial JSON message, just stop */
flb_debug("[json tokenise] incomplete");
return FLB_ERR_JSON_PART;
}
state->tokens_count += ret;
return 0;
}
static inline int process_line(char *line, struct flb_in_serial_config *ctx)
{
int line_len;
char *p = line;
char *end = NULL;
char msg[1024];
/* Increase buffer position */
ctx->buffer_id++;
line_len = strlen(p);
strncpy(msg, p, line_len);
msg[line_len] = '\0';
/*
* Store the new data into the MessagePack buffer,
* we handle this as a list of maps.
*/
msgpack_pack_array(&ctx->mp_pck, 2);
msgpack_pack_uint64(&ctx->mp_pck, time(NULL));
msgpack_pack_map(&ctx->mp_pck, 1);
msgpack_pack_bin(&ctx->mp_pck, 3);
msgpack_pack_bin_body(&ctx->mp_pck, "msg", 3);
msgpack_pack_bin(&ctx->mp_pck, line_len);
msgpack_pack_bin_body(&ctx->mp_pck, p, line_len);
flb_debug("[in_serial] '%s'",
(const char *) msg);
return 0;
}
ssize_t flb_io_net_read(struct flb_io_upstream *u, void *buf, size_t len)
{
int ret = -1;
struct flb_thread *th;
flb_debug("[io] trying to read up to %zd bytes", len);
th = pthread_getspecific(flb_thread_key);
if (u->flags & FLB_IO_TCP) {
ret = net_io_read(th, u, buf, len);
}
#ifdef HAVE_TLS
else if (u->flags & FLB_IO_TLS) {
ret = net_io_tls_read(th, u, buf, len);
}
#endif
flb_debug("[io] thread has finished");
return ret;
}
/* Callback to gather CPU usage between now and previous snapshot */
int in_cpu_collect(struct flb_config *config, void *in_context)
{
int i;
int maps;
int len;
double usage;
double total;
(void) config;
struct flb_in_cpu_config *ctx = in_context;
struct cpu_stats *cstats = &ctx->cstats;
struct cpu_snapshot *s;
/* Get the current CPU usage */
total = proc_cpu_load(ctx->n_processors, cstats);
cstats->load_now = total;
/* Calculate the difference between the two samples */
usage = fabs(cstats->load_now - cstats->load_pre) / ctx->cpu_ticks;
total = (usage * 100) / ctx->n_processors;
/* Put current load back */
cstats->load_pre = cstats->load_now;
/*
* Store the new data into the MessagePack buffer,
*/
msgpack_pack_array(&ctx->mp_pck, 2);
msgpack_pack_uint64(&ctx->mp_pck, time(NULL));
msgpack_pack_map(&ctx->mp_pck, (ctx->n_processors * 7 ) + 1);
msgpack_pack_bin(&ctx->mp_pck, 3);
msgpack_pack_bin_body(&ctx->mp_pck, "cpu", 3);
msgpack_pack_double(&ctx->mp_pck, total);
for (i = 1; i < ctx->n_processors + 1; i++) {
s = &cstats->info[i];
CPU_PACK_SNAP(s, user);
CPU_PACK_SNAP(s, nice);
CPU_PACK_SNAP(s, system);
CPU_PACK_SNAP(s, idle);
CPU_PACK_SNAP(s, iowait);
CPU_PACK_SNAP(s, irq);
CPU_PACK_SNAP(s, softirq);
}
flb_debug("[in_cpu] CPU %0.2f%%", total);
flb_stats_update(in_cpu_plugin.stats_fd, 0, 1);
return 0;
}
/* Callback triggered by timer */
int in_xbee_collect(struct flb_config *config, void *in_context)
{
int ret;
void *p = NULL;
(void) config;
struct flb_in_xbee_config *ctx = in_context;
if ((ret = xbee_conCallbackGet(ctx->con,
(xbee_t_conCallback*) &p)) != XBEE_ENONE) {
flb_debug("xbee_conCallbackGet() returned: %d", ret);
return ret;
}
return 0;
}
static inline int process_line(char *line, struct flb_in_serial_config *ctx)
{
int line_len;
uint64_t val;
char *p = line;
char *end = NULL;
char msg[1024];
/* Increase buffer position */
ctx->buffer_id++;
errno = 0;
val = strtol(p, &end, 10);
if ((errno == ERANGE && (val == INT_MAX || val == INT_MIN))
|| (errno != 0 && val == 0)) {
goto fail;
}
/* Now process the human readable message */
line_len = strlen(p);
strncpy(msg, p, line_len);
msg[line_len] = '\0';
/*
* Store the new data into the MessagePack buffer,
* we handle this as a list of maps.
*/
msgpack_pack_array(&ctx->mp_pck, 2);
msgpack_pack_uint64(&ctx->mp_pck, time(NULL));
msgpack_pack_map(&ctx->mp_pck, 1);
msgpack_pack_bin(&ctx->mp_pck, 3);
msgpack_pack_bin_body(&ctx->mp_pck, "msg", 3);
msgpack_pack_bin(&ctx->mp_pck, line_len);
msgpack_pack_bin_body(&ctx->mp_pck, p, line_len);
flb_debug("[in_serial] '%s'",
(const char *) msg);
return 0;
fail:
ctx->buffer_id--;
return -1;
}
void cb_es_flush(void *data, size_t bytes,
char *tag, int tag_len,
struct flb_input_instance *i_ins, void *out_context,
struct flb_config *config)
{
int ret;
int bytes_out;
char *pack;
size_t b_sent;
struct flb_out_es_config *ctx = out_context;
struct flb_upstream_conn *u_conn;
struct flb_http_client *c;
(void) i_ins;
(void) tag;
(void) tag_len;
/* Convert format */
pack = es_format(data, bytes, &bytes_out, ctx);
if (!pack) {
FLB_OUTPUT_RETURN(FLB_ERROR);
}
/* Get upstream connection */
u_conn = flb_upstream_conn_get(ctx->u);
if (!u_conn) {
flb_free(pack);
FLB_OUTPUT_RETURN(FLB_RETRY);
}
/* Compose HTTP Client request */
c = flb_http_client(u_conn, FLB_HTTP_POST, "/_bulk",
pack, bytes_out, NULL, 0, NULL);
flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10);
flb_http_add_header(c, "Content-Type", 12, "application/json", 16);
ret = flb_http_do(c, &b_sent);
flb_debug("[out_es] http_do=%i", ret);
flb_http_client_destroy(c);
flb_free(pack);
/* Release the connection */
flb_upstream_conn_release(u_conn);
FLB_OUTPUT_RETURN(FLB_OK);
}
/* Initialize plugin */
int in_mqtt_init(struct flb_input_instance *in,
struct flb_config *config, void *data)
{
int ret;
struct flb_in_mqtt_config *ctx;
(void) data;
/* Allocate space for the configuration */
ctx = mqtt_config_init(in);
if (!ctx) {
return -1;
}
ctx->msgp_len = 0;
/* Set the context */
flb_input_set_context(in, ctx);
/* Create TCP server */
ctx->server_fd = flb_net_server(ctx->tcp_port, ctx->listen);
if (ctx->server_fd > 0) {
flb_debug("[in_mqtt] binding %s:%s", ctx->listen, ctx->tcp_port);
}
else {
flb_error("[in_mqtt] could not bind address %s:%s",
ctx->listen, ctx->tcp_port);
mqtt_config_free(ctx);
return -1;
}
ctx->evl = config->evl;
/* Collect upon data available on the standard input */
ret = flb_input_set_collector_event(in,
in_mqtt_collect,
ctx->server_fd,
config);
if (ret == -1) {
flb_error("[in_mqtt] Could not set collector for MQTT input plugin");
mqtt_config_free(ctx);
return -1;
}
return 0;
}
struct flb_out_td_config *td_config_init(struct mk_rconf *conf)
{
char *api;
char *db_name;
char *db_table;
struct mk_rconf_section *section;
struct flb_out_td_config *config;
section = mk_rconf_section_get(conf, "TD");
if (!section) {
return NULL;
}
/* Validate TD section keys */
api = mk_rconf_section_get_key(section, "API", MK_RCONF_STR);
db_name = mk_rconf_section_get_key(section, "Database", MK_RCONF_STR);
db_table = mk_rconf_section_get_key(section, "Table", MK_RCONF_STR);
if (!api) {
flb_utils_error_c("[TD] error reading API key value");
}
if (!db_name) {
flb_utils_error_c("[TD] error reading Database name");
}
if (!db_table) {
flb_utils_error_c("[TD] error reading Table name");
}
config = malloc(sizeof(struct flb_out_td_config));
config->fd = -1;
config->api = api;
config->db_name = db_name;
config->db_table = db_table;
flb_debug("TreasureData / database='%s' table='%s'",
config->db_name, config->db_table);
return config;
}
static inline int process_line(char *line, int len,
struct flb_in_serial_config *ctx)
{
/* Increase buffer position */
ctx->buffer_id++;
/*
* Store the new data into the MessagePack buffer,
* we handle this as a list of maps.
*/
msgpack_pack_array(&ctx->mp_pck, 2);
msgpack_pack_uint64(&ctx->mp_pck, time(NULL));
msgpack_pack_map(&ctx->mp_pck, 1);
msgpack_pack_bin(&ctx->mp_pck, 3);
msgpack_pack_bin_body(&ctx->mp_pck, "msg", 3);
msgpack_pack_bin(&ctx->mp_pck, len);
msgpack_pack_bin_body(&ctx->mp_pck, line, len);
flb_debug("[in_serial] message '%s'",
(const char *) line);
return 0;
}
void in_xbee_cb(struct xbee *xbee, struct xbee_con *con,
struct xbee_pkt **pkt, void **data)
{
struct flb_in_xbee_config *ctx;
if ((*pkt)->dataLen == 0) {
flb_debug("xbee data length too short, skip");
return;
}
ctx = *data;
#if 0
int i;
for (i = 0; i < (*pkt)->dataLen; i++) {
printf("%2.2x ", *((unsigned char*) (*pkt)->data + i));
}
printf("\n");
#endif
if (! in_xbee_rx_queue_msgpack(ctx, (const char*) (*pkt)->data, (*pkt)->dataLen)) {
in_xbee_rx_queue_raw(ctx, (const char*) (*pkt)->data, (*pkt)->dataLen);
}
}
static inline int process_line(char *line, struct flb_in_kmsg_config *ctx)
{
uint64_t val;
char *p = line;
char *end = NULL;
struct kmsg_line *buf;
/* Increase buffer position */
ctx->buffer_id++;
errno = 0;
val = strtol(p, &end, 10);
if ((errno == ERANGE && (val == INT_MAX || val == INT_MIN))
|| (errno != 0 && val == 0)) {
goto fail;
}
/* Lookup */
buf = &ctx->buffer[ctx->buffer_id];
/* Priority */
buf->priority = FLB_LOG_PRI(val);
/* Sequence */
p = strchr(p, ',');
p++;
val = strtol(p, &end, 10);
if ((errno == ERANGE && (val == INT_MAX || val == INT_MIN))
|| (errno != 0 && val == 0)) {
goto fail;
}
buf->sequence = val;
p = ++end;
/* Timestamp */
val = strtol(p, &end, 10);
if ((errno == ERANGE && (val == INT_MAX || val == INT_MIN))
|| (errno != 0 && val == 0)) {
goto fail;
}
buf->tv.tv_sec = val;
if (*end == '.') {
p = ++end;
val = strtol(p, &end, 10);
if ((errno == ERANGE && (val == INT_MAX || val == INT_MIN))
|| (errno != 0 && val == 0)) {
goto fail;
}
buf->tv.tv_usec = val;
}
else {
buf->tv.tv_usec = 0;
}
/* Now process the human readable message */
p = strchr(p, ';');
if (!p) {
goto fail;
}
p++;
buf->line_len = strlen(p);
strncpy(buf->line, p, buf->line_len);
buf->line[buf->line_len] = '\0';
flb_debug("[in_kmsg] pri=%i seq=%" PRIu64 " sec=%ld usec=%ld '%s'",
buf->priority,
buf->sequence,
(long int) buf->tv.tv_sec,
(long int) buf->tv.tv_usec,
(const char *) buf->line);
return 0;
fail:
ctx->buffer_id--;
return -1;
}
/* Callback triggered when some serial msgs are available */
int in_serial_collect(struct flb_config *config, void *in_context)
{
int ret;
int bytes = 0;
int available;
int len;
int hits;
char *sep;
char *buf;
jsmntok_t *t;
struct flb_in_serial_config *ctx = in_context;
while (1) {
available = (sizeof(ctx->buf_data) -1) - ctx->buf_len;
if (available > 1) {
bytes = read(ctx->fd, ctx->buf_data + ctx->buf_len, available);
if (bytes == -1) {
if (errno == EPIPE || errno == EINTR) {
return -1;
}
return 0;
}
else if (bytes == 0) {
return 0;
}
}
ctx->buf_len += bytes;
/* Always set a delimiter to avoid buffer trash */
ctx->buf_data[ctx->buf_len] = '\0';
/* Check if our buffer is full */
if (ctx->buffer_id + 1 == SERIAL_BUFFER_SIZE) {
ret = flb_engine_flush(config, &in_serial_plugin);
if (ret == -1) {
ctx->buffer_id = 0;
}
}
sep = NULL;
buf = ctx->buf_data;
len = ctx->buf_len;
hits = 0;
/* Handle FTDI handshake */
if (ctx->buf_data[0] == '\0') {
consume_bytes(ctx->buf_data, 1, ctx->buf_len);
ctx->buf_len--;
}
/* Strip CR or LF if found at first byte */
if (ctx->buf_data[0] == '\r' || ctx->buf_data[0] == '\n') {
/* Skip message with one byte with CR or LF */
flb_trace("[in_serial] skip one byte message with ASCII code=%i",
ctx->buf_data[0]);
consume_bytes(ctx->buf_data, 1, ctx->buf_len);
ctx->buf_len--;
}
/* Handle the case when a Separator is set */
if (ctx->separator) {
while ((sep = strstr(ctx->buf_data, ctx->separator))) {
len = (sep - ctx->buf_data);
if (len > 0) {
/* process the line based in the separator position */
process_line(buf, len, ctx);
consume_bytes(ctx->buf_data, len + ctx->sep_len, ctx->buf_len);
ctx->buf_len -= (len + ctx->sep_len);
hits++;
}
else {
consume_bytes(ctx->buf_data, ctx->sep_len, ctx->buf_len);
ctx->buf_len -= ctx->sep_len;
}
ctx->buf_data[ctx->buf_len] = '\0';
}
if (hits == 0 && available <= 1) {
flb_debug("[in_serial] no separator found, no more space");
ctx->buf_len = 0;
return 0;
}
}
else if (ctx->format == FLB_SERIAL_FORMAT_JSON) {
/* JSON Format handler */
char *pack;
int out_size;
ret = flb_pack_json_state(ctx->buf_data, ctx->buf_len,
&pack, &out_size, &ctx->pack_state);
if (ret == FLB_ERR_JSON_PART) {
flb_debug("[in_serial] JSON incomplete, waiting for more data...");
return 0;
}
else if (ret == FLB_ERR_JSON_INVAL) {
flb_debug("[in_serial] invalid JSON message, skipping");
flb_pack_state_reset(&ctx->pack_state);
flb_pack_state_init(&ctx->pack_state);
ctx->pack_state.multiple = FLB_TRUE;
return -1;
}
/*
* Given the Tokens used for the packaged message, append
* the records and then adjust buffer.
*/
process_pack(ctx, pack, out_size);
flb_free(pack);
t = &ctx->pack_state.tokens[0];
consume_bytes(ctx->buf_data, t->end, ctx->buf_len);
ctx->buf_len -= t->end;
ctx->buf_data[ctx->buf_len] = '\0';
flb_pack_state_reset(&ctx->pack_state);
flb_pack_state_init(&ctx->pack_state);
ctx->pack_state.multiple = FLB_TRUE;
}
else {
/* Process and enqueue the received line */
process_line(ctx->buf_data, ctx->buf_len, ctx);
ctx->buf_len = 0;
}
}
return 0;
}
/* It parse a JSON string and convert it to MessagePack format */
char *flb_pack_json(char *js, size_t len, int *size)
{
int i;
int flen;
int arr_size;
char *p;
char *buf;
jsmntok_t *t;
jsmntok_t *tokens;
msgpack_packer pck;
msgpack_sbuffer sbuf;
if (!js) {
return NULL;
}
tokens = json_tokenise(js, len, &arr_size);
if (!tokens) {
return NULL;
}
flb_debug("JSON to pack: '%s'", js);
/* initialize buffers */
msgpack_sbuffer_init(&sbuf);
msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write);
for (i = 0; i < arr_size ; i++) {
t = &tokens[i];
if (t->start == -1 || t->end == -1 || (t->start == 0 && t->end == 0)) {
break;
}
flen = (t->end - t->start);
switch (t->type) {
case JSMN_OBJECT:
flb_debug("json_pack: token=%i is OBJECT (size=%i)", i, t->size);
msgpack_pack_map(&pck, t->size);
break;
case JSMN_ARRAY:
flb_debug("json_pack: token=%i is ARRAY (size=%i)", i, t->size);
msgpack_pack_array(&pck, t->size);
break;
case JSMN_STRING:
flb_debug("json_pack: token=%i is STRING (len=%i)\n", i, flen);
msgpack_pack_bin(&pck, flen);
msgpack_pack_bin_body(&pck, js + t->start, flen);
break;
case JSMN_PRIMITIVE:
p = js + t->start;
if (strncmp(p, "false", 5) == 0) {
flb_debug("json_pack: token=%i is FALSE", i);
msgpack_pack_false(&pck);
}
else if (strncmp(p, "true", 4) == 0) {
flb_debug("json_pack: token=%i is TRUE", i);
msgpack_pack_true(&pck);
}
else if (strncmp(p, "null", 4) == 0) {
flb_debug("json_pack: token=%i is NULL", i);
msgpack_pack_nil(&pck);
}
else {
flb_debug("json_pack: token=%i is INT64", i);
msgpack_pack_int64(&pck, atol(p));
}
break;
}
}
/* dump data back to a new buffer */
*size = sbuf.size;
buf = malloc(sbuf.size);
memcpy(buf, sbuf.data, sbuf.size);
msgpack_sbuffer_destroy(&sbuf);
free(tokens);
return buf;
}
FLB_INLINE int flb_io_net_connect(struct flb_io_upstream *u,
struct flb_thread *th)
{
int fd;
int ret;
int error = 0;
socklen_t len = sizeof(error);
if (u->fd > 0) {
close(u->fd);
}
/* Create the socket */
fd = flb_net_socket_create(AF_INET, FLB_TRUE);
if (fd == -1) {
flb_error("[io] could not create socket");
return -1;
}
u->fd = fd;
/* Make the socket non-blocking */
flb_net_socket_nonblocking(u->fd);
/* Start the connection */
ret = flb_net_tcp_fd_connect(fd, u->tcp_host, u->tcp_port);
if (ret == -1) {
if (errno == EINPROGRESS) {
flb_debug("[upstream] connection in process");
}
else {
}
u->event.mask = MK_EVENT_EMPTY;
u->event.status = MK_EVENT_NONE;
u->thread = th;
ret = mk_event_add(u->evl,
fd,
FLB_ENGINE_EV_THREAD,
MK_EVENT_WRITE, &u->event);
if (ret == -1) {
/*
* If we failed here there no much that we can do, just
* let the caller we failed
*/
close(fd);
return -1;
}
/*
* Return the control to the parent caller, we need to wait for
* the event loop to get back to us.
*/
flb_thread_yield(th, FLB_FALSE);
/* We got a notification, remove the event registered */
ret = mk_event_del(u->evl, &u->event);
assert(ret == 0);
/* Check the connection status */
if (u->event.mask & MK_EVENT_WRITE) {
ret = getsockopt(fd, SOL_SOCKET, SO_ERROR, &error, &len);
if (ret == -1) {
flb_error("[io] could not validate socket status");
return -1;
}
if (error != 0) {
/* Connection is broken, not much to do here */
flb_debug("[io] connection failed");
return -1;
}
u->event.mask = MK_EVENT_EMPTY;
u->event.status = MK_EVENT_NONE;
}
else {
return -1;
}
}
flb_debug("[io] connection OK");
return 0;
}
void cb_forward_flush(void *data, size_t bytes,
char *tag, int tag_len,
struct flb_input_instance *i_ins, void *out_context,
struct flb_config *config)
{
int ret = -1;
int entries = 0;
size_t off = 0;
size_t total;
size_t bytes_sent;
msgpack_packer mp_pck;
msgpack_sbuffer mp_sbuf;
msgpack_unpacked result;
struct flb_out_forward_config *ctx = out_context;
struct flb_upstream_conn *u_conn;
(void) i_ins;
(void) config;
flb_debug("[out_forward] request %lu bytes to flush", bytes);
/* Initialize packager */
msgpack_sbuffer_init(&mp_sbuf);
msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write);
/* Count number of entries, is there a better way to do this ? */
msgpack_unpacked_init(&result);
while (msgpack_unpack_next(&result, data, bytes, &off)) {
entries++;
}
flb_debug("[out_fw] %i entries tag='%s' tag_len=%i",
entries, tag, tag_len);
msgpack_unpacked_destroy(&result);
/* Output: root array */
msgpack_pack_array(&mp_pck, 2);
msgpack_pack_str(&mp_pck, tag_len);
msgpack_pack_str_body(&mp_pck, tag, tag_len);
msgpack_pack_array(&mp_pck, entries);
/* Get a TCP connection instance */
u_conn = flb_upstream_conn_get(ctx->u);
if (!u_conn) {
flb_error("[out_forward] no upstream connections available");
msgpack_sbuffer_destroy(&mp_sbuf);
FLB_OUTPUT_RETURN(FLB_RETRY);
}
/* Write message header */
ret = flb_io_net_write(u_conn, mp_sbuf.data, mp_sbuf.size, &bytes_sent);
if (ret == -1) {
flb_error("[out_forward] could not write chunk header");
msgpack_sbuffer_destroy(&mp_sbuf);
flb_upstream_conn_release(u_conn);
FLB_OUTPUT_RETURN(FLB_RETRY);
}
msgpack_sbuffer_destroy(&mp_sbuf);
total = ret;
/* Write body */
ret = flb_io_net_write(u_conn, data, bytes, &bytes_sent);
if (ret == -1) {
flb_error("[out_forward] error writing content body");
flb_upstream_conn_release(u_conn);
FLB_OUTPUT_RETURN(FLB_RETRY);
}
total += bytes_sent;
flb_upstream_conn_release(u_conn);
flb_trace("[out_forward] ended write()=%d bytes", total);
FLB_OUTPUT_RETURN(FLB_OK);
}
int cb_es_init(struct flb_output_instance *ins,
struct flb_config *config,
void *data)
{
int io_type;
char *tmp;
struct flb_uri *uri = ins->host.uri;
struct flb_uri_field *f_index = NULL;
struct flb_uri_field *f_type = NULL;
struct flb_out_es_config *ctx = NULL;
struct flb_upstream *upstream;
(void) data;
if (uri) {
if (uri->count >= 2) {
f_index = flb_uri_get(uri, 0);
f_type = flb_uri_get(uri, 1);
}
}
/* Get network configuration */
if (!ins->host.name) {
ins->host.name = flb_strdup("127.0.0.1");
}
if (ins->host.port == 0) {
ins->host.port = 9200;
}
/* Allocate plugin context */
ctx = flb_malloc(sizeof(struct flb_out_es_config));
if (!ctx) {
perror("malloc");
return -1;
}
if (ins->use_tls == FLB_TRUE) {
io_type = FLB_IO_TLS;
}
else {
io_type = FLB_IO_TCP;
}
/* Prepare an upstream handler */
upstream = flb_upstream_create(config,
ins->host.name,
ins->host.port,
io_type,
&ins->tls);
if (!upstream) {
flb_free(ctx);
return -1;
}
/* Set the context */
ctx->u = upstream;
if (f_index) {
ctx->index = f_index->value;
}
else {
tmp = flb_output_get_property("index", ins);
if (!tmp) {
ctx->index = "fluentbit";
}
else {
ctx->index = tmp;
}
}
if (f_type) {
ctx->type = f_type->value;
}
else {
tmp = flb_output_get_property("type", ins);
if (!tmp) {
ctx->type = "test";
}
else {
ctx->type = tmp;
}
}
flb_debug("[es] host=%s port=%i index=%s type=%s",
ins->host.name, ins->host.port,
ctx->index, ctx->type);
flb_output_set_context(ins, ctx);
return 0;
}
/* This routine is called from a co-routine thread */
FLB_INLINE int net_io_write(struct flb_thread *th, struct flb_io_upstream *u,
void *data, size_t len, size_t *out_len)
{
int ret = 0;
ssize_t bytes;
size_t total = 0;
retry:
bytes = write(u->fd, data + total, len - total);
flb_debug("[io] write(2)=%d", bytes);
if (bytes == -1) {
if (errno == EAGAIN) {
return 0;
}
else {
/* The connection routine may yield */
flb_debug("[io] trying to reconnect");
ret = flb_io_net_connect(u, th);
if (ret == -1) {
return -1;
}
/*
* Reach here means the connection was successful, let's retry
* to write(2).
*
* note: the flb_io_connect() uses asyncronous sockets, register
* the file descriptor with the main event loop and yields until
* the main event loop returns the control back (resume), doing a
* 'retry' it's pretty safe.
*/
goto retry;
}
}
/* Update statistics */
//flb_stats_update(out->stats_fd, ret, 0);
/* Update counters */
total += bytes;
if (total < len) {
if (u->event.status == MK_EVENT_NONE) {
u->event.mask = MK_EVENT_EMPTY;
u->thread = th;
ret = mk_event_add(u->evl,
u->fd,
FLB_ENGINE_EV_THREAD,
MK_EVENT_WRITE, &u->event);
if (ret == -1) {
/*
* If we failed here there no much that we can do, just
* let the caller we failed
*/
close(u->fd);
return -1;
}
}
flb_thread_yield(th, MK_FALSE);
goto retry;
}
if (u->event.status == MK_EVENT_REGISTERED) {
/* We got a notification, remove the event registered */
ret = mk_event_del(u->evl, &u->event);
assert(ret == 0);
}
*out_len = total;
return bytes;
}
/* Callback triggered when some serial msgs are available */
int in_serial_collect(struct flb_config *config, void *in_context)
{
int ret;
int bytes;
int available;
int len;
int hits;
char *sep;
char *buf;
struct flb_in_serial_config *ctx = in_context;
while (1) {
available = (sizeof(ctx->buf_data) -1) - ctx->buf_len;
if (available > 1) {
bytes = read(ctx->fd, ctx->buf_data + ctx->buf_len, available);
if (bytes == -1) {
if (errno == EPIPE || errno == EINTR) {
return -1;
}
return 0;
}
else if (bytes == 0) {
return 0;
}
}
ctx->buf_len += bytes;
/* Always set a delimiter to avoid buffer trash */
ctx->buf_data[ctx->buf_len] = '\0';
/* Check if our buffer is full */
if (ctx->buffer_id + 1 == SERIAL_BUFFER_SIZE) {
ret = flb_engine_flush(config, &in_serial_plugin);
if (ret == -1) {
ctx->buffer_id = 0;
}
}
sep = NULL;
buf = ctx->buf_data;
len = ctx->buf_len;
hits = 0;
/* Handle FTDI handshake */
if (ctx->buf_data[0] == '\0') {
consume_bytes(ctx->buf_data, 1, ctx->buf_len);
ctx->buf_len--;
}
/* Strip CR or LF if found at first byte */
if (ctx->buf_data[0] == '\r' || ctx->buf_data[0] == '\n') {
/* Skip message with one byte with CR or LF */
flb_trace("[in_serial] skip one byte message with ASCII code=%i",
ctx->buf_data[0]);
consume_bytes(ctx->buf_data, 1, ctx->buf_len);
ctx->buf_len--;
}
if (ctx->separator) {
while ((sep = strstr(ctx->buf_data, ctx->separator))) {
len = (sep - ctx->buf_data);
if (len > 0) {
/* process the line based in the separator position */
process_line(buf, len, ctx);
consume_bytes(ctx->buf_data, len + ctx->sep_len, ctx->buf_len);
ctx->buf_len -= (len + ctx->sep_len);
hits++;
}
else {
consume_bytes(ctx->buf_data, ctx->sep_len, ctx->buf_len);
ctx->buf_len -= ctx->sep_len;
}
ctx->buf_data[ctx->buf_len] = '\0';
}
if (hits == 0 && available <= 1) {
flb_debug("[in_serial] no separator found, no more space");
ctx->buf_len = 0;
return 0;
}
}
else {
/* Process and enqueue the received line */
process_line(ctx->buf_data, ctx->buf_len, ctx);
ctx->buf_len = 0;
}
}
}
/* Callback invoked every time an event is triggered for a connection */
int tcp_conn_event(void *data)
{
int ret;
int bytes;
int available;
int size;
char *tmp;
struct mk_event *event;
struct tcp_conn *conn = data;
struct flb_in_tcp_config *ctx = conn->ctx;
jsmntok_t *t;
event = &conn->event;
if (event->mask & MK_EVENT_READ) {
available = (conn->buf_size - conn->buf_len);
if (available < 1) {
if (conn->buf_size + ctx->chunk_size > ctx->buffer_size) {
flb_trace("[in_tcp] fd=%i incoming data exceed limit (%i KB)",
event->fd, (ctx->buffer_size / 1024));
tcp_conn_del(conn);
return -1;
}
size = conn->buf_size + ctx->chunk_size;
tmp = flb_realloc(conn->buf_data, size);
if (!tmp) {
perror("realloc");
return -1;
}
flb_trace("[in_tcp] fd=%i buffer realloc %i -> %i",
event->fd, conn->buf_size, size);
conn->buf_data = tmp;
conn->buf_size = size;
available = (conn->buf_size - conn->buf_len);
}
/* Read data */
bytes = read(conn->fd,
conn->buf_data + conn->buf_len, available);
if (bytes <= 0) {
flb_trace("[in_tcp] fd=%i closed connection", event->fd);
tcp_conn_del(conn);
return -1;
}
flb_trace("[in_tcp] read()=%i pre_len=%i now_len=%i",
bytes, conn->buf_len, conn->buf_len + bytes);
conn->buf_len += bytes;
conn->buf_data[conn->buf_len] = '\0';
/* Strip CR or LF if found at first byte */
if (conn->buf_data[0] == '\r' || conn->buf_data[0] == '\n') {
/* Skip message with one byte with CR or LF */
flb_trace("[in_tcp] skip one byte message with ASCII code=%i",
conn->buf_data[0]);
consume_bytes(conn->buf_data, 1, conn->buf_len);
conn->buf_len--;
}
/* JSON Format handler */
char *pack;
int out_size;
ret = flb_pack_json_state(conn->buf_data, conn->buf_len,
&pack, &out_size, &conn->pack_state);
if (ret == FLB_ERR_JSON_PART) {
flb_debug("[in_serial] JSON incomplete, waiting for more data...");
return 0;
}
else if (ret == FLB_ERR_JSON_INVAL) {
flb_debug("[in_serial] invalid JSON message, skipping");
flb_pack_state_reset(&conn->pack_state);
flb_pack_state_init(&conn->pack_state);
conn->pack_state.multiple = FLB_TRUE;
return -1;
}
/*
* Given the Tokens used for the packaged message, append
* the records and then adjust buffer.
*/
process_pack(conn, pack, out_size);
t = &conn->pack_state.tokens[0];
consume_bytes(conn->buf_data, t->end, conn->buf_len);
conn->buf_len -= t->end;
conn->buf_data[conn->buf_len] = '\0';
flb_pack_state_reset(&conn->pack_state);
flb_pack_state_init(&conn->pack_state);
conn->pack_state.multiple = FLB_TRUE;
flb_free(pack);
return bytes;
}
if (event->mask & MK_EVENT_CLOSE) {
flb_trace("[in_tcp] fd=%i hangup", event->fd);
tcp_conn_del(conn);
return -1;
}
return 0;
}
/*
* This routine perform a TCP connection and the required TLS/SSL
* handshake,
*/
FLB_INLINE int flb_io_net_tls_connect(struct flb_io_upstream *u,
struct flb_thread *th)
{
int fd;
int ret;
int error = 0;
int flag;
socklen_t len = sizeof(error);
struct flb_tls_session *session;
if (u->fd > 0) {
close(u->fd);
}
/* Create the socket */
fd = flb_net_socket_create(AF_INET, FLB_TRUE);
if (fd == -1) {
flb_error("[io] could not create socket");
return -1;
}
u->fd = fd;
/* Make the socket non-blocking */
flb_net_socket_nonblocking(u->fd);
/* Start the connection */
ret = flb_net_tcp_fd_connect(fd, u->tcp_host, u->tcp_port);
if (ret == -1) {
if (errno == EINPROGRESS) {
flb_debug("[upstream] connection in process");
}
else {
close(u->fd);
if (u->tls_session) {
tls_session_destroy(u->tls_session);
u->tls_session = NULL;
}
return -1;
}
MK_EVENT_NEW(&u->event);
u->thread = th;
ret = mk_event_add(u->evl,
fd,
FLB_ENGINE_EV_THREAD,
MK_EVENT_WRITE, &u->event);
if (ret == -1) {
/*
* If we failed here there no much that we can do, just
* let the caller we failed
*/
flb_error("[io_tls] connect failed registering event");
close(fd);
return -1;
}
/*
* Return the control to the parent caller, we need to wait for
* the event loop to get back to us.
*/
flb_thread_yield(th, FLB_FALSE);
/* Check the connection status */
if (u->event.mask & MK_EVENT_WRITE) {
ret = getsockopt(fd, SOL_SOCKET, SO_ERROR, &error, &len);
if (ret == -1) {
flb_error("[io_tls] could not validate socket status");
goto error;
}
if (error != 0) {
/* Connection is broken, not much to do here */
flb_error("[io_tls] connection failed");
goto error;
}
}
else {
return -1;
}
}
/* Configure TLS and prepare handshake */
session = u->tls_session;
mbedtls_ssl_set_bio(&session->ssl,
u,
mbedtls_net_send, mbedtls_net_recv, NULL);
retry_handshake:
ret = mbedtls_ssl_handshake(&session->ssl);
if (ret != 0) {
if (ret != MBEDTLS_ERR_SSL_WANT_READ &&
ret != MBEDTLS_ERR_SSL_WANT_WRITE) {
io_tls_error(ret);
goto error;
}
if (ret == MBEDTLS_ERR_SSL_WANT_WRITE) {
flag = MK_EVENT_WRITE;
}
else if (ret == MBEDTLS_ERR_SSL_WANT_READ) {
flag = MK_EVENT_READ;
}
else {
}
/*
* FIXME: if we need multiple reads we are invoking the same
* system call multiple times.
*/
ret = mk_event_add(u->evl,
u->event.fd,
FLB_ENGINE_EV_THREAD,
flag, &u->event);
if (ret == -1) {
goto error;
}
flb_thread_yield(th, FLB_FALSE);
goto retry_handshake;
}
else {
flb_debug("[io_tls] Handshake OK");
}
if (u->event.status == MK_EVENT_REGISTERED) {
mk_event_del(u->evl, &u->event);
}
flb_debug("[io_tls] connection OK");
return 0;
error:
if (u->event.status == MK_EVENT_REGISTERED) {
mk_event_del(u->evl, &u->event);
}
return -1;
}
