void
log_proto_buffered_server_queued(LogProtoServer *s)
{
LogProtoBufferedServer *self = (LogProtoBufferedServer *) s;
LogProtoBufferedServerState *state = log_proto_buffered_server_get_state(self);
/* NOTE: we modify the current file position _after_ updating
buffer_pos, since if we crash right here, at least we
won't lose data on the next restart, but rather we
duplicate some data */
state->buffer_pos = state->pending_buffer_pos;
state->raw_stream_pos = state->pending_raw_stream_pos;
state->raw_buffer_size = state->pending_raw_buffer_size;
if (state->pending_buffer_pos == state->pending_buffer_end)
{
state->pending_buffer_end = 0;
state->buffer_pos = state->pending_buffer_pos = 0;
}
if (self->pos_tracking)
{
if (state->buffer_pos == state->pending_buffer_end)
{
state->raw_stream_pos += state->raw_buffer_size;
state->raw_buffer_size = 0;
}
}
msg_trace("Last message got confirmed",
evt_tag_int("raw_stream_pos", state->raw_stream_pos),
evt_tag_int("raw_buffer_len", state->raw_buffer_size),
evt_tag_int("buffer_pos", state->buffer_pos),
evt_tag_int("buffer_end", state->pending_buffer_end),
NULL);
log_proto_buffered_server_put_state(self);
}
static gboolean
log_proto_binary_record_server_fetch_from_buffer(LogProtoBufferedServer *s, const guchar *buffer_start, gsize buffer_bytes, const guchar **msg, gsize *msg_len)
{
LogProtoBufferedServerState *state = log_proto_buffered_server_get_state(s);
*msg_len = buffer_bytes;
state->pending_buffer_pos = state->pending_buffer_end;
*msg = buffer_start;
log_proto_buffered_server_put_state(s);
return TRUE;
}
static void
_buffered_server_bookmark_fill(LogProtoBufferedServer *self, Bookmark *bookmark)
{
LogProtoBufferedServerState *state = log_proto_buffered_server_get_state(self);
BufferedServerBookmarkData *data = (BufferedServerBookmarkData *)(&bookmark->container);
data->pending_buffer_pos = state->pending_buffer_pos;
data->pending_raw_stream_pos = state->pending_raw_stream_pos;
data->pending_raw_buffer_size = state->pending_raw_buffer_size;
data->persist_handle = self->persist_handle;
bookmark->save = _buffered_server_bookmark_save;
log_proto_buffered_server_put_state(self);
}
static gboolean
log_proto_padded_record_server_fetch_from_buffer(LogProtoBufferedServer *s, const guchar *buffer_start, gsize buffer_bytes, const guchar **msg, gsize *msg_len)
{
LogProtoBufferedServerState *state = log_proto_buffered_server_get_state(s);
const guchar *eol;
eol = find_eom(buffer_start, buffer_bytes);
*msg_len = (eol ? eol - buffer_start : buffer_bytes);
state->pending_buffer_pos = state->pending_buffer_end;
*msg = buffer_start;
log_proto_buffered_server_put_state(s);
return TRUE;
}
static gboolean
log_proto_dgram_server_fetch_from_buffer(LogProtoBufferedServer *s, const guchar *buffer_start, gsize buffer_bytes,
const guchar **msg, gsize *msg_len)
{
LogProtoBufferedServerState *state = log_proto_buffered_server_get_state(s);
/*
* we are set to packet terminating mode
*/
*msg = buffer_start;
*msg_len = buffer_bytes;
state->pending_buffer_pos = state->pending_buffer_end;
log_proto_buffered_server_put_state(s);
return TRUE;
}
static void
_buffered_server_update_pos(LogProtoServer *s)
{
LogProtoBufferedServer *self = (LogProtoBufferedServer *) s;
LogProtoBufferedServerState *state = log_proto_buffered_server_get_state(self);
if (state->pending_buffer_pos == state->pending_buffer_end)
{
state->pending_buffer_end = 0;
state->pending_buffer_pos = 0;
if (self->pos_tracking)
{
state->pending_raw_stream_pos += state->pending_raw_buffer_size;
state->pending_raw_buffer_size = 0;
}
}
log_proto_buffered_server_put_state(self);
}
static gboolean
log_proto_buffered_server_fetch_from_buffer(LogProtoBufferedServer *self, const guchar **msg, gsize *msg_len, LogTransportAuxData *aux)
{
gsize buffer_bytes;
const guchar *buffer_start;
LogProtoBufferedServerState *state = log_proto_buffered_server_get_state(self);
gboolean success = FALSE;
buffer_start = self->buffer + state->pending_buffer_pos;
buffer_bytes = state->pending_buffer_end - state->pending_buffer_pos;
if (buffer_bytes == 0)
{
/* if buffer_bytes is zero bytes, it means that we completely
* processed our buffer without having a fraction of a line still
* there. It is important to reset
* pending_buffer_pos/pending_buffer_end to zero as the caller assumes
* that if we return no message from the buffer, then buffer_pos is
* _zero_.
*/
if (G_UNLIKELY(self->pos_tracking))
{
state->pending_raw_stream_pos += state->pending_raw_buffer_size;
state->pending_raw_buffer_size = 0;
}
state->pending_buffer_pos = state->pending_buffer_end = 0;
goto exit;
}
success = self->fetch_from_buffer(self, buffer_start, buffer_bytes, msg, msg_len);
if (aux)
log_transport_aux_data_copy(aux, &self->buffer_aux);
exit:
log_proto_buffered_server_put_state(self);
return success;
}
static gboolean
log_proto_buffered_server_convert_from_raw(LogProtoBufferedServer *self, const guchar *raw_buffer, gsize raw_buffer_len)
{
/* some data was read */
gsize avail_in = raw_buffer_len;
gsize avail_out;
gchar *out;
gint ret = -1;
gboolean success = FALSE;
LogProtoBufferedServerState *state = log_proto_buffered_server_get_state(self);
do
{
avail_out = state->buffer_size - state->pending_buffer_end;
out = (gchar *) self->buffer + state->pending_buffer_end;
ret = g_iconv(self->convert, (gchar **) &raw_buffer, &avail_in, (gchar **) &out, &avail_out);
if (ret == (gsize) -1)
{
switch (errno)
{
case EINVAL:
if (self->stream_based)
{
/* Incomplete text, do not report an error, rather try to read again */
state->pending_buffer_end = state->buffer_size - avail_out;
if (avail_in > 0)
{
if (avail_in > sizeof(state->raw_buffer_leftover))
{
msg_error("Invalid byte sequence, the remaining raw buffer is larger than the supported leftover size",
evt_tag_str("encoding", self->super.options->encoding),
evt_tag_int("avail_in", avail_in),
evt_tag_int("leftover_size", sizeof(state->raw_buffer_leftover)));
goto error;
}
memcpy(state->raw_buffer_leftover, raw_buffer, avail_in);
state->raw_buffer_leftover_size = avail_in;
state->raw_buffer_size -= avail_in;
msg_trace("Leftover characters remained after conversion, delaying message until another chunk arrives",
evt_tag_str("encoding", self->super.options->encoding),
evt_tag_int("avail_in", avail_in));
goto success;
}
}
else
{
msg_error("Byte sequence too short, cannot convert an individual frame in its entirety",
evt_tag_str("encoding", self->super.options->encoding),
evt_tag_int("avail_in", avail_in));
goto error;
}
break;
case E2BIG:
state->pending_buffer_end = state->buffer_size - avail_out;
/* extend the buffer */
if (state->buffer_size < self->super.options->max_buffer_size)
{
state->buffer_size *= 2;
if (state->buffer_size > self->super.options->max_buffer_size)
state->buffer_size = self->super.options->max_buffer_size;
self->buffer = g_realloc(self->buffer, state->buffer_size);
/* recalculate the out pointer, and add what we have now */
ret = -1;
}
else
{
msg_error("Incoming byte stream requires a too large conversion buffer, probably invalid character sequence",
evt_tag_str("encoding", self->super.options->encoding),
evt_tag_printf("buffer", "%.*s", (gint) state->pending_buffer_end, self->buffer));
goto error;
}
break;
case EILSEQ:
default:
msg_notice("Invalid byte sequence or other error while converting input, skipping character",
evt_tag_str("encoding", self->super.options->encoding),
evt_tag_printf("char", "0x%02x", *(guchar *) raw_buffer));
goto error;
}
}
else
{
state->pending_buffer_end = state->buffer_size - avail_out;
}
}
while (avail_in > 0);
success:
success = TRUE;
error:
log_proto_buffered_server_put_state(self);
return success;
}
static GIOStatus
log_proto_buffered_server_fetch_into_buffer(LogProtoBufferedServer *self)
{
guchar *raw_buffer = NULL;
gint avail;
gint rc;
LogProtoBufferedServerState *state = log_proto_buffered_server_get_state(self);
GIOStatus result = G_IO_STATUS_NORMAL;
if (G_UNLIKELY(!self->buffer))
log_proto_buffered_server_allocate_buffer(self, state);
if (self->convert == (GIConv) -1)
{
/* no conversion, we read directly into our buffer */
raw_buffer = self->buffer + state->pending_buffer_end;
avail = state->buffer_size - state->pending_buffer_end;
}
else
{
/* if conversion is needed, we first read into an on-stack
* buffer, and then convert it into our internal buffer */
raw_buffer = g_alloca(self->super.options->init_buffer_size + state->raw_buffer_leftover_size);
memcpy(raw_buffer, state->raw_buffer_leftover, state->raw_buffer_leftover_size);
avail = self->super.options->init_buffer_size;
}
if (avail == 0)
goto exit;
rc = log_proto_buffered_server_read_data(self, raw_buffer + state->raw_buffer_leftover_size, avail);
if (rc < 0)
{
if (errno == EAGAIN)
{
/* ok we don't have any more data to read, return to main poll loop */
result = G_IO_STATUS_AGAIN;
}
else
{
/* an error occurred while reading */
msg_error("I/O error occurred while reading",
evt_tag_int(EVT_TAG_FD, self->super.transport->fd),
evt_tag_errno(EVT_TAG_OSERROR, errno));
result = G_IO_STATUS_ERROR;
}
}
else if (rc == 0)
{
/* EOF read */
msg_verbose("EOF occurred while reading",
evt_tag_int(EVT_TAG_FD, self->super.transport->fd));
if (state->raw_buffer_leftover_size > 0)
{
msg_error("EOF read on a channel with leftovers from previous character conversion, dropping input");
state->pending_buffer_pos = state->pending_buffer_end = 0;
}
result = G_IO_STATUS_EOF;
}
else
{
state->pending_raw_buffer_size += rc;
rc += state->raw_buffer_leftover_size;
state->raw_buffer_leftover_size = 0;
if (self->convert == (GIConv) -1)
{
state->pending_buffer_end += rc;
}
else if (!log_proto_buffered_server_convert_from_raw(self, raw_buffer, rc))
{
result = G_IO_STATUS_ERROR;
}
}
exit:
log_proto_buffered_server_put_state(self);
return result;
}
static void
log_proto_buffered_server_apply_state(LogProtoBufferedServer *self, PersistEntryHandle handle, const gchar *persist_name)
{
struct stat st;
gint64 ofs = 0;
LogProtoBufferedServerState *state;
gint fd;
fd = self->super.transport->fd;
self->persist_handle = handle;
if (fstat(fd, &st) < 0)
return;
state = log_proto_buffered_server_get_state(self);
if (!self->buffer)
{
self->buffer = g_malloc(state->buffer_size);
}
state->pending_buffer_end = 0;
if (state->file_inode &&
state->file_inode == st.st_ino &&
state->file_size <= st.st_size &&
state->raw_stream_pos <= st.st_size)
{
ofs = state->raw_stream_pos;
lseek(fd, ofs, SEEK_SET);
}
else
{
if (state->file_inode)
{
/* the stored state does not match the current file */
msg_notice("The current log file has a mismatching size/inode information, restarting from the beginning",
evt_tag_str("state", persist_name),
evt_tag_int("stored_inode", state->file_inode),
evt_tag_int("cur_file_inode", st.st_ino),
evt_tag_int("stored_size", state->file_size),
evt_tag_int("cur_file_size", st.st_size),
evt_tag_int("raw_stream_pos", state->raw_stream_pos));
}
goto error;
}
if (state->raw_buffer_size)
{
gssize rc;
guchar *raw_buffer;
if (!self->super.options->encoding)
{
/* no conversion, we read directly into our buffer */
if (state->raw_buffer_size > state->buffer_size)
{
msg_notice("Invalid LogProtoBufferedServerState.raw_buffer_size, larger than buffer_size and no encoding is set, restarting from the beginning",
evt_tag_str("state", persist_name),
evt_tag_int("raw_buffer_size", state->raw_buffer_size),
evt_tag_int("buffer_size", state->buffer_size),
evt_tag_int("init_buffer_size", self->super.options->init_buffer_size));
goto error;
}
raw_buffer = self->buffer;
}
else
{
if (state->raw_buffer_size > self->super.options->max_buffer_size)
{
msg_notice("Invalid LogProtoBufferedServerState.raw_buffer_size, larger than max_buffer_size, restarting from the beginning",
evt_tag_str("state", persist_name),
evt_tag_int("raw_buffer_size", state->raw_buffer_size),
evt_tag_int("init_buffer_size", self->super.options->init_buffer_size),
evt_tag_int("max_buffer_size", self->super.options->max_buffer_size));
goto error;
}
raw_buffer = g_alloca(state->raw_buffer_size);
}
rc = log_transport_read(self->super.transport, raw_buffer, state->raw_buffer_size, NULL);
if (rc != state->raw_buffer_size)
{
msg_notice("Error re-reading buffer contents of the file to be continued, restarting from the beginning",
evt_tag_str("state", persist_name));
goto error;
}
state->pending_buffer_end = 0;
if (self->super.options->encoding)
{
if (!log_proto_buffered_server_convert_from_raw(self, raw_buffer, rc))
{
msg_notice("Error re-converting buffer contents of the file to be continued, restarting from the beginning",
evt_tag_str("state", persist_name));
goto error;
}
}
else
{
state->pending_buffer_end += rc;
}
if (state->buffer_pos > state->pending_buffer_end)
{
msg_notice("Converted buffer contents is smaller than the current buffer position, starting from the beginning of the buffer, some lines may be duplicated",
evt_tag_str("state", persist_name));
state->buffer_pos = state->pending_buffer_pos = 0;
}
}
else
{
/* although we do have buffer position information, but the
* complete contents of the buffer is already processed, instead
* of reading and then dropping it, position the file after the
* indicated block */
state->raw_stream_pos += state->raw_buffer_size;
ofs = state->raw_stream_pos;
state->raw_buffer_size = 0;
state->buffer_pos = state->pending_buffer_end = 0;
lseek(fd, state->raw_stream_pos, SEEK_SET);
}
goto exit;
error:
ofs = 0;
state->buffer_pos = 0;
state->pending_buffer_end = 0;
state->__deprecated_buffer_cached_eol = 0;
state->raw_stream_pos = 0;
state->raw_buffer_size = 0;
state->raw_buffer_leftover_size = 0;
lseek(fd, 0, SEEK_SET);
exit:
state->file_inode = st.st_ino;
state->file_size = st.st_size;
state->raw_stream_pos = ofs;
state->pending_buffer_pos = state->buffer_pos;
state->pending_raw_stream_pos = state->raw_stream_pos;
state->pending_raw_buffer_size = state->raw_buffer_size;
state->__deprecated_buffer_cached_eol = 0;
state = NULL;
log_proto_buffered_server_put_state(self);
}
