void log_proto_buffered_server_queued(LogProtoServer *s) { LogProtoBufferedServer *self = (LogProtoBufferedServer *) s; LogProtoBufferedServerState *state = log_proto_buffered_server_get_state(self); /* NOTE: we modify the current file position _after_ updating buffer_pos, since if we crash right here, at least we won't lose data on the next restart, but rather we duplicate some data */ state->buffer_pos = state->pending_buffer_pos; state->raw_stream_pos = state->pending_raw_stream_pos; state->raw_buffer_size = state->pending_raw_buffer_size; if (state->pending_buffer_pos == state->pending_buffer_end) { state->pending_buffer_end = 0; state->buffer_pos = state->pending_buffer_pos = 0; } if (self->pos_tracking) { if (state->buffer_pos == state->pending_buffer_end) { state->raw_stream_pos += state->raw_buffer_size; state->raw_buffer_size = 0; } } msg_trace("Last message got confirmed", evt_tag_int("raw_stream_pos", state->raw_stream_pos), evt_tag_int("raw_buffer_len", state->raw_buffer_size), evt_tag_int("buffer_pos", state->buffer_pos), evt_tag_int("buffer_end", state->pending_buffer_end), NULL); log_proto_buffered_server_put_state(self); }
static gboolean log_proto_binary_record_server_fetch_from_buffer(LogProtoBufferedServer *s, const guchar *buffer_start, gsize buffer_bytes, const guchar **msg, gsize *msg_len) { LogProtoBufferedServerState *state = log_proto_buffered_server_get_state(s); *msg_len = buffer_bytes; state->pending_buffer_pos = state->pending_buffer_end; *msg = buffer_start; log_proto_buffered_server_put_state(s); return TRUE; }
static void _buffered_server_bookmark_fill(LogProtoBufferedServer *self, Bookmark *bookmark) { LogProtoBufferedServerState *state = log_proto_buffered_server_get_state(self); BufferedServerBookmarkData *data = (BufferedServerBookmarkData *)(&bookmark->container); data->pending_buffer_pos = state->pending_buffer_pos; data->pending_raw_stream_pos = state->pending_raw_stream_pos; data->pending_raw_buffer_size = state->pending_raw_buffer_size; data->persist_handle = self->persist_handle; bookmark->save = _buffered_server_bookmark_save; log_proto_buffered_server_put_state(self); }
static gboolean log_proto_padded_record_server_fetch_from_buffer(LogProtoBufferedServer *s, const guchar *buffer_start, gsize buffer_bytes, const guchar **msg, gsize *msg_len) { LogProtoBufferedServerState *state = log_proto_buffered_server_get_state(s); const guchar *eol; eol = find_eom(buffer_start, buffer_bytes); *msg_len = (eol ? eol - buffer_start : buffer_bytes); state->pending_buffer_pos = state->pending_buffer_end; *msg = buffer_start; log_proto_buffered_server_put_state(s); return TRUE; }
static gboolean log_proto_dgram_server_fetch_from_buffer(LogProtoBufferedServer *s, const guchar *buffer_start, gsize buffer_bytes, const guchar **msg, gsize *msg_len) { LogProtoBufferedServerState *state = log_proto_buffered_server_get_state(s); /* * we are set to packet terminating mode */ *msg = buffer_start; *msg_len = buffer_bytes; state->pending_buffer_pos = state->pending_buffer_end; log_proto_buffered_server_put_state(s); return TRUE; }
static void _buffered_server_update_pos(LogProtoServer *s) { LogProtoBufferedServer *self = (LogProtoBufferedServer *) s; LogProtoBufferedServerState *state = log_proto_buffered_server_get_state(self); if (state->pending_buffer_pos == state->pending_buffer_end) { state->pending_buffer_end = 0; state->pending_buffer_pos = 0; if (self->pos_tracking) { state->pending_raw_stream_pos += state->pending_raw_buffer_size; state->pending_raw_buffer_size = 0; } } log_proto_buffered_server_put_state(self); }
static gboolean log_proto_buffered_server_fetch_from_buffer(LogProtoBufferedServer *self, const guchar **msg, gsize *msg_len, LogTransportAuxData *aux) { gsize buffer_bytes; const guchar *buffer_start; LogProtoBufferedServerState *state = log_proto_buffered_server_get_state(self); gboolean success = FALSE; buffer_start = self->buffer + state->pending_buffer_pos; buffer_bytes = state->pending_buffer_end - state->pending_buffer_pos; if (buffer_bytes == 0) { /* if buffer_bytes is zero bytes, it means that we completely * processed our buffer without having a fraction of a line still * there. It is important to reset * pending_buffer_pos/pending_buffer_end to zero as the caller assumes * that if we return no message from the buffer, then buffer_pos is * _zero_. */ if (G_UNLIKELY(self->pos_tracking)) { state->pending_raw_stream_pos += state->pending_raw_buffer_size; state->pending_raw_buffer_size = 0; } state->pending_buffer_pos = state->pending_buffer_end = 0; goto exit; } success = self->fetch_from_buffer(self, buffer_start, buffer_bytes, msg, msg_len); if (aux) log_transport_aux_data_copy(aux, &self->buffer_aux); exit: log_proto_buffered_server_put_state(self); return success; }
static gboolean log_proto_buffered_server_convert_from_raw(LogProtoBufferedServer *self, const guchar *raw_buffer, gsize raw_buffer_len) { /* some data was read */ gsize avail_in = raw_buffer_len; gsize avail_out; gchar *out; gint ret = -1; gboolean success = FALSE; LogProtoBufferedServerState *state = log_proto_buffered_server_get_state(self); do { avail_out = state->buffer_size - state->pending_buffer_end; out = (gchar *) self->buffer + state->pending_buffer_end; ret = g_iconv(self->convert, (gchar **) &raw_buffer, &avail_in, (gchar **) &out, &avail_out); if (ret == (gsize) -1) { switch (errno) { case EINVAL: if (self->stream_based) { /* Incomplete text, do not report an error, rather try to read again */ state->pending_buffer_end = state->buffer_size - avail_out; if (avail_in > 0) { if (avail_in > sizeof(state->raw_buffer_leftover)) { msg_error("Invalid byte sequence, the remaining raw buffer is larger than the supported leftover size", evt_tag_str("encoding", self->super.options->encoding), evt_tag_int("avail_in", avail_in), evt_tag_int("leftover_size", sizeof(state->raw_buffer_leftover))); goto error; } memcpy(state->raw_buffer_leftover, raw_buffer, avail_in); state->raw_buffer_leftover_size = avail_in; state->raw_buffer_size -= avail_in; msg_trace("Leftover characters remained after conversion, delaying message until another chunk arrives", evt_tag_str("encoding", self->super.options->encoding), evt_tag_int("avail_in", avail_in)); goto success; } } else { msg_error("Byte sequence too short, cannot convert an individual frame in its entirety", evt_tag_str("encoding", self->super.options->encoding), evt_tag_int("avail_in", avail_in)); goto error; } break; case E2BIG: state->pending_buffer_end = state->buffer_size - avail_out; /* extend the buffer */ if (state->buffer_size < self->super.options->max_buffer_size) { state->buffer_size *= 2; if (state->buffer_size > self->super.options->max_buffer_size) state->buffer_size = self->super.options->max_buffer_size; self->buffer = g_realloc(self->buffer, state->buffer_size); /* recalculate the out pointer, and add what we have now */ ret = -1; } else { msg_error("Incoming byte stream requires a too large conversion buffer, probably invalid character sequence", evt_tag_str("encoding", self->super.options->encoding), evt_tag_printf("buffer", "%.*s", (gint) state->pending_buffer_end, self->buffer)); goto error; } break; case EILSEQ: default: msg_notice("Invalid byte sequence or other error while converting input, skipping character", evt_tag_str("encoding", self->super.options->encoding), evt_tag_printf("char", "0x%02x", *(guchar *) raw_buffer)); goto error; } } else { state->pending_buffer_end = state->buffer_size - avail_out; } } while (avail_in > 0); success: success = TRUE; error: log_proto_buffered_server_put_state(self); return success; }
static GIOStatus log_proto_buffered_server_fetch_into_buffer(LogProtoBufferedServer *self) { guchar *raw_buffer = NULL; gint avail; gint rc; LogProtoBufferedServerState *state = log_proto_buffered_server_get_state(self); GIOStatus result = G_IO_STATUS_NORMAL; if (G_UNLIKELY(!self->buffer)) log_proto_buffered_server_allocate_buffer(self, state); if (self->convert == (GIConv) -1) { /* no conversion, we read directly into our buffer */ raw_buffer = self->buffer + state->pending_buffer_end; avail = state->buffer_size - state->pending_buffer_end; } else { /* if conversion is needed, we first read into an on-stack * buffer, and then convert it into our internal buffer */ raw_buffer = g_alloca(self->super.options->init_buffer_size + state->raw_buffer_leftover_size); memcpy(raw_buffer, state->raw_buffer_leftover, state->raw_buffer_leftover_size); avail = self->super.options->init_buffer_size; } if (avail == 0) goto exit; rc = log_proto_buffered_server_read_data(self, raw_buffer + state->raw_buffer_leftover_size, avail); if (rc < 0) { if (errno == EAGAIN) { /* ok we don't have any more data to read, return to main poll loop */ result = G_IO_STATUS_AGAIN; } else { /* an error occurred while reading */ msg_error("I/O error occurred while reading", evt_tag_int(EVT_TAG_FD, self->super.transport->fd), evt_tag_errno(EVT_TAG_OSERROR, errno)); result = G_IO_STATUS_ERROR; } } else if (rc == 0) { /* EOF read */ msg_verbose("EOF occurred while reading", evt_tag_int(EVT_TAG_FD, self->super.transport->fd)); if (state->raw_buffer_leftover_size > 0) { msg_error("EOF read on a channel with leftovers from previous character conversion, dropping input"); state->pending_buffer_pos = state->pending_buffer_end = 0; } result = G_IO_STATUS_EOF; } else { state->pending_raw_buffer_size += rc; rc += state->raw_buffer_leftover_size; state->raw_buffer_leftover_size = 0; if (self->convert == (GIConv) -1) { state->pending_buffer_end += rc; } else if (!log_proto_buffered_server_convert_from_raw(self, raw_buffer, rc)) { result = G_IO_STATUS_ERROR; } } exit: log_proto_buffered_server_put_state(self); return result; }
static void log_proto_buffered_server_apply_state(LogProtoBufferedServer *self, PersistEntryHandle handle, const gchar *persist_name) { struct stat st; gint64 ofs = 0; LogProtoBufferedServerState *state; gint fd; fd = self->super.transport->fd; self->persist_handle = handle; if (fstat(fd, &st) < 0) return; state = log_proto_buffered_server_get_state(self); if (!self->buffer) { self->buffer = g_malloc(state->buffer_size); } state->pending_buffer_end = 0; if (state->file_inode && state->file_inode == st.st_ino && state->file_size <= st.st_size && state->raw_stream_pos <= st.st_size) { ofs = state->raw_stream_pos; lseek(fd, ofs, SEEK_SET); } else { if (state->file_inode) { /* the stored state does not match the current file */ msg_notice("The current log file has a mismatching size/inode information, restarting from the beginning", evt_tag_str("state", persist_name), evt_tag_int("stored_inode", state->file_inode), evt_tag_int("cur_file_inode", st.st_ino), evt_tag_int("stored_size", state->file_size), evt_tag_int("cur_file_size", st.st_size), evt_tag_int("raw_stream_pos", state->raw_stream_pos)); } goto error; } if (state->raw_buffer_size) { gssize rc; guchar *raw_buffer; if (!self->super.options->encoding) { /* no conversion, we read directly into our buffer */ if (state->raw_buffer_size > state->buffer_size) { msg_notice("Invalid LogProtoBufferedServerState.raw_buffer_size, larger than buffer_size and no encoding is set, restarting from the beginning", evt_tag_str("state", persist_name), evt_tag_int("raw_buffer_size", state->raw_buffer_size), evt_tag_int("buffer_size", state->buffer_size), evt_tag_int("init_buffer_size", self->super.options->init_buffer_size)); goto error; } raw_buffer = self->buffer; } else { if (state->raw_buffer_size > self->super.options->max_buffer_size) { msg_notice("Invalid LogProtoBufferedServerState.raw_buffer_size, larger than max_buffer_size, restarting from the beginning", evt_tag_str("state", persist_name), evt_tag_int("raw_buffer_size", state->raw_buffer_size), evt_tag_int("init_buffer_size", self->super.options->init_buffer_size), evt_tag_int("max_buffer_size", self->super.options->max_buffer_size)); goto error; } raw_buffer = g_alloca(state->raw_buffer_size); } rc = log_transport_read(self->super.transport, raw_buffer, state->raw_buffer_size, NULL); if (rc != state->raw_buffer_size) { msg_notice("Error re-reading buffer contents of the file to be continued, restarting from the beginning", evt_tag_str("state", persist_name)); goto error; } state->pending_buffer_end = 0; if (self->super.options->encoding) { if (!log_proto_buffered_server_convert_from_raw(self, raw_buffer, rc)) { msg_notice("Error re-converting buffer contents of the file to be continued, restarting from the beginning", evt_tag_str("state", persist_name)); goto error; } } else { state->pending_buffer_end += rc; } if (state->buffer_pos > state->pending_buffer_end) { msg_notice("Converted buffer contents is smaller than the current buffer position, starting from the beginning of the buffer, some lines may be duplicated", evt_tag_str("state", persist_name)); state->buffer_pos = state->pending_buffer_pos = 0; } } else { /* although we do have buffer position information, but the * complete contents of the buffer is already processed, instead * of reading and then dropping it, position the file after the * indicated block */ state->raw_stream_pos += state->raw_buffer_size; ofs = state->raw_stream_pos; state->raw_buffer_size = 0; state->buffer_pos = state->pending_buffer_end = 0; lseek(fd, state->raw_stream_pos, SEEK_SET); } goto exit; error: ofs = 0; state->buffer_pos = 0; state->pending_buffer_end = 0; state->__deprecated_buffer_cached_eol = 0; state->raw_stream_pos = 0; state->raw_buffer_size = 0; state->raw_buffer_leftover_size = 0; lseek(fd, 0, SEEK_SET); exit: state->file_inode = st.st_ino; state->file_size = st.st_size; state->raw_stream_pos = ofs; state->pending_buffer_pos = state->buffer_pos; state->pending_raw_stream_pos = state->raw_stream_pos; state->pending_raw_buffer_size = state->raw_buffer_size; state->__deprecated_buffer_cached_eol = 0; state = NULL; log_proto_buffered_server_put_state(self); }