static void
log_reader_init_watches(LogReader *self)
{
gint fd;
GIOCondition cond;
log_proto_prepare(self->proto, &fd, &cond);
IV_FD_INIT(&self->fd_watch);
self->fd_watch.cookie = self;
IV_TIMER_INIT(&self->follow_timer);
self->follow_timer.cookie = self;
self->follow_timer.handler = log_reader_io_follow_file;
IV_TASK_INIT(&self->restart_task);
self->restart_task.cookie = self;
self->restart_task.handler = log_reader_io_process_input;
IV_EVENT_INIT(&self->schedule_wakeup);
self->schedule_wakeup.cookie = self;
self->schedule_wakeup.handler = log_reader_wakeup_triggered;
main_loop_io_worker_job_init(&self->io_job);
self->io_job.user_data = self;
self->io_job.work = (void (*)(void *)) log_reader_work_perform;
self->io_job.completion = (void (*)(void *)) log_reader_work_finished;
}
static void
log_writer_start_watches(LogWriter *self)
{
gint fd;
GIOCondition cond;
if (!self->watches_running)
{
log_proto_prepare(self->proto, &fd, &cond);
if (self->pollable_state < 0)
{
if (is_file_regular(fd))
self->pollable_state = 0;
else
self->pollable_state = iv_fd_pollable(fd);
}
if (self->pollable_state)
{
self->fd_watch.fd = fd;
iv_fd_register(&self->fd_watch);
}
log_writer_update_watches(self);
self->watches_running = TRUE;
}
}
static gboolean
log_reader_fd_prepare(GSource *source,
gint *timeout)
{
LogReaderWatch *self = (LogReaderWatch *) source;
GIOCondition proto_cond;
self->pollfd.revents = 0;
self->pollfd.events = G_IO_ERR;
/* never indicate readability if flow control prevents us from sending messages */
if (!log_source_free_to_send(&self->reader->super))
return FALSE;
if (log_proto_prepare(self->proto, &self->pollfd.fd, &proto_cond, timeout))
return TRUE;
if (self->reader->immediate_check)
{
*timeout = 0;
self->reader->immediate_check = FALSE;
return FALSE;
}
if (self->reader->options->follow_freq > 0)
{
*timeout = self->reader->options->follow_freq;
return FALSE;
}
self->pollfd.events |= proto_cond;
return FALSE;
}
/* NOTE: the return value is only used during initialization, and it is not
* expected that it'd change once it returns success */
static gboolean
log_reader_start_watches(LogReader *self)
{
gint fd;
GIOCondition cond;
log_proto_prepare(self->proto, &fd, &cond);
if (self->options->follow_freq > 0)
{
/* follow freq specified (only the file source does that, go into timed polling */
/* NOTE: the fd may not be set here, as it may not have been opened yet */
iv_timer_register(&self->follow_timer);
}
else if (fd < 0)
{
msg_error("In order to poll non-yet-existing files, follow_freq() must be set",
NULL);
return FALSE;
}
else
{
/* we have an FD, it is possible to poll it, register it */
self->fd_watch.fd = fd;
if (self->pollable_state < 0)
{
if (iv_fd_register_try(&self->fd_watch) == 0)
self->pollable_state = 1;
else
self->pollable_state = 0;
}
else if (self->pollable_state > 0)
{
iv_fd_register(&self->fd_watch);
}
else
{
msg_error("Unable to determine how to monitor this fd, follow_freq() not set and it is not possible to poll it with the current ivykis polling method, try changing IV_EXCLUDE_POLL_METHOD environment variable",
evt_tag_int("fd", fd),
NULL);
return FALSE;
}
}
log_reader_update_watches(self);
return TRUE;
}
static void
log_writer_update_watches(LogWriter *self)
{
gint fd;
GIOCondition cond = 0;
gboolean partial_batch;
gint timeout_msec = 0;
main_loop_assert_main_thread();
/* NOTE: we either start the suspend_timer or enable the fd_watch. The two MUST not happen at the same time. */
if (log_proto_prepare(self->proto, &fd, &cond) ||
self->flush_waiting_for_timeout ||
log_queue_check_items(self->queue, self->options->flush_lines, &partial_batch, &timeout_msec,
(LogQueuePushNotifyFunc) log_writer_schedule_update_watches, self, NULL))
{
/* flush_lines number of element is already available and throttle would permit us to send. */
log_writer_update_fd_callbacks(self, cond);
}
else if (partial_batch || timeout_msec)
{
/* few elements are available, but less than flush_lines, we need to start a timer to initiate a flush */
log_writer_update_fd_callbacks(self, 0);
self->flush_waiting_for_timeout = TRUE;
log_writer_arm_suspend_timer(self, (void (*)(void *)) log_writer_update_watches, timeout_msec ? timeout_msec : self->options->flush_timeout);
}
else
{
/* no elements or no throttle space, wait for a wakeup by the queue
* when the required number of items are added. see the
* log_queue_check_items and its parallel_push argument above
*/
log_writer_update_fd_callbacks(self, 0);
}
}
static void
log_reader_update_watches(LogReader *self)
{
gint fd;
GIOCondition cond;
gboolean free_to_send;
main_loop_assert_main_thread();
self->suspended = FALSE;
free_to_send = log_source_free_to_send(&self->super);
if (!free_to_send ||
self->immediate_check ||
log_proto_prepare(self->proto, &fd, &cond))
{
/* we disable all I/O related callbacks here because we either know
* that we can continue (e.g. immediate_check == TRUE) or we know
* that we can't continue even if data would be available (e.g.
* free_to_send == FALSE)
*/
self->immediate_check = FALSE;
if (iv_fd_registered(&self->fd_watch))
{
iv_fd_set_handler_in(&self->fd_watch, NULL);
iv_fd_set_handler_out(&self->fd_watch, NULL);
/* we disable the error handler too, as it might be
* triggered even when we don't want to read data
* (e.g. log_source_free_to_send() is FALSE).
*
* And at least on Linux, it may happen that EPOLLERR is
* set, while there's still data in the socket buffer. Thus
* in reaction to an EPOLLERR, we could possibly send
* further messages without validating the
* log_source_free_to_send() would allow us to, potentially
* overflowing our window (and causing a failed assertion in
* log_source_queue().
*/
iv_fd_set_handler_err(&self->fd_watch, NULL);
}
if (iv_timer_registered(&self->follow_timer))
iv_timer_unregister(&self->follow_timer);
if (free_to_send)
{
/* we have data in our input buffer, we need to start working
* on it immediately, without waiting for I/O events */
if (!iv_task_registered(&self->restart_task))
{
iv_task_register(&self->restart_task);
}
}
else
{
self->suspended = TRUE;
}
return;
}
if (iv_fd_registered(&self->fd_watch))
{
/* this branch is executed when our fd is connected to a non-file
* source (e.g. TCP, UDP socket). We set up I/O callbacks here.
* files cannot be polled using epoll, as it causes an I/O error
* (thus abort in ivykis).
*/
if (cond & G_IO_IN)
iv_fd_set_handler_in(&self->fd_watch, log_reader_io_process_input);
else
iv_fd_set_handler_in(&self->fd_watch, NULL);
if (cond & G_IO_OUT)
iv_fd_set_handler_out(&self->fd_watch, log_reader_io_process_input);
else
iv_fd_set_handler_out(&self->fd_watch, NULL);
if (cond & (G_IO_IN + G_IO_OUT))
iv_fd_set_handler_err(&self->fd_watch, log_reader_io_process_input);
else
iv_fd_set_handler_err(&self->fd_watch, NULL);
}
else
{
if (self->options->follow_freq > 0)
{
if (iv_timer_registered(&self->follow_timer))
iv_timer_unregister(&self->follow_timer);
iv_validate_now();
self->follow_timer.expires = iv_now;
timespec_add_msec(&self->follow_timer.expires, self->options->follow_freq);
iv_timer_register(&self->follow_timer);
}
else
{
/* NOTE: we don't need to unregister the timer here as follow_freq
* never changes during runtime, thus if ever it was registered that
* also means that we go into the if branch above. */
}
}
}
static gboolean
log_writer_fd_prepare(GSource *source,
gint *timeout)
{
LogWriterWatch *self = (LogWriterWatch *) source;
gint64 num_elements = log_queue_get_length(self->writer->queue);
GTimeVal now;
GIOCondition proto_cond;
self->pollfd.events = G_IO_ERR;
self->pollfd.revents = 0;
g_source_get_current_time(source, &now);
if (log_proto_prepare(self->proto, &self->pollfd.fd, &proto_cond, timeout))
return TRUE;
/* recalculate buckets */
if (self->writer->options->throttle > 0)
{
gint64 diff;
gint new_buckets;
/* throttling is enabled, calculate new buckets */
if (self->last_throttle_check.tv_sec != 0)
{
diff = g_time_val_diff(&now, &self->last_throttle_check);
}
else
{
diff = 0;
self->last_throttle_check = now;
}
new_buckets = (self->writer->options->throttle * diff) / G_USEC_PER_SEC;
if (new_buckets)
{
/* if new_buckets is zero, we don't save the current time as
* last_throttle_check. The reason is that new_buckets could be
* rounded to zero when only a minimal interval passes between
* poll iterations.
*/
self->writer->throttle_buckets = MIN(self->writer->options->throttle, self->writer->throttle_buckets + new_buckets);
self->last_throttle_check = now;
}
}
if (G_UNLIKELY(self->error_suspend))
{
*timeout = g_time_val_diff(&self->error_suspend_target, &now) / 1000;
if (*timeout <= 0)
{
msg_notice("Error suspend timeout has elapsed, attempting to write again",
evt_tag_int("fd", log_proto_get_fd(self->proto)),
NULL);
self->error_suspend = FALSE;
*timeout = -1;
}
else
{
return FALSE;
}
}
if ((self->writer->options->flush_lines == 0 && (!log_writer_throttling(self->writer) && num_elements != 0)) ||
(self->writer->options->flush_lines > 0 && (!log_writer_throttling(self->writer) && num_elements >= self->writer->options->flush_lines)))
{
/* we need to flush our buffers */
self->pollfd.events |= proto_cond;
}
else if (num_elements && !log_writer_throttling(self->writer))
{
/* our buffer does not contain enough elements to flush, but we do not
* want to wait more than flush_timeout time */
if (!self->flush_waiting_for_timeout)
{
/* start waiting */
*timeout = self->writer->options->flush_timeout;
g_source_get_current_time(source, &self->flush_target);
g_time_val_add(&self->flush_target, *timeout * 1000);
self->flush_waiting_for_timeout = TRUE;
}
else
{
glong to = g_time_val_diff(&self->flush_target, &now) / 1000;
if (to <= 0)
{
/* timeout elapsed, start polling again */
if (self->writer->flags & LW_ALWAYS_WRITABLE)
return TRUE;
self->pollfd.events = proto_cond;
}
else
{
*timeout = to;
}
}
return FALSE;
}
else
{
if (num_elements && log_writer_throttling(self->writer))
{
/* we are unable to send because of throttling, make sure that we
* wake up when the rate limits lets us send at least 1 message */
*timeout = (1000 / self->writer->options->throttle) + 1;
msg_debug("Throttling output",
evt_tag_int("wait", *timeout),
NULL);
}
}
if (self->writer->flags & LW_DETECT_EOF && (self->pollfd.events & G_IO_IN) == 0)
{
self->pollfd.events |= G_IO_HUP | G_IO_IN;
self->input_means_connection_broken = TRUE;
}
else
{
self->input_means_connection_broken = FALSE;
}
self->flush_waiting_for_timeout = FALSE;
if ((self->pollfd.events & G_IO_OUT) && (self->writer->flags & LW_ALWAYS_WRITABLE))
{
self->pollfd.revents = G_IO_OUT;
return TRUE;
}
return FALSE;
}
