static void
_prepare_eof_test(LogQueueDiskReliable *dq, LogMessage **msg1, LogMessage **msg2)
{
  LogPathOptions local_options = LOG_PATH_OPTIONS_INIT;
  gint64 start_pos = TEST_DISKQ_SIZE;

  *msg1 = log_msg_new_mark();
  *msg2 = log_msg_new_mark();

  (*msg1)->ack_func = _dummy_ack;
  log_msg_add_ack(*msg1, &local_options);

  (*msg2)->ack_func = _dummy_ack;
  log_msg_add_ack(*msg2, &local_options);

  dq->super.qdisk->hdr->write_head = start_pos;
  dq->super.qdisk->hdr->read_head = QDISK_RESERVED_SPACE + mark_message_serialized_size + 1;
  dq->super.qdisk->hdr->backlog_head = dq->super.qdisk->hdr->read_head;

  log_queue_push_tail(&dq->super.super, *msg1, &local_options);
  log_queue_push_tail(&dq->super.super, *msg2, &local_options);

  assert_gint(dq->qreliable->length, NUMBER_MESSAGES_IN_QUEUE(2), ASSERTION_ERROR("Messages aren't in qreliable"));
  assert_gint64(dq->super.qdisk->hdr->write_head, QDISK_RESERVED_SPACE + mark_message_serialized_size, ASSERTION_ERROR("Bad write head"));
  assert_gint(num_of_ack, 0, ASSERTION_ERROR("Messages are acked"));

  dq->super.qdisk->hdr->read_head = start_pos;
  dq->super.qdisk->hdr->backlog_head = dq->super.qdisk->hdr->read_head;

}
/*
 * The method make the following situation
 * the backlog contains 6 messages
 * the qbacklog contains 3 messages,
 * but messages in qbacklog are the end of the backlog
 */
void
_prepare_rewind_backlog_test(LogQueueDiskReliable *dq, gint64 *start_pos)
{
  gint i;

  for (i = 0; i < 8; i++)
    {
      LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
      LogMessage *mark_message;
      mark_message = log_msg_new_mark();
      mark_message->ack_func = _dummy_ack;
      log_queue_push_tail(&dq->super.super, mark_message, &path_options);
    }

  /* Lets read the messages and leave them in the backlog */
  for (i = 0; i < 8; i++)
    {
      LogMessage *msg;
      LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
      msg = log_queue_pop_head(&dq->super.super, &path_options);
      log_msg_unref(msg);
    }

  /* Ack the messages which are not in the qbacklog */
  log_queue_ack_backlog(&dq->super.super, 5);
  assert_gint(dq->qbacklog->length, NUMBER_MESSAGES_IN_QUEUE(3), ASSERTION_ERROR("Incorrect number of items in the qbacklog"));

  *start_pos = dq->super.qdisk->hdr->read_head;

  /* Now write 3 more messages and read them from buffer
   * the number of messages in the qbacklog should not be changed
   * The backlog should contain 6 messages
   * from these 6 messages 3 messages are cached in the qbacklog
   * No readable messages are in the queue
   */
  for (i = 0; i < 3; i++)
    {
      LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
      LogMessage *mark_message;
      mark_message = log_msg_new_mark();
      mark_message->ack_func = _dummy_ack;
      log_queue_push_tail(&dq->super.super, mark_message, &path_options);
      mark_message = log_queue_pop_head(&dq->super.super, &path_options);
      assert_gint(dq->qreliable->length, 0,
          ASSERTION_ERROR("Incorrect number of items in the qreliable"));
      assert_gint(dq->qbacklog->length, NUMBER_MESSAGES_IN_QUEUE(3),
          ASSERTION_ERROR("Incorrect number of items in the qbacklog"));
      log_msg_unref(mark_message);
    }
  assert_gint(dq->super.qdisk->hdr->backlog_len, 6,
      ASSERTION_ERROR("Incorrect number of messages in the backlog"));
  assert_gint(dq->super.qdisk->hdr->length, 0,
      ASSERTION_ERROR("Reliable diskq isn't empty"));
}
  static void
log_writer_queue(LogPipe *s, LogMessage *lm, const LogPathOptions *path_options)
{
  LogWriter *self = (LogWriter *) s;

  if (self->options->suppress > 0 && log_writer_last_msg_check(self, lm, path_options))
    return;

  stats_counter_inc(self->processed_messages);
  if (!log_queue_push_tail(self->queue, lm, path_options))
  {
    /* drop incoming message, we must ack here, otherwise the sender might
     * block forever, however this should not happen unless the sum of
     * window_sizes of sources feeding this writer exceeds log_fifo_size
     * or if flow control is not turned on.
     */

    /* we don't send a message here since the system is draining anyway */

    stats_counter_inc(self->dropped_messages);
    msg_debug("Destination queue full, dropping message",
        evt_tag_int("queue_len", log_queue_get_length(self->queue)),
        evt_tag_int("mem_fifo_size", self->options->mem_fifo_size),
        NULL);
    log_msg_drop(lm, path_options);
    return;
  }
}
static void
test_rewind_over_eof(LogQueueDiskReliable *dq)
{
  LogMessage *msg3 = log_msg_new_mark();
  LogMessage *read_message3;

  LogPathOptions local_options = LOG_PATH_OPTIONS_INIT;
  msg3->ack_func = _dummy_ack;

  log_queue_push_tail(&dq->super.super, msg3, &local_options);
  gint64 previous_read_head = dq->super.qdisk->hdr->read_head;
  read_message3 = log_queue_pop_head(&dq->super.super, &local_options);
  assert_true(read_message3 != NULL, ASSERTION_ERROR("Can't read message from queue"));
  assert_gint(dq->super.qdisk->hdr->read_head, dq->super.qdisk->hdr->write_head, ASSERTION_ERROR("Read head in bad position"));

  assert_true(msg3 == read_message3, ASSERTION_ERROR("Message 3 isn't read from qreliable"));
  log_msg_unref(read_message3);

  log_queue_rewind_backlog(&dq->super.super, 1);

  assert_gint(dq->super.qdisk->hdr->read_head, previous_read_head, ASSERTION_ERROR("Read head is corrupted"));

  read_message3 = log_queue_pop_head(&dq->super.super, &local_options);
  assert_true(read_message3 != NULL, ASSERTION_ERROR("Can't read message from queue"));
  assert_gint(dq->super.qdisk->hdr->read_head, dq->super.qdisk->hdr->write_head, ASSERTION_ERROR("Read head in bad position"));
  assert_true(msg3 == read_message3, ASSERTION_ERROR("Message 3 isn't read from qreliable"));

  log_msg_drop(msg3, &local_options, AT_PROCESSED);
}
示例#5
0
gpointer
threaded_feed(gpointer st)
{
  LogQueue *q = (LogQueue *) st;
  char *msg_str = "<155>2006-02-11T10:34:56+01:00 bzorp syslog-ng[23323]: árvíztűrőtükörfúrógép";
  gint i;
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
  LogMessage *msg;

  for (i = 0; i < 100000; i++)
    {
      msg = log_msg_new(msg_str, strlen(msg_str), g_sockaddr_inet_new("10.10.10.10", 1010), 0, NULL, -1);
      log_msg_add_ack(msg, &path_options);
      msg->ack_func = test_ack;

      g_static_mutex_lock(&threaded_lock);
      if (!log_queue_push_tail(q, msg, &path_options))
        {
          fprintf(stderr, "Queue unable to consume enough messages: %d\n", fed_messages);
          return GUINT_TO_POINTER(1);
        }
      g_static_mutex_unlock(&threaded_lock);
    }
  return NULL;
}
示例#6
0
/*
 * NOTE: suppress_lock must be held.
 */
static void
log_writer_last_msg_flush(LogWriter *self)
{
  LogMessage *m;
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
  gchar buf[1024];
  gssize len;
  const gchar *p;

  msg_debug("Suppress timer elapsed, emitting suppression summary", 
            NULL);

  m = log_msg_new_empty();
  m->timestamps[LM_TS_STAMP] = m->timestamps[LM_TS_RECVD];
  m->pri = self->last_msg->pri;
  m->flags = LF_INTERNAL | LF_LOCAL;

  p = log_msg_get_value(self->last_msg, LM_V_HOST, &len);
  log_msg_set_value(m, LM_V_HOST, p, len);
  p = log_msg_get_value(self->last_msg, LM_V_PROGRAM, &len);
  log_msg_set_value(m, LM_V_PROGRAM, p, len);

  len = g_snprintf(buf, sizeof(buf), "Last message '%.20s' repeated %d times, suppressed by syslog-ng on %s",
                   log_msg_get_value(self->last_msg, LM_V_MESSAGE, NULL),
                   self->last_msg_count,
                   get_local_hostname(NULL));
  log_msg_set_value(m, LM_V_MESSAGE, buf, len);

  path_options.ack_needed = FALSE;

  log_queue_push_tail(self->queue, m, &path_options);
  log_writer_last_msg_release(self);
}
示例#7
0
static void
afmongodb_dd_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options, gpointer user_data)
{
  MongoDBDestDriver *self = (MongoDBDestDriver *)s;
  gboolean queue_was_empty;
  LogPathOptions local_options;

  if (!path_options->flow_control_requested)
    path_options = log_msg_break_ack(msg, path_options, &local_options);

  g_mutex_lock(self->queue_mutex);
  self->last_msg_stamp = cached_g_current_time_sec ();
  queue_was_empty = log_queue_get_length(self->queue) == 0;
  g_mutex_unlock(self->queue_mutex);

  log_msg_add_ack(msg, path_options);
  log_queue_push_tail(self->queue, log_msg_ref(msg), path_options);

  g_mutex_lock(self->suspend_mutex);
  if (queue_was_empty && !self->writer_thread_suspended)
    {
      g_mutex_lock(self->queue_mutex);
      log_queue_set_parallel_push(self->queue, 1, afmongodb_dd_queue_notify, self, NULL);
      g_mutex_unlock(self->queue_mutex);
    }
  g_mutex_unlock(self->suspend_mutex);
  log_dest_driver_queue_method(s, msg, path_options, user_data);
}
示例#8
0
static void
afamqp_dd_queue(LogPipe *s, LogMessage *msg,
                const LogPathOptions *path_options, gpointer user_data)
{
  AMQPDestDriver *self = (AMQPDestDriver *) s;
  LogPathOptions local_options;

  if (!path_options->flow_control_requested)
    path_options = log_msg_break_ack(msg, path_options, &local_options);

  log_msg_add_ack(msg, path_options);
  log_queue_push_tail(self->queue, msg, path_options);
  log_dest_driver_queue_method(s, msg, path_options, user_data);
}
示例#9
0
static void
afmongodb_dd_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options, gpointer user_data)
{
  MongoDBDestDriver *self = (MongoDBDestDriver *)s;
  LogPathOptions local_options;

  if (!path_options->flow_control_requested)
    path_options = log_msg_break_ack(msg, path_options, &local_options);

  self->last_msg_stamp = cached_g_current_time_sec ();

  log_msg_add_ack(msg, path_options);
  log_queue_push_tail(self->queue, log_msg_ref(msg), path_options);

  log_dest_driver_queue_method(s, msg, path_options, user_data);
}
gpointer
threaded_feed(gpointer args)
{
  LogQueue *q = args;
  char *msg_str = "<155>2006-02-11T10:34:56+01:00 bzorp syslog-ng[23323]: árvíztűrőtükörfúrógép";
  gint msg_len = strlen(msg_str);
  gint i;
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
  LogMessage *msg, *tmpl;
  GTimeVal start, end;
  GSockAddr *sa;
  glong diff;

  iv_init();
  
  /* emulate main loop for LogQueue */
  main_loop_worker_thread_start(NULL);

  sa = g_sockaddr_inet_new("10.10.10.10", 1010);
  tmpl = log_msg_new(msg_str, msg_len, sa, &parse_options);
  g_sockaddr_unref(sa);

  g_get_current_time(&start);
  for (i = 0; i < MESSAGES_PER_FEEDER; i++)
    {
      msg = log_msg_clone_cow(tmpl, &path_options);
      log_msg_add_ack(msg, &path_options);
      msg->ack_func = test_ack;

      log_queue_push_tail(q, msg, &path_options);
      
      if ((i & 0xFF) == 0)
        main_loop_worker_invoke_batch_callbacks();
    }
  main_loop_worker_invoke_batch_callbacks();
  g_get_current_time(&end);
  diff = g_time_val_diff(&end, &start);
  g_static_mutex_lock(&tlock);
  sum_time += diff;
  g_static_mutex_unlock(&tlock);
  log_msg_unref(tmpl);
  iv_deinit();
  main_loop_worker_thread_stop();
  return NULL;
}
示例#11
0
static void
log_threaded_dest_driver_queue(LogPipe *s, LogMessage *msg,
                               const LogPathOptions *path_options,
                               gpointer user_data)
{
  LogThrDestDriver *self = (LogThrDestDriver *)s;
  LogPathOptions local_options;

  if (!path_options->flow_control_requested)
    path_options = log_msg_break_ack(msg, path_options, &local_options);

  if (self->queue_method)
    self->queue_method(self);

  log_msg_add_ack(msg, path_options);
  log_queue_push_tail(self->queue, log_msg_ref(msg), path_options);

  log_dest_driver_queue_method(s, msg, path_options, user_data);
}
示例#12
0
  static void
log_writer_last_msg_flush(LogWriter *self)
{
  LogMessage *m;
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
  gchar hostname[256];
  gchar buf[1024];
  gssize len;
  const gchar *p;

  msg_debug("Suppress timer elapsed, emitting suppression summary", 
      NULL);

  getlonghostname(hostname, sizeof(hostname));
  m = log_msg_new_empty();
  m->timestamps[LM_TS_STAMP] = m->timestamps[LM_TS_RECVD];
  m->pri = self->last_msg->pri;
  m->flags = LF_INTERNAL | LF_LOCAL;

  p = log_msg_get_value(self->last_msg, LM_V_HOST, &len);
  log_msg_set_value(m, LM_V_HOST, p, len);
  p = log_msg_get_value(self->last_msg, LM_V_PROGRAM, &len);
  log_msg_set_value(m, LM_V_PROGRAM, p, len);

  len = g_snprintf(buf, sizeof(buf), "Last message '%.20s' repeated %d times, supressed by syslog-ng on %s",
      log_msg_get_value(self->last_msg, LM_V_MESSAGE, NULL),
      self->last_msg_count,
      hostname);
  log_msg_set_value(m, LM_V_MESSAGE, buf, len);

  path_options.flow_control = FALSE;
  if (!log_queue_push_tail(self->queue, m, &path_options))
  {
    stats_counter_inc(self->dropped_messages);
    msg_debug("Destination queue full, dropping suppressed message",
        evt_tag_int("queue_len", log_queue_get_length(self->queue)),
        evt_tag_int("mem_fifo_size", self->options->mem_fifo_size),
        NULL);
    log_msg_drop(m, &path_options);
  }

  log_writer_last_msg_release(self);
}
示例#13
0
/* NOTE: runs in the reader thread */
static void
log_writer_queue(LogPipe *s, LogMessage *lm, const LogPathOptions *path_options, gpointer user_data)
{
  LogWriter *self = (LogWriter *) s;
  LogPathOptions local_options;

  if (!path_options->flow_control_requested &&
      (self->suspended || !(self->flags & LW_SOFT_FLOW_CONTROL)))
    {
      /* NOTE: this code ACKs the message back if there's a write error in
       * order not to hang the client in case of a disk full */

      path_options = log_msg_break_ack(lm, path_options, &local_options);
    }
  if (self->options->suppress > 0 && log_writer_last_msg_check(self, lm, path_options))
    return;

  stats_counter_inc(self->processed_messages);
  log_queue_push_tail(self->queue, lm, path_options);
}
示例#14
0
static gpointer
_threaded_feed(gpointer args)
{
  LogQueue *q = args;
  gint i;
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
  LogMessage *msg, *tmpl;
  GTimeVal start, end;
  glong diff;

  iv_init();

  /* emulate main loop for LogQueue */
  main_loop_worker_thread_start(NULL);

  tmpl = log_msg_new_empty();

  g_get_current_time(&start);
  for (i = 0; i < MESSAGES_PER_FEEDER; i++)
    {
      msg = log_msg_clone_cow(tmpl, &path_options);
      log_msg_add_ack(msg, &path_options);
      msg->ack_func = test_ack;

      log_queue_push_tail(q, msg, &path_options);

      if ((i & 0xFF) == 0)
        main_loop_worker_invoke_batch_callbacks();
    }
  main_loop_worker_invoke_batch_callbacks();
  g_get_current_time(&end);
  diff = g_time_val_diff(&end, &start);
  g_static_mutex_lock(&tlock);
  sum_time += diff;
  g_static_mutex_unlock(&tlock);
  log_msg_unref(tmpl);
  main_loop_worker_thread_stop();
  iv_deinit();
  return NULL;
}
示例#15
0
/* NOTE: runs in the reader thread */
static void
log_writer_queue(LogPipe *s, LogMessage *lm, const LogPathOptions *path_options, gpointer user_data)
{
  LogWriter *self = (LogWriter *) s;
  LogPathOptions local_options;
  gint mark_mode = self->options->mark_mode;

  if (!path_options->flow_control_requested &&
      (self->suspended || !(self->flags & LW_SOFT_FLOW_CONTROL)))
    {
      /* NOTE: this code ACKs the message back if there's a write error in
       * order not to hang the client in case of a disk full */

      path_options = log_msg_break_ack(lm, path_options, &local_options);
    }

  if (log_writer_is_msg_suppressed(self, lm))
    {
      log_msg_drop(lm, path_options);
      return;
    }

  if (mark_mode != MM_INTERNAL && (lm->flags & LF_INTERNAL) && (lm->flags & LF_MARK))
    {
      /* drop MARK messages generated by internal() in case our mark-mode != internal */
      log_msg_drop(lm, path_options);
      return;
    }

  if (mark_mode == MM_DST_IDLE || (mark_mode == MM_HOST_IDLE && !(lm->flags & LF_LOCAL)))
    {
      /* in dst-idle and host-idle most, messages postpone the MARK itself */
      log_writer_postpone_mark_timer(self);
    }

  stats_counter_inc(self->processed_messages);
  log_queue_push_tail(self->queue, lm, path_options);
}
示例#16
0
/* this is the callback function that gets called when the MARK timeout
 * elapsed. It runs in the main thread.
 */
static void
log_writer_mark_timeout(void *cookie)
{
  LogWriter *self = (LogWriter *)cookie;
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
  const gchar *hostname;
  gsize hostname_len;
  LogMessage *msg;

  main_loop_assert_main_thread();

  msg = log_msg_new_mark();
  /* timeout: there was no new message on the writer or it is in periodical mode */
  hostname = resolve_sockaddr_to_hostname(&hostname_len, msg->saddr, &self->options->host_resolve_options);

  log_msg_set_value(msg, LM_V_HOST, hostname, hostname_len);

  /* set the current time in the message stamp */
  msg->timestamps[LM_TS_STAMP] = msg->timestamps[LM_TS_RECVD];

  if (!log_writer_is_msg_suppressed(self, msg))
    {
      log_queue_push_tail(self->queue, msg, &path_options);
      stats_counter_inc(self->processed_messages);
    }
  else
    {
      log_msg_drop(msg, &path_options);
    }

  /* we need to issue another MARK in all mark-mode cases that already
   * triggered this callback (dst-idle, host-idle, periodical).  The
   * original setup of the timer is at a different location:
   *   - log_writer_queue() for "*-idle" modes
   *   - log_writer_init() for periodical mode
   */
  log_writer_postpone_mark_timer(self);
}
void
feed_some_messages(LogQueue **q, int n)
{
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
  LogMessage *msg;
  gint i;

  path_options.ack_needed = (*q)->use_backlog;
  for (i = 0; i < n; i++)
    {
      char *msg_str = "<155>2006-02-11T10:34:56+01:00 bzorp syslog-ng[23323]: árvíztűrőtükörfúrógép";
      GSockAddr *sa;

      sa = g_sockaddr_inet_new("10.10.10.10", 1010);
      msg = log_msg_new(msg_str, strlen(msg_str), sa, &parse_options);
      g_sockaddr_unref(sa);
      log_msg_add_ack(msg, &path_options);
      msg->ack_func = test_ack;
      log_queue_push_tail((*q), msg, &path_options);
      fed_messages++;
    }

}
示例#18
0
void
feed_some_messages(LogQueue **q, int n, gboolean flow_control)
{
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
  LogMessage *msg;
  gint i;

  path_options.flow_control = flow_control;
  for (i = 0; i < n; i++)
    {
      char *msg_str = "<155>2006-02-11T10:34:56+01:00 bzorp syslog-ng[23323]: árvíztűrőtükörfúrógép";

      msg = log_msg_new(msg_str, strlen(msg_str), g_sockaddr_inet_new("10.10.10.10", 1010), 0, NULL, -1, 0xFFFF);
      log_msg_add_ack(msg, &path_options);
      msg->ack_func = test_ack;
      if (!log_queue_push_tail((*q), msg, &path_options))
        {
          fprintf(stderr, "Queue unable to consume enough messages: %d\n", fed_messages);
          exit(1);
        }
      fed_messages++;
    }

}