示例#1
0
static gboolean
afamqp_worker_insert(LogThrDestDriver *s)
{
    AMQPDestDriver *self = (AMQPDestDriver *)s;
    gboolean success;
    LogMessage *msg;
    LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;

    afamqp_dd_connect(self, TRUE);

    success = log_queue_pop_head(s->queue, &msg, &path_options, FALSE, FALSE);
    if (!success)
        return TRUE;

    msg_set_context(msg);
    success = afamqp_worker_publish (self, msg);
    msg_set_context(NULL);

    if (success)
    {
        stats_counter_inc(s->stored_messages);
        step_sequence_number(&self->seq_num);
        log_msg_ack(msg, &path_options);
        log_msg_unref(msg);
    }
    else
        log_queue_push_head(s->queue, msg, &path_options);

    return success;
}
示例#2
0
static inline void
afsql_dd_rollback_msg(AFSqlDestDriver *self, LogMessage *msg, LogPathOptions *path_options)
{
  if (self->flags & AFSQL_DDF_EXPLICIT_COMMITS)
    log_queue_rewind_backlog(self->queue, 1);
  else
    log_queue_push_head(self->queue, msg, path_options);
}
示例#3
0
static gboolean
afmongodb_worker_insert (MongoDBDestDriver *self)
{
  gboolean success;
  guint8 *oid;
  LogMessage *msg;
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;

  afmongodb_dd_connect(self, TRUE);

  success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, FALSE);
  if (!success)
    return TRUE;

  msg_set_context(msg);

  bson_reset (self->bson);

  oid = mongo_util_oid_new_with_time (self->last_msg_stamp, self->seq_num);
  bson_append_oid (self->bson, "_id", oid);
  g_free (oid);

  value_pairs_walk(self->vp,
                   afmongodb_vp_obj_start,
                   afmongodb_vp_process_value,
                   afmongodb_vp_obj_end,
                   msg, self->seq_num, self->bson);
  bson_finish (self->bson);

  if (!mongo_sync_cmd_insert_n(self->conn, self->ns, 1,
                               (const bson **)&self->bson))
    {
      msg_error("Network error while inserting into MongoDB",
                evt_tag_int("time_reopen", self->time_reopen),
                NULL);
      success = FALSE;
    }

  msg_set_context(NULL);

  if (success)
    {
      stats_counter_inc(self->stored_messages);
      step_sequence_number(&self->seq_num);
      log_msg_ack(msg, &path_options);
      log_msg_unref(msg);
    }
  else
    {
      log_queue_push_head(self->queue, msg, &path_options);
    }

  return success;
}
示例#4
0
static gpointer
_threaded_consume(gpointer st)
{
  LogQueue *q = (LogQueue *) st;
  LogMessage *msg;
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
  gint loops = 0;
  gint msg_count = 0;

  /* just to make sure time is properly cached */
  iv_init();

  while (msg_count < MESSAGES_SUM)
    {
      gint slept = 0;
      msg = NULL;

      while((msg = log_queue_pop_head(q, &path_options)) == NULL)
        {
          struct timespec ns;

          /* sleep 1 msec */
          ns.tv_sec = 0;
          ns.tv_nsec = 1000000;
          nanosleep(&ns, NULL);
          slept++;
          if (slept > 10000)
            {
              /* slept for more than 10 seconds */
              fprintf(stderr, "The wait for messages took too much time, loops=%d, msg_count=%d\n", loops, msg_count);
              return GUINT_TO_POINTER(1);
            }
        }

      if ((loops % 10) == 0)
        {
          /* push the message back to the queue */
          log_queue_push_head(q, msg, &path_options);
        }
      else
        {
          log_msg_ack(msg, &path_options, AT_PROCESSED);
          log_msg_unref(msg);
          msg_count++;
        }
      loops++;
    }

  iv_deinit();
  return NULL;
}
示例#5
0
static inline gboolean
afsql_dd_handle_insert_row_error_depending_on_connection_availability(AFSqlDestDriver *self,
                                                                      LogMessage *msg,
                                                                      LogPathOptions *path_options)
{
  const gchar *dbi_error, *error_message;

  if (dbi_conn_ping(self->dbi_ctx) == 1)
    {
      log_queue_push_head(self->queue, msg, path_options);
      return TRUE;
    }

  if (afsql_dd_is_transaction_handling_enabled(self))
    {
      error_message = "SQL connection lost in the middle of a transaction,"
                      " rewinding backlog and starting again";
      afsql_dd_handle_transaction_error(self);
    }
  else
    {
      error_message = "Error, no SQL connection after failed query attempt";
      log_queue_push_head(self->queue, msg, path_options);
    }

  dbi_conn_error(self->dbi_ctx, &dbi_error);
  msg_error(error_message,
            evt_tag_str("type", self->type),
            evt_tag_str("host", self->host),
            evt_tag_str("port", self->port),
            evt_tag_str("username", self->user),
            evt_tag_str("database", self->database),
            evt_tag_str("error", dbi_error),
            NULL);

  return FALSE;
}
示例#6
0
static gboolean
afsql_dd_insert_fail_handler(AFSqlDestDriver *self, LogMessage *msg,
                             LogPathOptions *path_options)
{
  if (self->failed_message_counter < self->num_retries - 1)
    {
      log_queue_push_head(self->queue, msg, path_options);

      /* database connection status sanity check after failed query */
      if (dbi_conn_ping(self->dbi_ctx) != 1)
        {
          const gchar *dbi_error;

          dbi_conn_error(self->dbi_ctx, &dbi_error);
          msg_error("Error, no SQL connection after failed query attempt",
                    evt_tag_str("type", self->type),
                    evt_tag_str("host", self->host),
                    evt_tag_str("port", self->port),
                    evt_tag_str("username", self->user),
                    evt_tag_str("database", self->database),
                    evt_tag_str("error", dbi_error),
                    NULL);
          return FALSE;
        }

      self->failed_message_counter++;
      return FALSE;
    }

  msg_error("Multiple failures while inserting this record into the database, message dropped",
            evt_tag_int("attempts", self->num_retries),
            NULL);
  stats_counter_inc(self->dropped_messages);
  log_msg_drop(msg, path_options);
  self->failed_message_counter = 0;
  return TRUE;
}
示例#7
0
static gboolean
afmongodb_worker_insert (MongoDBDestDriver *self)
{
  gboolean success;
  guint8 *oid;
  LogMessage *msg;
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;

  afmongodb_dd_connect(self, TRUE);

  g_mutex_lock(self->queue_mutex);
  log_queue_reset_parallel_push(self->queue);
  success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, FALSE);
  g_mutex_unlock(self->queue_mutex);
  if (!success)
    return TRUE;

  msg_set_context(msg);

  bson_reset (self->bson_sel);
  bson_reset (self->bson_upd);
  bson_reset (self->bson_set);

  oid = mongo_util_oid_new_with_time (self->last_msg_stamp, self->seq_num);
  bson_append_oid (self->bson_sel, "_id", oid);
  g_free (oid);
  bson_finish (self->bson_sel);

  value_pairs_foreach (self->vp, afmongodb_vp_foreach,
		       msg, self->seq_num, self->bson_set);

  bson_finish (self->bson_set);

  bson_append_document (self->bson_upd, "$set", self->bson_set);
  bson_finish (self->bson_upd);

  if (!mongo_sync_cmd_update (self->conn, self->ns, MONGO_WIRE_FLAG_UPDATE_UPSERT,
			      self->bson_sel, self->bson_upd))
    {
      msg_error ("Network error while inserting into MongoDB",
		 evt_tag_int("time_reopen", self->time_reopen),
		 NULL);
      success = FALSE;
    }

  msg_set_context(NULL);

  if (success)
    {
      stats_counter_inc(self->stored_messages);
      step_sequence_number(&self->seq_num);
      log_msg_ack(msg, &path_options);
      log_msg_unref(msg);
    }
  else
    {
      g_mutex_lock(self->queue_mutex);
      log_queue_push_head(self->queue, msg, &path_options);
      g_mutex_unlock(self->queue_mutex);
    }

  return success;
}
示例#8
0
static gboolean
redis_worker_insert(LogThrDestDriver *s)
{
  RedisDriver *self = (RedisDriver *)s;
  gboolean success;
  LogMessage *msg;
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
  redisReply *reply;
  const char *argv[5];
  size_t argvlen[5];
  int argc = 2;

  redis_dd_connect(self, TRUE);

  if (self->c->err)
    return FALSE;

  success = log_queue_pop_head(self->super.queue, &msg, &path_options, FALSE, FALSE);
  if (!success)
    return TRUE;

  msg_set_context(msg);

  log_template_format(self->key, msg, &self->template_options, LTZ_SEND,
                      self->seq_num, NULL, self->key_str);

  if (self->param1)
    log_template_format(self->param1, msg, &self->template_options, LTZ_SEND,
                        self->seq_num, NULL, self->param1_str);
  if (self->param2)
    log_template_format(self->param2, msg, &self->template_options, LTZ_SEND,
                        self->seq_num, NULL, self->param2_str);

  argv[0] = self->command->str;
  argvlen[0] = self->command->len;
  argv[1] = self->key_str->str;
  argvlen[1] = self->key_str->len;

  if (self->param1)
    {
      argv[2] = self->param1_str->str;
      argvlen[2] = self->param1_str->len;
      argc++;
    }

  if (self->param2)
    {
      argv[3] = self->param2_str->str;
      argvlen[3] = self->param2_str->len;
      argc++;
    }

  reply = redisCommandArgv(self->c, argc, argv, argvlen);

  msg_debug("REDIS command sent",
            evt_tag_str("driver", self->super.super.super.id),
            evt_tag_str("command", self->command->str),
            evt_tag_str("key", self->key_str->str),
            evt_tag_str("param1", self->param1_str->str),
            evt_tag_str("param2", self->param2_str->str),
            NULL);
  success = TRUE;

  freeReplyObject(reply);

  msg_set_context(NULL);

  if (success)
    {
      stats_counter_inc(self->super.stored_messages);
      step_sequence_number(&self->seq_num);
      log_msg_ack(msg, &path_options);
      log_msg_unref(msg);
    }
  else
    {
      log_queue_push_head(self->super.queue, msg, &path_options);
    }

  return success;
}