Пример #1
0
static gboolean
afamqp_worker_insert(LogThrDestDriver *s)
{
    AMQPDestDriver *self = (AMQPDestDriver *)s;
    gboolean success;
    LogMessage *msg;
    LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;

    afamqp_dd_connect(self, TRUE);

    success = log_queue_pop_head(s->queue, &msg, &path_options, FALSE, FALSE);
    if (!success)
        return TRUE;

    msg_set_context(msg);
    success = afamqp_worker_publish (self, msg);
    msg_set_context(NULL);

    if (success)
    {
        stats_counter_inc(s->stored_messages);
        step_sequence_number(&self->seq_num);
        log_msg_ack(msg, &path_options);
        log_msg_unref(msg);
    }
    else
        log_queue_push_head(s->queue, msg, &path_options);

    return success;
}
Пример #2
0
/**
 * afsql_dd_insert_db:
 *
 * This function is running in the database thread
 *
 * Returns: FALSE to indicate that the connection should be closed and
 * this destination suspended for time_reopen() time.
 **/
static gboolean
afsql_dd_insert_db(AFSqlDestDriver *self)
{
  GString *table, *query_string;
  LogMessage *msg;
  gboolean success;
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;

  afsql_dd_connect(self);

  success = log_queue_pop_head(self->queue, &msg, &path_options, (self->flags & AFSQL_DDF_EXPLICIT_COMMITS), FALSE);
  if (!success)
    return TRUE;

  msg_set_context(msg);

  table = afsql_dd_validate_table(self, msg);
  if (!table)
    {
      /* If validate table is FALSE then close the connection and wait time_reopen time (next call) */
      msg_error("Error checking table, disconnecting from database, trying again shortly",
                evt_tag_int("time_reopen", self->time_reopen),
                NULL);
      msg_set_context(NULL);
      g_string_free(table, TRUE);
      return afsql_dd_insert_fail_handler(self, msg, &path_options);
    }

  query_string = afsql_dd_construct_query(self, table, msg);

  if (self->flush_lines_queued == 0 && !afsql_dd_begin_txn(self))
    return FALSE;

  success = afsql_dd_run_query(self, query_string->str, FALSE, NULL);
  if (success && self->flush_lines_queued != -1)
    {
      self->flush_lines_queued++;

      if (self->flush_lines && self->flush_lines_queued == self->flush_lines && !afsql_dd_commit_txn(self))
        return FALSE;
    }

  g_string_free(table, TRUE);
  g_string_free(query_string, TRUE);

  msg_set_context(NULL);

  if (!success)
    return afsql_dd_insert_fail_handler(self, msg, &path_options);

  /* we only ACK if each INSERT is a separate transaction */
  if ((self->flags & AFSQL_DDF_EXPLICIT_COMMITS) == 0)
    log_msg_ack(msg, &path_options);
  log_msg_unref(msg);
  step_sequence_number(&self->seq_num);
  self->failed_message_counter = 0;

  return TRUE;
}
Пример #3
0
void
log_threaded_dest_driver_message_accept(LogThrDestDriver *self,
                                        LogMessage *msg)
{
  self->retries.counter = 0;
  step_sequence_number(&self->seq_num);
  log_queue_ack_backlog(self->queue, 1);
  log_msg_unref(msg);
}
Пример #4
0
static gboolean
afmongodb_worker_insert (MongoDBDestDriver *self)
{
  gboolean success;
  guint8 *oid;
  LogMessage *msg;
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;

  afmongodb_dd_connect(self, TRUE);

  success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, FALSE);
  if (!success)
    return TRUE;

  msg_set_context(msg);

  bson_reset (self->bson);

  oid = mongo_util_oid_new_with_time (self->last_msg_stamp, self->seq_num);
  bson_append_oid (self->bson, "_id", oid);
  g_free (oid);

  value_pairs_walk(self->vp,
                   afmongodb_vp_obj_start,
                   afmongodb_vp_process_value,
                   afmongodb_vp_obj_end,
                   msg, self->seq_num, self->bson);
  bson_finish (self->bson);

  if (!mongo_sync_cmd_insert_n(self->conn, self->ns, 1,
                               (const bson **)&self->bson))
    {
      msg_error("Network error while inserting into MongoDB",
                evt_tag_int("time_reopen", self->time_reopen),
                NULL);
      success = FALSE;
    }

  msg_set_context(NULL);

  if (success)
    {
      stats_counter_inc(self->stored_messages);
      step_sequence_number(&self->seq_num);
      log_msg_ack(msg, &path_options);
      log_msg_unref(msg);
    }
  else
    {
      log_queue_push_head(self->queue, msg, &path_options);
    }

  return success;
}
Пример #5
0
static gboolean
afmongodb_worker_insert (MongoDBDestDriver *self)
{
  gboolean success;
  guint8 *oid;
  LogMessage *msg;
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;

  afmongodb_dd_connect(self, TRUE);

  g_mutex_lock(self->queue_mutex);
  log_queue_reset_parallel_push(self->queue);
  success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, FALSE);
  g_mutex_unlock(self->queue_mutex);
  if (!success)
    return TRUE;

  msg_set_context(msg);

  bson_reset (self->bson_sel);
  bson_reset (self->bson_upd);
  bson_reset (self->bson_set);

  oid = mongo_util_oid_new_with_time (self->last_msg_stamp, self->seq_num);
  bson_append_oid (self->bson_sel, "_id", oid);
  g_free (oid);
  bson_finish (self->bson_sel);

  value_pairs_foreach (self->vp, afmongodb_vp_foreach,
		       msg, self->seq_num, self->bson_set);

  bson_finish (self->bson_set);

  bson_append_document (self->bson_upd, "$set", self->bson_set);
  bson_finish (self->bson_upd);

  if (!mongo_sync_cmd_update (self->conn, self->ns, MONGO_WIRE_FLAG_UPDATE_UPSERT,
			      self->bson_sel, self->bson_upd))
    {
      msg_error ("Network error while inserting into MongoDB",
		 evt_tag_int("time_reopen", self->time_reopen),
		 NULL);
      success = FALSE;
    }

  msg_set_context(NULL);

  if (success)
    {
      stats_counter_inc(self->stored_messages);
      step_sequence_number(&self->seq_num);
      log_msg_ack(msg, &path_options);
      log_msg_unref(msg);
    }
  else
    {
      g_mutex_lock(self->queue_mutex);
      log_queue_push_head(self->queue, msg, &path_options);
      g_mutex_unlock(self->queue_mutex);
    }

  return success;
}
Пример #6
0
/**
 * afsql_dd_insert_db:
 *
 * This function is running in the database thread
 *
 * Returns: FALSE to indicate that the connection should be closed and
 * this destination suspended for time_reopen() time.
 **/
static gboolean
afsql_dd_insert_db(AFSqlDestDriver *self)
{
  GString *table = NULL;
  GString *insert_command = NULL;
  LogMessage *msg;
  gboolean success;
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;

  if (!afsql_dd_ensure_initialized_connection(self))
    return FALSE;

  /* connection established, try to insert a message */
  success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, self->flags & AFSQL_DDF_EXPLICIT_COMMITS);
  if (!success)
    return TRUE;

  msg_set_context(msg);

  table = afsql_dd_ensure_accessible_database_table(self, msg);

  if (!table)
    {
      success = FALSE;
      goto out;
    }

  if (afsql_dd_should_start_new_transaction(self) && !afsql_dd_begin_txn(self))
    {
      success = FALSE;
      goto out;
    }

  insert_command = afsql_dd_build_insert_command(self, msg, table);
  success = afsql_dd_run_query(self, insert_command->str, FALSE, NULL);

  if (success && self->flush_lines_queued != -1)
    {
      self->flush_lines_queued++;

      if (afsql_dd_should_commit_transaction(self) && !afsql_dd_commit_txn(self))
        {
          /* Assuming that in case of error, the queue is rewound by afsql_dd_commit_txn() */

          g_string_free(insert_command, TRUE);
          msg_set_context(NULL);

          return FALSE;
        }
    }

 out:

  if (table != NULL)
    g_string_free(table, TRUE);

  if (insert_command != NULL)
    g_string_free(insert_command, TRUE);

  msg_set_context(NULL);

  if (success)
    {
      log_msg_ack(msg, &path_options);
      log_msg_unref(msg);
      step_sequence_number(&self->seq_num);
      self->failed_message_counter = 0;
    }
  else
    {
      if (self->failed_message_counter < self->num_retries - 1)
        {
          if (!afsql_dd_handle_insert_row_error_depending_on_connection_availability(self, msg, &path_options))
            return FALSE;

          self->failed_message_counter++;
        }
      else
        {
          msg_error("Multiple failures while inserting this record into the database, message dropped",
                    evt_tag_int("attempts", self->num_retries),
                    NULL);
          stats_counter_inc(self->dropped_messages);
          log_msg_drop(msg, &path_options);
          self->failed_message_counter = 0;
          success = TRUE;
        }
    }

  return success;
}
Пример #7
0
static gboolean
perl_worker_eval(LogThrDestDriver *d)
{
  PerlDestDriver *self = (PerlDestDriver *)d;
  gboolean success, vp_ok;
  LogMessage *msg;
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
  PerlInterpreter *my_perl = self->perl;
  int count;
  HV *kvmap;
  gpointer args[3];
  dSP;

  success = log_queue_pop_head(self->super.queue, &msg, &path_options, FALSE, FALSE);
  if (!success)
    return TRUE;

  msg_set_context(msg);

  ENTER;
  SAVETMPS;

  PUSHMARK(SP);

  kvmap = newHV();

  args[0] = self->perl;
  args[1] = kvmap;
  args[2] = self;
  vp_ok = value_pairs_foreach(self->vp, perl_worker_vp_add_one,
                              msg, self->seq_num, LTZ_SEND,
                              &self->template_options,
                              args);
  if (!vp_ok && (self->template_options.on_error & ON_ERROR_DROP_MESSAGE))
    goto exit;

  XPUSHs(sv_2mortal(newRV_noinc((SV *)kvmap)));

  PUTBACK;

  count = call_pv(self->queue_func_name, G_EVAL | G_SCALAR);

  SPAGAIN;

  msg_set_context(NULL);

  if (SvTRUE(ERRSV))
    {
      msg_error("Error while calling a Perl function",
                evt_tag_str("driver", self->super.super.super.id),
                evt_tag_str("script", self->filename),
                evt_tag_str("function", self->queue_func_name),
                evt_tag_str("error-message", SvPV_nolen(ERRSV)),
                NULL);
      (void) POPs;
      success = FALSE;
    }

  if (count != 1)
    {
      msg_error("Too many values returned by a Perl function",
                evt_tag_str("driver", self->super.super.super.id),
                evt_tag_str("script", self->filename),
                evt_tag_str("function", self->queue_func_name),
                evt_tag_int("returned-values", count),
                evt_tag_int("expected-values", 1),
                NULL);
      success = FALSE;
    }
  else
    {
      int r = POPi;

      success = (r != 0);
    }

 exit:
  PUTBACK;
  FREETMPS;
  LEAVE;

  if (success && vp_ok)
    {
      stats_counter_inc(self->super.stored_messages);
      step_sequence_number(&self->seq_num);
      log_msg_ack(msg, &path_options);
      log_msg_unref(msg);
    }
  else
    {
      stats_counter_inc(self->super.dropped_messages);
      step_sequence_number(&self->seq_num);
      log_msg_ack(msg, &path_options);
      log_msg_unref(msg);
    }

  return success;
}
Пример #8
0
/**
 * afsql_dd_insert_db:
 *
 * This function is running in the database thread
 *
 * Returns: FALSE to indicate that the connection should be closed and
 * this destination suspended for time_reopen() time.
 **/
static gboolean
afsql_dd_insert_db(AFSqlDestDriver *self)
{
  GString *table, *query_string;
  LogMessage *msg;
  gboolean success;
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;

  afsql_dd_connect(self);

  g_mutex_lock(self->db_thread_mutex);

  /* FIXME: this is a workaround because of the non-proper locking semantics
   * of the LogQueue.  It might happen that the _queue() method sees 0
   * elements in the queue, while the thread is still busy processing the
   * previous message.  In that case arming the parallel push callback is
   * not needed and will cause assertions to fail.  This is ugly and should
   * be fixed by properly defining the "blocking" semantics of the LogQueue
   * object w/o having to rely on user-code messing with parallel push
   * callbacks. */
  log_queue_reset_parallel_push(self->queue);
  success = log_queue_pop_head(self->queue, &msg, &path_options, (self->flags & AFSQL_DDF_EXPLICIT_COMMITS), FALSE);
  g_mutex_unlock(self->db_thread_mutex);
  if (!success)
    return TRUE;

  msg_set_context(msg);

  table = afsql_dd_validate_table(self, msg);
  if (!table)
    {
      /* If validate table is FALSE then close the connection and wait time_reopen time (next call) */
      msg_error("Error checking table, disconnecting from database, trying again shortly",
                evt_tag_int("time_reopen", self->time_reopen),
                NULL);
      msg_set_context(NULL);
      g_string_free(table, TRUE);
      return afsql_dd_insert_fail_handler(self, msg, &path_options);
    }

  query_string = afsql_dd_construct_query(self, table, msg);

  if (self->flush_lines_queued == 0 && !afsql_dd_begin_txn(self))
    return FALSE;

  success = afsql_dd_run_query(self, query_string->str, FALSE, NULL);
  if (success && self->flush_lines_queued != -1)
    {
      self->flush_lines_queued++;

      if (self->flush_lines && self->flush_lines_queued == self->flush_lines && !afsql_dd_commit_txn(self, TRUE))
        return FALSE;
    }

  g_string_free(table, TRUE);
  g_string_free(query_string, TRUE);

  msg_set_context(NULL);

  if (!success)
    return afsql_dd_insert_fail_handler(self, msg, &path_options);

  /* we only ACK if each INSERT is a separate transaction */
  if ((self->flags & AFSQL_DDF_EXPLICIT_COMMITS) == 0)
    log_msg_ack(msg, &path_options);
  log_msg_unref(msg);
  step_sequence_number(&self->seq_num);
  self->failed_message_counter = 0;

  return TRUE;
}
Пример #9
0
static gboolean
redis_worker_insert(LogThrDestDriver *s)
{
  RedisDriver *self = (RedisDriver *)s;
  gboolean success;
  LogMessage *msg;
  LogPathOptions path_options = LOG_PATH_OPTIONS_INIT;
  redisReply *reply;
  const char *argv[5];
  size_t argvlen[5];
  int argc = 2;

  redis_dd_connect(self, TRUE);

  if (self->c->err)
    return FALSE;

  success = log_queue_pop_head(self->super.queue, &msg, &path_options, FALSE, FALSE);
  if (!success)
    return TRUE;

  msg_set_context(msg);

  log_template_format(self->key, msg, &self->template_options, LTZ_SEND,
                      self->seq_num, NULL, self->key_str);

  if (self->param1)
    log_template_format(self->param1, msg, &self->template_options, LTZ_SEND,
                        self->seq_num, NULL, self->param1_str);
  if (self->param2)
    log_template_format(self->param2, msg, &self->template_options, LTZ_SEND,
                        self->seq_num, NULL, self->param2_str);

  argv[0] = self->command->str;
  argvlen[0] = self->command->len;
  argv[1] = self->key_str->str;
  argvlen[1] = self->key_str->len;

  if (self->param1)
    {
      argv[2] = self->param1_str->str;
      argvlen[2] = self->param1_str->len;
      argc++;
    }

  if (self->param2)
    {
      argv[3] = self->param2_str->str;
      argvlen[3] = self->param2_str->len;
      argc++;
    }

  reply = redisCommandArgv(self->c, argc, argv, argvlen);

  msg_debug("REDIS command sent",
            evt_tag_str("driver", self->super.super.super.id),
            evt_tag_str("command", self->command->str),
            evt_tag_str("key", self->key_str->str),
            evt_tag_str("param1", self->param1_str->str),
            evt_tag_str("param2", self->param2_str->str),
            NULL);
  success = TRUE;

  freeReplyObject(reply);

  msg_set_context(NULL);

  if (success)
    {
      stats_counter_inc(self->super.stored_messages);
      step_sequence_number(&self->seq_num);
      log_msg_ack(msg, &path_options);
      log_msg_unref(msg);
    }
  else
    {
      log_queue_push_head(self->super.queue, msg, &path_options);
    }

  return success;
}